From 10893b9b33e4d549a01e4864030e980013fd68b2 Mon Sep 17 00:00:00 2001
From: Raja Subramanian <raja.gobi@tutanota.com>
Date: Tue, 22 Aug 2023 00:03:37 +0530
Subject: [PATCH] Store referenceLayerSpatial in Forwarder state. (#1986)

When restoring state, reference layer could change before this change.
That meant the time stamp base would change and cause jumps.

But, the solution in this change to store the reference layer state
and restoring it has a different issue. It is possible that the reference is
layer 2 (HIGH) for example. On a migration when the down track has
to re-attach and resume to a moved up stream track, it is possible that
layer 2 is not published due to bandwidth constraint after publisher
migrates to new node. In that case, the stream cannot be resumed as
time stamp adjustment cannot be calculated.

An option is to set referenceSpatialLayer always at layer 0 (LOW).
But, that also has a couple of issues
- Browsers like FF have shown issues with layer mapping.
- Layer 0 is lowest bit rate. So, it will have RTCP at lower frequency.
  That could introduce a slight latency in stream start as we need
  RTCP sender report to calculate the time stamp.

Open to ideas on how to handle this better.
---
 pkg/sfu/forwarder.go | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go
index 1d5c5ac7f..88876d307 100644
--- a/pkg/sfu/forwarder.go
+++ b/pkg/sfu/forwarder.go
@@ -154,12 +154,13 @@ type TranslationParams struct {
 // -------------------------------------------------------------------
 
 type ForwarderState struct {
-	Started      bool
-	PreStartTime time.Time
-	FirstTS      uint32
-	RefTSOffset  uint32
-	RTP          RTPMungerState
-	Codec        interface{}
+	Started               bool
+	ReferenceLayerSpatial int32
+	PreStartTime          time.Time
+	FirstTS               uint32
+	RefTSOffset           uint32
+	RTP                   RTPMungerState
+	Codec                 interface{}
 }
 
 func (f ForwarderState) String() string {
@@ -168,8 +169,9 @@ func (f ForwarderState) String() string {
 	case codecmunger.VP8State:
 		codecString = codecState.String()
 	}
-	return fmt.Sprintf("ForwarderState{started: %v, preStartTime: %s, firstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}",
+	return fmt.Sprintf("ForwarderState{started: %v, referenceLayerSpatial: %d, preStartTime: %s, firstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}",
 		f.Started,
+		f.ReferenceLayerSpatial,
 		f.PreStartTime.String(),
 		f.FirstTS,
 		f.RefTSOffset,
@@ -330,12 +332,13 @@ func (f *Forwarder) GetState() ForwarderState {
 	}
 
 	return ForwarderState{
-		Started:      f.started,
-		PreStartTime: f.preStartTime,
-		FirstTS:      f.firstTS,
-		RefTSOffset:  f.refTSOffset,
-		RTP:          f.rtpMunger.GetLast(),
-		Codec:        f.codecMunger.GetState(),
+		Started:               f.started,
+		ReferenceLayerSpatial: f.referenceLayerSpatial,
+		PreStartTime:          f.preStartTime,
+		FirstTS:               f.firstTS,
+		RefTSOffset:           f.refTSOffset,
+		RTP:                   f.rtpMunger.GetLast(),
+		Codec:                 f.codecMunger.GetState(),
 	}
 }
 
@@ -351,6 +354,7 @@ func (f *Forwarder) SeedState(state ForwarderState) {
 	f.codecMunger.SeedState(state.Codec)
 
 	f.started = true
+	f.referenceLayerSpatial = state.ReferenceLayerSpatial
 	f.preStartTime = state.PreStartTime
 	f.firstTS = state.FirstTS
 	f.refTSOffset = state.RefTSOffset
@@ -1451,11 +1455,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 		)
 	}
 
-	if f.referenceLayerSpatial == buffer.InvalidLayerSpatial {
-		// on a resume, reference layer may not be set, so only set when it is invalid
-		f.referenceLayerSpatial = layer
-	}
-
 	// Compute how much time passed between the previous forwarded packet
 	// and the current incoming (to be forwarded) packet and calculate
 	// timestamp offset on source change.