diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 430bb882f..8eb026205 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -988,7 +988,8 @@ func (d *DownTrack) IsDeficient() bool { } func (d *DownTrack) BandwidthRequested() int64 { - return d.forwarder.BandwidthRequested() + _, brs := d.receiver.GetLayeredBitrate() + return d.forwarder.BandwidthRequested(brs) } func (d *DownTrack) DistanceToDesired() float64 { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b203c27da..94d764bdd 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -484,11 +484,11 @@ func (f *Forwarder) IsDeficient() bool { return f.isDeficientLocked() } -func (f *Forwarder) BandwidthRequested() int64 { +func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() - return f.lastAllocation.BandwidthRequested + return getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { @@ -619,7 +619,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if alloc.TargetLayer.IsValid() { alloc.BandwidthRequested = optimalBandwidthNeeded } - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) alloc.DistanceToDesired = getDistanceToDesired( f.muted, f.pubMuted, @@ -719,22 +719,26 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b f.lock.Lock() defer f.lock.Unlock() + existingTargetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { + bandwidthRequired := int64(0) f.provisional.allocatedLayer = buffer.InvalidLayer if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute f.provisional.allocatedLayer = f.provisional.currentLayer + if f.provisional.allocatedLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] + } } return VideoTransition{ From: f.vls.GetTarget(), To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } // check if we should preserve current target - targetLayer := f.vls.GetTarget() - if targetLayer.IsValid() { + if existingTargetLayer.IsValid() { // what is the highest that is available maximalLayer := buffer.InvalidLayer maximalBandwidthRequired := int64(0) @@ -753,24 +757,24 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } if maximalLayer.IsValid() { - if !targetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] != 0 { - // currently streaming and maybe wanting an upgrade (targetLayer <= maximalLayer), + if !existingTargetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[existingTargetLayer.Spatial][existingTargetLayer.Temporal] != 0 { + // currently streaming and maybe wanting an upgrade (existingTargetLayer <= maximalLayer), // just preserve current target in the cooperative scheme of things - f.provisional.allocatedLayer = targetLayer + f.provisional.allocatedLayer = existingTargetLayer return VideoTransition{ - From: targetLayer, - To: targetLayer, + From: existingTargetLayer, + To: existingTargetLayer, BandwidthDelta: 0, } } - if targetLayer.GreaterThan(maximalLayer) { - // maximalLayer < targetLayer, make the down move + if existingTargetLayer.GreaterThan(maximalLayer) { + // maximalLayer < existingTargetLayer, make the down move f.provisional.allocatedLayer = maximalLayer return VideoTransition{ - From: targetLayer, + From: existingTargetLayer, To: maximalLayer, - BandwidthDelta: maximalBandwidthRequired - f.lastAllocation.BandwidthRequested, + BandwidthDelta: maximalBandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } } @@ -799,8 +803,9 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b return layers, bw } + targetLayer := buffer.InvalidLayer bandwidthRequired := int64(0) - if !targetLayer.IsValid() { + if !existingTargetLayer.IsValid() { // currently not streaming, find minimal // NOTE: a layer in feed could have paused and there could be other options than going back to minimal, // but the cooperative scheme knocks things back to minimal @@ -825,13 +830,17 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } else { targetLayer = f.provisional.currentLayer } + + if targetLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] + } } f.provisional.allocatedLayer = targetLayer return VideoTransition{ From: f.vls.GetTarget(), To: targetLayer, - BandwidthDelta: bandwidthRequired - f.lastAllocation.BandwidthRequested, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } @@ -856,15 +865,12 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti targetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { + // if publisher muted, give up opportunistic resume and give back the bandwidth f.provisional.allocatedLayer = buffer.InvalidLayer - if f.provisional.pubMuted { - // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayer = f.provisional.currentLayer - } return VideoTransition{ From: targetLayer, To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } @@ -893,12 +899,13 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti return VideoTransition{ From: targetLayer, To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } // starting from minimum to target, find transition which gives the best // transition taking into account bits saved vs cost of such a transition + existingBandwidthNeeded := getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested) bestLayer := buffer.InvalidLayer bestBandwidthDelta := int64(0) bestValue := float32(0) @@ -908,7 +915,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti break } - BandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.BandwidthRequested-f.provisional.Bitrates[s][t]))) + bandwidthDelta := int64(math.Max(float64(0), float64(existingBandwidthNeeded-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) // LK-TODO: SVC will need a different cost transition @@ -920,11 +927,11 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti value := float32(0) if (transitionCost + qualityCost) != 0 { - value = float32(BandwidthDelta) / float32(transitionCost+qualityCost) + value = float32(bandwidthDelta) / float32(transitionCost+qualityCost) } - if value > bestValue || (value == bestValue && BandwidthDelta > bestBandwidthDelta) { + if value > bestValue || (value == bestValue && bandwidthDelta > bestBandwidthDelta) { bestValue = value - bestBandwidthDelta = BandwidthDelta + bestBandwidthDelta = bandwidthDelta bestLayer = buffer.VideoLayer{Spatial: s, Temporal: t} } } @@ -951,7 +958,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { ) alloc := VideoAllocation{ BandwidthRequested: 0, - BandwidthDelta: -f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), Bitrates: f.provisional.Bitrates, BandwidthNeeded: optimalBandwidthNeeded, TargetLayer: f.provisional.allocatedLayer, @@ -979,7 +986,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { if f.provisional.allocatedLayer.IsValid() { // overshoot alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } else { alloc.PauseReason = VideoPauseReasonFeedDry @@ -995,7 +1002,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { if f.provisional.allocatedLayer.IsValid() { alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) if f.provisional.allocatedLayer.GreaterThan(f.provisional.maxLayer) || alloc.BandwidthRequested >= getOptimalBandwidthNeeded( @@ -1226,7 +1233,7 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alloc := VideoAllocation{ BandwidthRequested: 0, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), Bitrates: brs, BandwidthNeeded: optimalBandwidthNeeded, TargetLayer: buffer.InvalidLayer, @@ -1606,6 +1613,14 @@ func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int3 return 0 } +func getBandwidthNeeded(brs Bitrates, layer buffer.VideoLayer, fallback int64) int64 { + if layer.IsValid() && brs[layer.Spatial][layer.Temporal] > 0 { + return brs[layer.Spatial][layer.Temporal] + } + + return fallback +} + func getDistanceToDesired( muted bool, pubMuted bool, diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 86657e270..bbb079258 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -221,7 +221,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonNone, BandwidthRequested: bitrates[1][3], - BandwidthDelta: bitrates[1][3], + BandwidthDelta: bitrates[1][3] - bitrates[0][1], BandwidthNeeded: bitrates[1][3], Bitrates: bitrates, TargetLayer: buffer.DefaultMaxLayer,