Create channel observer on probe reset. (#1783)

On a state change, it was possible an aborted probe was pending
finalize. When probe controller is reset, the probe channel
observer was not reset. Create a new non-probe channel observer
on state change to get a fresh start.

Also limit probe finalize wait to 10 seconds max. It is possible
that the estimate is very low and we have sent a bunch of probes.
Calculating wait based on that could lead to finalize waiting for
a long time (could be minutes).

pkg/sfu/streamallocator/probe_controller.go

@@ -12,6 +12,7 @@ const (
 	ProbeBackoffFactor = 1.5
 	ProbeWaitMax       = 30 * time.Second
 	ProbeSettleWait    = 250
+	ProbeSettleWaitMax = 10 * time.Second
 	ProbeTrendWait     = 2 * time.Second
 
 	ProbePct         = 120
@@ -82,6 +83,9 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate
 		queueTime = 0.0
 	}
 	queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond
+	if queueWait > ProbeSettleWaitMax {
+		queueWait = ProbeSettleWaitMax
+	}
 	p.probeEndTime = p.lastProbeStartTime.Add(queueWait)
 	p.params.Logger.Infow(
 		"setting probe end time",

pkg/sfu/streamallocator/streamallocator.go

@@ -743,6 +743,8 @@ func (s *StreamAllocator) setState(state streamAllocatorState) {
 
 	// reset probe to enforce a delay after state change before probing
 	s.probeController.Reset()
+	// a fresh channel observer after state transition to get clean data
+	s.channelObserver = s.newChannelObserverNonProbe()
 }
 
 func (s *StreamAllocator) adjustState() {