Add option to use different pacer with send side bwe. (#3552)

Seeing a lot of queuing delay based back offs. Trying a couple of things
1. Accept a bit more queuing.
2. An option to try a different pacer. Would like to try with pass
   through. That will produce some out-of-order packets. Remains to be
   seen if it will have a negative impact.

pkg/config/config.go

@@ -31,6 +31,7 @@ import (
 	"github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe"
 	"github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe"
 	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/sfu/pacer"
 	"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
 	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
 	"github.com/livekit/protocol/livekit"
@@ -140,8 +141,9 @@ type CongestionControlConfig struct {
 
 	UseSendSideBWEInterceptor bool `yaml:"use_send_side_bwe_interceptor,omitempty"`
 
-	UseSendSideBWE bool                          `yaml:"use_send_side_bwe,omitempty"`
-	SendSideBWE    sendsidebwe.SendSideBWEConfig `yaml:"send_side_bwe,omitempty"`
+	UseSendSideBWE   bool                          `yaml:"use_send_side_bwe,omitempty"`
+	SendSideBWEPacer string                        `yaml:"send_side_bwe_pacer,omitempty"`
+	SendSideBWE      sendsidebwe.SendSideBWEConfig `yaml:"send_side_bwe,omitempty"`
 }
 
 type PlayoutDelayConfig struct {
@@ -319,6 +321,7 @@ var DefaultConfig = Config{
 			RemoteBWE:                 remotebwe.DefaultRemoteBWEConfig,
 			UseSendSideBWEInterceptor: false,
 			UseSendSideBWE:            false,
+			SendSideBWEPacer:          string(pacer.PacerBehaviorNoQueue),
 			SendSideBWE:               sendsidebwe.DefaultSendSideBWEConfig,
 		},
 	},

pkg/rtc/transport.go

@@ -491,7 +491,14 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) {
 				Config: params.CongestionControlConfig.SendSideBWE,
 				Logger: params.Logger,
 			})
-			t.pacer = pacer.NewNoQueue(params.Logger, t.bwe)
+			switch pacer.PacerBehavior(params.CongestionControlConfig.SendSideBWEPacer) {
+			case pacer.PacerBehaviorPassThrough:
+				t.pacer = pacer.NewPassThrough(params.Logger, t.bwe)
+			case pacer.PacerBehaviorNoQueue:
+				t.pacer = pacer.NewNoQueue(params.Logger, t.bwe)
+			default:
+				t.pacer = pacer.NewNoQueue(params.Logger, t.bwe)
+			}
 		} else {
 			t.bwe = remotebwe.NewRemoteBWE(remotebwe.RemoteBWEParams{
 				Config: params.CongestionControlConfig.RemoteBWE,

pkg/sfu/bwe/sendsidebwe/congestion_detector.go

@@ -121,8 +121,8 @@ var (
 		MinBytesRatio:    0.5,
 		MinDurationRatio: 0.5,
 
-		JQRMinDelay: 40 * time.Millisecond,
-		DQRMaxDelay: 15 * time.Millisecond,
+		JQRMinDelay: 50 * time.Millisecond,
+		DQRMaxDelay: 20 * time.Millisecond,
 
 		WeightedLoss:       defaultWeightedLossConfig,
 		JQRMinWeightedLoss: 0.25,
@@ -452,8 +452,8 @@ var (
 		ProbeRegulator:   ccutils.DefaultProbeRegulatorConfig,
 		ProbeSignal:      defaultProbeSignalConfig,
 
-		JQRMinDelay: 40 * time.Millisecond,
-		DQRMaxDelay: 15 * time.Millisecond,
+		JQRMinDelay: 50 * time.Millisecond,
+		DQRMaxDelay: 20 * time.Millisecond,
 
 		WeightedLoss:       defaultWeightedLossConfig,
 		JQRMinWeightedLoss: 0.25,

pkg/sfu/pacer/pacer.go

@@ -23,6 +23,14 @@ import (
 	"github.com/pion/webrtc/v4"
 )
 
+type PacerBehavior string
+
+const (
+	PacerBehaviorPassThrough PacerBehavior = "pass-through"
+	PacerBehaviorNoQueue     PacerBehavior = "no-queue"
+	PacerBehaviorLeakybucket PacerBehavior = "leaky-bucket"
+)
+
 type Packet struct {
 	Header             *rtp.Header
 	HeaderSize         int