Encoding primary packet only if red encoding don't have enough space (#1196)

* Encoding primary packet only if red encoding don't have enough space

* clean code

pkg/sfu/redreceiver.go

@@ -131,6 +131,16 @@ func (r *RedReceiver) encodeRedForPrimary(pkt *rtp.Packet, redPayload []byte) (i
 }
 
 func encodeRedForPrimary(redPkts []*rtp.Packet, primary *rtp.Packet, redPayload []byte) (int, error) {
+	payloadSize := len(primary.Payload) + 1
+	for _, p := range redPkts {
+		payloadSize += len(p.Payload) + 4
+	}
+
+	// if required payload size is larger than the redPayload buffer, encode the primary packet only
+	if payloadSize > len(redPayload) {
+		redPkts = redPkts[:0]
+	}
+
 	var index int
 	for _, p := range redPkts {
 		/* RED payload https://datatracker.ietf.org/doc/html/rfc2198#section-3

pkg/sfu/redreceiver_test.go

@@ -125,6 +125,27 @@ func TestRedReceiver(t *testing.T) {
 		}, 0)
 		verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
 	})
+
+	t.Run("encoding excceed space", func(t *testing.T) {
+		w := &WebRTCReceiver{isRED: true, kind: webrtc.RTPCodecTypeAudio}
+		require.Equal(t, w.GetRedReceiver(), w)
+		w.isRED = false
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NotNil(t, red)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
+		for _, pkt := range generatePkts(header, 10, tsStep) {
+			// make sure red encodings don't have enough space to encoding redundant packet
+			pkt.Payload = make([]byte, 1000)
+			expectPkt = append(expectPkt[:0], pkt)
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+		}
+	})
 }
 
 func verifyRedEncodings(t *testing.T, red *rtp.Packet, redPkts []*rtp.Packet) {