diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go
index 505ddd694..ccaf9cd4c 100644
--- a/pkg/sfu/buffer/buffer.go
+++ b/pkg/sfu/buffer/buffer.go
@@ -66,6 +66,7 @@ type ExtPacket struct {
 	RawPacket            []byte
 	DependencyDescriptor *ExtDependencyDescriptor
 	AbsCaptureTimeExt    *act.AbsCaptureTime
+	IsOutOfOrder         bool
 }
 
 // Buffer contains all packets
@@ -763,6 +764,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
 			Spatial:  InvalidLayerSpatial,
 			Temporal: InvalidLayerTemporal,
 		},
+		IsOutOfOrder: flowState.IsOutOfOrder,
 	}
 
 	if len(rtpPacket.Payload) == 0 {
diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go
index 54c56714c..b94cd308d 100644
--- a/pkg/sfu/downtrack.go
+++ b/pkg/sfu/downtrack.go
@@ -745,7 +745,8 @@ func (d *DownTrack) maxLayerNotifierWorker() {
 
 // WriteRTP writes an RTP Packet to the DownTrack
 func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error {
-	if !d.writable.Load() {
+	if !d.writable.Load() || (extPkt.IsOutOfOrder && !d.rtpStats.IsActive()) {
+		// do not start on an out-of-order packet
 		return nil
 	}
 
@@ -801,7 +802,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error {
 			// the corresponding sequence number is received.
 			// The extreme case is all packets containing the play out delay are lost and
 			// all of them retransmitted and an RTCP Receiver Report received for those
-			// retransmited sequence numbers. But, that is highly improbable, if not impossible.
+			// retransmitted sequence numbers. But, that is highly improbable, if not impossible.
 		}
 	}
 	var actBytes []byte