diff --git a/pkg/sfu/buffer/buffer_base.go b/pkg/sfu/buffer/buffer_base.go
index 4f4c49577..95b855264 100644
--- a/pkg/sfu/buffer/buffer_base.go
+++ b/pkg/sfu/buffer/buffer_base.go
@@ -769,6 +769,12 @@ func (b *BufferBase) HandleIncomingPacketLocked(
 		return 0, fmt.Errorf("unhandled reason: %s", flowState.UnhandledReason.String())
 	}
 
+	if b.params.IsOOBSequenceNumber {
+		b.updateOOBNACKState(oobSequenceNumber, arrivalTime, len(rawPkt))
+	} else {
+		b.updateNACKState(rtpPacket.SequenceNumber, flowState)
+	}
+
 	if len(rtpPacket.Payload) == 0 && (!flowState.IsOutOfOrder || flowState.IsDuplicate) {
 		// drop padding only in-order or duplicate packet
 		if !flowState.IsOutOfOrder {
@@ -873,12 +879,6 @@ func (b *BufferBase) HandleIncomingPacketLocked(
 
 	b.maybeGrowBucket(arrivalTime)
 
-	if b.params.IsOOBSequenceNumber {
-		b.updateOOBNACKState(oobSequenceNumber, arrivalTime, len(rawPkt))
-	} else {
-		b.updateNACKState(rtpPacket.SequenceNumber, flowState)
-	}
-
 	return ep.ExtSequenceNumber, nil
 }