diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go
index c6cd98dd2..b1c4c5df9 100644
--- a/pkg/sfu/buffer/rtpstats_sender.go
+++ b/pkg/sfu/buffer/rtpstats_sender.go
@@ -327,17 +327,6 @@ func (r *RTPStatsSender) Update(
 			r.extStartSN = extSequenceNumber
 		}
 
-		if extTimestamp < r.extStartTS {
-			r.logger.Infow(
-				"adjusting start timestamp",
-				"snBefore", r.extStartSN,
-				"snAfter", extSequenceNumber,
-				"tsBefore", r.extStartTS,
-				"tsAfter", extTimestamp,
-			)
-			r.extStartTS = extTimestamp
-		}
-
 		if gapSN != 0 {
 			r.packetsOutOfOrder++
 		}
@@ -377,23 +366,27 @@ func (r *RTPStatsSender) Update(
 
 		r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, false)
 
-		if extTimestamp != r.extHighestTS {
-			// update only on first packet as same timestamp could be in multiple packets.
-			// NOTE: this may not be the first packet with this time stamp if there is packet loss.
-			if payloadSize == 0 {
-				r.logger.Infow(
-					"updating highest time on padding packet",
-					"extSequenceNumber", extSequenceNumber,
-					"extHighestSN", r.extHighestSN,
-					"extTimestamp", extTimestamp,
-					"extHighestTS", r.extHighestTS,
-					"highestTime", r.highestTime.String(),
-					"packetTime", packetTime.String(),
-				)
-			}
+		r.extHighestSN = extSequenceNumber
+	}
+
+	if extTimestamp < r.extStartTS {
+		r.logger.Infow(
+			"adjusting start timestamp",
+			"snBefore", r.extStartSN,
+			"snAfter", extSequenceNumber,
+			"tsBefore", r.extStartTS,
+			"tsAfter", extTimestamp,
+		)
+		r.extStartTS = extTimestamp
+	}
+
+	if extTimestamp > r.extHighestTS {
+		// update only on first packet as same timestamp could be in multiple packets.
+		// NOTE: this may not be the first packet with this time stamp if there is packet loss.
+		if payloadSize > 0 {
+			// skip updating on padding only packets as they could re-use an old timestamp
 			r.highestTime = packetTime
 		}
-		r.extHighestSN = extSequenceNumber
 		r.extHighestTS = extTimestamp
 	}
 
diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go
index 8cfb4be27..31c415f12 100644
--- a/pkg/sfu/rtpmunger.go
+++ b/pkg/sfu/rtpmunger.go
@@ -54,10 +54,11 @@ type RTPMungerState struct {
 	ExtLastSN       uint64
 	ExtSecondLastSN uint64
 	ExtLastTS       uint64
+	ExtSecondLastTS uint64
 }
 
 func (r RTPMungerState) String() string {
-	return fmt.Sprintf("RTPMungerState{extLastSN: %d, extSecondLastSN: %d, extLastTS: %d)", r.ExtLastSN, r.ExtSecondLastSN, r.ExtLastTS)
+	return fmt.Sprintf("RTPMungerState{extLastSN: %d, extSecondLastSN: %d, extLastTS: %d, extSecondLastTS: %d)", r.ExtLastSN, r.ExtSecondLastSN, r.ExtLastTS, r.ExtSecondLastTS)
 }
 
 // ----------------------------------------------------------------------
@@ -72,8 +73,9 @@ type RTPMunger struct {
 	extSecondLastSN uint64
 	snOffset        uint64
 
-	extLastTS uint64
-	tsOffset  uint64
+	extLastTS       uint64
+	extSecondLastTS uint64
+	tsOffset        uint64
 
 	lastMarker bool
 
@@ -95,6 +97,7 @@ func (r *RTPMunger) DebugInfo() map[string]interface{} {
 		"ExtSecondLastSN":      r.extSecondLastSN,
 		"SNOffset":             r.snOffset,
 		"ExtLastTS":            r.extLastTS,
+		"ExtSecondLastTS":      r.extSecondLastTS,
 		"TSOffset":             r.tsOffset,
 		"LastMarker":           r.lastMarker,
 	}
@@ -105,6 +108,7 @@ func (r *RTPMunger) GetLast() RTPMungerState {
 		ExtLastSN:       r.extLastSN,
 		ExtSecondLastSN: r.extSecondLastSN,
 		ExtLastTS:       r.extLastTS,
+		ExtSecondLastTS: r.extSecondLastTS,
 	}
 }
 
@@ -112,6 +116,7 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) {
 	r.extLastSN = state.ExtLastSN
 	r.extSecondLastSN = state.ExtSecondLastSN
 	r.extLastTS = state.ExtLastTS
+	r.extSecondLastTS = state.ExtSecondLastTS
 }
 
 func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
@@ -123,6 +128,7 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
 	r.updateSnOffset()
 
 	r.extLastTS = extPkt.ExtTimestamp
+	r.extSecondLastTS = extPkt.ExtTimestamp
 }
 
 func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) {
@@ -156,6 +162,8 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) {
 
 	r.extLastSN = r.extSecondLastSN
 	r.updateSnOffset()
+
+	r.extLastTS = r.extSecondLastTS
 }
 
 func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) {
@@ -174,6 +182,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara
 
 		r.extSecondLastSN = r.extLastSN
 		r.extLastSN = extMungedSN
+		r.extSecondLastTS = r.extLastTS
 		r.extLastTS = extMungedTS
 		r.lastMarker = extPkt.Packet.Marker
 
@@ -306,6 +315,11 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
 	r.snRangeMap.DecValue(r.extHighestIncomingSN, uint64(num))
 	r.updateSnOffset()
 
+	if len(vals) == 1 {
+		r.extSecondLastTS = r.extLastTS
+	} else {
+		r.extSecondLastTS = vals[len(vals)-2].extTimestamp
+	}
 	r.tsOffset -= extLastTS - r.extLastTS
 	r.extLastTS = extLastTS