diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go
index 77c6c58ab..3bb0a6715 100644
--- a/pkg/sfu/buffer/buffer.go
+++ b/pkg/sfu/buffer/buffer.go
@@ -614,7 +614,12 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow
 	ep.Temporal = 0
 	if b.ddParser != nil {
 		ddVal, videoLayer, err := b.ddParser.Parse(ep.Packet)
-		if err == nil && ddVal != nil {
+		if err != nil {
+			if err != ErrFrameEarlierThanKeyFrame {
+				b.logger.Warnw("could not parse dependency descriptor", err)
+			}
+			return nil
+		} else if ddVal != nil {
 			ep.DependencyDescriptor = ddVal
 			ep.VideoLayer = videoLayer
 			// DD-TODO : notify active decode target change if changed.
diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go
index ca69bd763..05b675ad0 100644
--- a/pkg/sfu/buffer/dependencydescriptorparser.go
+++ b/pkg/sfu/buffer/dependencydescriptorparser.go
@@ -26,6 +26,10 @@ import (
 	"github.com/livekit/protocol/logger"
 )
 
+var (
+	ErrFrameEarlierThanKeyFrame = fmt.Errorf("frame is earlier than current keyframe")
+)
+
 type DependencyDescriptorParser struct {
 	structure         *dd.FrameDependencyStructure
 	ddExtID           uint8
@@ -36,6 +40,7 @@ type DependencyDescriptorParser struct {
 	seqWrapAround             *utils.WrapAround[uint16, uint64]
 	frameWrapAround           *utils.WrapAround[uint16, uint64]
 	structureExtSeq           uint64
+	structureExtFrameNum      uint64
 	activeDecodeTargetsExtSeq uint64
 	activeDecodeTargetsMask   uint32
 	frameChecker              *FrameIntegrityChecker
@@ -88,6 +93,12 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
 	}
 
 	extFN := r.frameWrapAround.Update(ddVal.FrameNumber).ExtendedVal
+
+	if extFN < r.structureExtFrameNum {
+		r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum)
+		return nil, videoLayer, ErrFrameEarlierThanKeyFrame
+	}
+
 	r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
 
 	extDD := &ExtDependencyDescriptor{
@@ -97,11 +108,12 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
 	}
 
 	if ddVal.AttachedStructure != nil {
-		r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String()))
+		r.logger.Debugw("parsed dependency descriptor", "extSeq", extSeq, "extFN", extFN, "structureID", ddVal.AttachedStructure.StructureId, "descriptor", ddVal.String())
 		if extSeq > r.structureExtSeq {
 			r.structure = ddVal.AttachedStructure
 			r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
 			r.structureExtSeq = extSeq
+			r.structureExtFrameNum = extFN
 			extDD.StructureUpdated = true
 			extDD.ActiveDecodeTargetsUpdated = true
 			// The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present,
diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go
index 613f2d264..0777fb0fa 100644
--- a/pkg/sfu/videolayerselector/framechain.go
+++ b/pkg/sfu/videolayerselector/framechain.go
@@ -45,6 +45,11 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate
 		return false
 	}
 
+	if len(fd.ChainDiffs) <= fc.chainIdx {
+		fc.logger.Warnw("invalid frame chain diff", nil, "chanIdx", fc.chainIdx, "frame", extFrameNum, "fd", fd)
+		return fc.broken
+	}
+
 	// A decodable frame with frame_chain_fdiff equal to 0 indicates that the Chain is intact.
 	if fd.ChainDiffs[fc.chainIdx] == 0 {
 		if fc.broken {