From 9ddcc50767c6a460e2cb8794d9e5d4fb8616b406 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 11 Oct 2026 20:43:17 +0530 Subject: [PATCH] sfu: parse VP8 and VP9 payload descriptors without an allocation (#4981) * sfu: parse VP8 and VP9 payload descriptors without an allocation Replace ExtPacket.Payload with typed VP8 and VP9 pointer fields that point at backing structs inside the ExtPacket, so parsing does not allocate and a wrong codec type is a compile error. Co-Authored-By: Claude Opus 5.5 * sfu: copy the VP8 descriptor in the test packet helper Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- pkg/sfu/buffer/buffer_base.go | 19 ++++++++++++------- pkg/sfu/buffer/fps.go | 11 ++++------- pkg/sfu/buffer/fps_test.go | 2 +- pkg/sfu/codecmunger/vp8.go | 12 ++++++------ pkg/sfu/forwarder.go | 7 ++----- pkg/sfu/forwarder_test.go | 12 ++++++------ pkg/sfu/testutils/data.go | 3 ++- .../temporallayerselector/vp8.go | 5 ++--- pkg/sfu/videolayerselector/vp9.go | 6 ++---- 9 files changed, 37 insertions(+), 40 deletions(-) diff --git a/pkg/sfu/buffer/buffer_base.go b/pkg/sfu/buffer/buffer_base.go index a18847787..ec7ff6ac6 100644 --- a/pkg/sfu/buffer/buffer_base.go +++ b/pkg/sfu/buffer/buffer_base.go @@ -79,7 +79,8 @@ type ExtPacket struct { ExtSequenceNumber uint64 ExtTimestamp uint64 Packet *rtp.Packet - Payload any + VP8 *codec.VP8 // nil when not parsed + VP9 *codecs.VP9Packet // nil when not parsed IsKeyFrame bool RawPacket []byte DependencyDescriptor *ExtDependencyDescriptor @@ -91,6 +92,10 @@ type ExtPacket struct { // extension and CSRC slices point into the write input until ReadExtended re-points them into RawPacket pkt rtp.Packet extensions [8]rtp.Extension + + // backing storage for VP8 and VP9, so parsing does not allocate + vp8 codec.VP8 + vp9 codecs.VP9Packet } type BufferProvider interface { @@ -1106,7 +1111,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { switch b.mime { case mime.MimeTypeVP8: - vp8Packet := codec.VP8{} + vp8Packet := &ep.vp8 if err := vp8Packet.Unmarshal(ep.Packet.Payload); err != nil { b.logger.Warnw("could not unmarshal VP8 packet", err) return err @@ -1116,7 +1121,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { ep.Temporal = int32(vp8Packet.TID) if ep.IsKeyFrame { - if sz := codec.ExtractVP8VideoSize(&vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { + if sz := codec.ExtractVP8VideoSize(vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { videoSize = append(videoSize, sz) } } @@ -1124,12 +1129,12 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { // vp8 with DependencyDescriptor enabled, use the TID from the descriptor vp8Packet.TID = uint8(ep.Temporal) } - ep.Payload = vp8Packet + ep.VP8 = vp8Packet ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid case mime.MimeTypeVP9: if ep.DependencyDescriptor == nil { - var vp9Packet codecs.VP9Packet + vp9Packet := &ep.vp9 _, err := vp9Packet.Unmarshal(ep.Packet.Payload) if err != nil { b.logger.Warnw("could not unmarshal VP9 packet", err) @@ -1139,8 +1144,8 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { Spatial: int32(vp9Packet.SID), Temporal: int32(vp9Packet.TID), } - ep.Payload = vp9Packet - ep.IsKeyFrame = codec.IsVP9KeyFrame(&vp9Packet, ep.Packet.Payload) + ep.VP9 = vp9Packet + ep.IsKeyFrame = codec.IsVP9KeyFrame(vp9Packet, ep.Packet.Payload) if ep.IsKeyFrame { for i := 0; i < len(vp9Packet.Width); i++ { diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index d41dfe375..fa25dc584 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -17,9 +17,6 @@ package buffer import ( "container/list" - "github.com/pion/rtp/codecs" - - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -209,8 +206,8 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { return true } - vp8, ok := ep.Payload.(codec.VP8) - if !ok { + vp8 := ep.VP8 + if vp8 == nil { f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) return false } @@ -260,8 +257,8 @@ func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool { return true } - vp9, ok := ep.Payload.(codecs.VP9Packet) - if !ok { + vp9 := ep.VP9 + if vp9 == nil { f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber) return false } diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 8bdacb13f..6e7979abe 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -36,7 +36,7 @@ type testFrameInfo struct { func (f *testFrameInfo) toVP8() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - Payload: codec.VP8{ + VP8: &codec.VP8{ I: true, PictureID: f.framenumber, }, diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index 4ffc7e65a..748acfcef 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -104,8 +104,8 @@ func (v *VP8) SeedState(seed any) { } func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } @@ -129,8 +129,8 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { } func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } @@ -155,8 +155,8 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, [MaxHeaderSize]byte, int, error) { var hdr [MaxHeaderSize]byte - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return 0, hdr, 0, ErrNotVP8 } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index f69e6fec5..fea4a1b91 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -23,13 +23,11 @@ import ( "time" "github.com/pion/rtp" - "github.com/pion/rtp/codecs" "github.com/pion/webrtc/v4" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil" - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -2180,8 +2178,7 @@ func (f *Forwarder) isEndOfLayerFrame(extPkt *buffer.ExtPacket) bool { } if f.mime == mime.MimeTypeVP9 { - vp9, ok := extPkt.Payload.(codecs.VP9Packet) - return ok && vp9.E + return extPkt.VP9 != nil && extPkt.VP9.E } return false @@ -2204,7 +2201,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) } if _, ok := f.codecMunger.(*codecmunger.VP8DD); ok && extPkt.DependencyDescriptor != nil { - if vp8, ok := extPkt.Payload.(codec.VP8); ok && vp8.I { + if extPkt.VP8 != nil && extPkt.VP8.I { // The publisher sends picture IDs with the dependency descriptor, so they need // rewriting. The VP8 munger does that, the temporal layers still switch with the // dependency descriptor. diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 0ee950ae6..83682329a 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -2106,7 +2106,7 @@ func TestForwarderVP8DDWithPictureID(t *testing.T) { &testutils.TestDDParams{FirstPacketInFrame: true, ExtFrameNum: 2, TemporalID: 1, DTIs: "-SS", FrameDiffs: []int{1}, DecodeTargets: testutils.TestVP8DecodeTargets}, ) require.NoError(t, err) - extPkt.Payload = codec.VP8{S: true, I: true, PictureID: 2} + extPkt.VP8 = &codec.VP8{S: true, I: true, PictureID: 2} _, _ = f.GetTranslationParams(extPkt, 0) require.IsType(t, &codecmunger.VP8{}, f.codecMunger) require.True(t, f.codecMunger.GetState().(*livekit.VP8MungerState).PictureIdUsed, "seeded from the packet") @@ -2463,24 +2463,24 @@ func TestForwarderIsEndOfLayerFrame(t *testing.T) { vp9Codec := webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP9.String(), ClockRate: 90000} f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}})) f = newForwarder(vp9Codec, webrtc.RTPCodecTypeVideo) - require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) - require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: false}})) + require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: false}})) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{})) require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: true}, }, - Payload: codecs.VP9Packet{E: false}, + VP9: &codecs.VP9Packet{E: false}, })) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: false}, }, - Payload: codecs.VP9Packet{E: true}, + VP9: &codecs.VP9Packet{E: true}, })) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{}, diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index f4a89e49d..1056e4119 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -88,7 +88,8 @@ func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.E } ep.IsKeyFrame = vp8.IsKeyFrame - ep.Payload = *vp8 + vp8Copy := *vp8 // keep the packet independent of later changes to vp8 + ep.VP8 = &vp8Copy if ep.DependencyDescriptor == nil { ep.Temporal = int32(vp8.TID) } diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go index bc5679c58..57162b2f0 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -16,7 +16,6 @@ package temporallayerselector import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -37,8 +36,8 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi return } - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 4b11d37ab..7cb5de7cc 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -15,8 +15,6 @@ package videolayerselector import ( - "github.com/pion/rtp/codecs" - "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/logger" ) @@ -40,8 +38,8 @@ func (v *VP9) IsOvershootOkay() bool { } func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { - vp9, ok := extPkt.Payload.(codecs.VP9Packet) - if !ok { + vp9 := extPkt.VP9 + if vp9 == nil { return }