diff --git a/pkg/sfu/buffer/buffer_base.go b/pkg/sfu/buffer/buffer_base.go index a18847787..ec7ff6ac6 100644 --- a/pkg/sfu/buffer/buffer_base.go +++ b/pkg/sfu/buffer/buffer_base.go @@ -79,7 +79,8 @@ type ExtPacket struct { ExtSequenceNumber uint64 ExtTimestamp uint64 Packet *rtp.Packet - Payload any + VP8 *codec.VP8 // nil when not parsed + VP9 *codecs.VP9Packet // nil when not parsed IsKeyFrame bool RawPacket []byte DependencyDescriptor *ExtDependencyDescriptor @@ -91,6 +92,10 @@ type ExtPacket struct { // extension and CSRC slices point into the write input until ReadExtended re-points them into RawPacket pkt rtp.Packet extensions [8]rtp.Extension + + // backing storage for VP8 and VP9, so parsing does not allocate + vp8 codec.VP8 + vp9 codecs.VP9Packet } type BufferProvider interface { @@ -1106,7 +1111,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { switch b.mime { case mime.MimeTypeVP8: - vp8Packet := codec.VP8{} + vp8Packet := &ep.vp8 if err := vp8Packet.Unmarshal(ep.Packet.Payload); err != nil { b.logger.Warnw("could not unmarshal VP8 packet", err) return err @@ -1116,7 +1121,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { ep.Temporal = int32(vp8Packet.TID) if ep.IsKeyFrame { - if sz := codec.ExtractVP8VideoSize(&vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { + if sz := codec.ExtractVP8VideoSize(vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { videoSize = append(videoSize, sz) } } @@ -1124,12 +1129,12 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { // vp8 with DependencyDescriptor enabled, use the TID from the descriptor vp8Packet.TID = uint8(ep.Temporal) } - ep.Payload = vp8Packet + ep.VP8 = vp8Packet ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid case mime.MimeTypeVP9: if ep.DependencyDescriptor == nil { - var vp9Packet codecs.VP9Packet + vp9Packet := &ep.vp9 _, err := vp9Packet.Unmarshal(ep.Packet.Payload) if err != nil { b.logger.Warnw("could not unmarshal VP9 packet", err) @@ -1139,8 +1144,8 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { Spatial: int32(vp9Packet.SID), Temporal: int32(vp9Packet.TID), } - ep.Payload = vp9Packet - ep.IsKeyFrame = codec.IsVP9KeyFrame(&vp9Packet, ep.Packet.Payload) + ep.VP9 = vp9Packet + ep.IsKeyFrame = codec.IsVP9KeyFrame(vp9Packet, ep.Packet.Payload) if ep.IsKeyFrame { for i := 0; i < len(vp9Packet.Width); i++ { diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index d41dfe375..fa25dc584 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -17,9 +17,6 @@ package buffer import ( "container/list" - "github.com/pion/rtp/codecs" - - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -209,8 +206,8 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { return true } - vp8, ok := ep.Payload.(codec.VP8) - if !ok { + vp8 := ep.VP8 + if vp8 == nil { f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) return false } @@ -260,8 +257,8 @@ func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool { return true } - vp9, ok := ep.Payload.(codecs.VP9Packet) - if !ok { + vp9 := ep.VP9 + if vp9 == nil { f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber) return false } diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 8bdacb13f..6e7979abe 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -36,7 +36,7 @@ type testFrameInfo struct { func (f *testFrameInfo) toVP8() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - Payload: codec.VP8{ + VP8: &codec.VP8{ I: true, PictureID: f.framenumber, }, diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index 4ffc7e65a..748acfcef 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -104,8 +104,8 @@ func (v *VP8) SeedState(seed any) { } func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } @@ -129,8 +129,8 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { } func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } @@ -155,8 +155,8 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, [MaxHeaderSize]byte, int, error) { var hdr [MaxHeaderSize]byte - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return 0, hdr, 0, ErrNotVP8 } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index f69e6fec5..fea4a1b91 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -23,13 +23,11 @@ import ( "time" "github.com/pion/rtp" - "github.com/pion/rtp/codecs" "github.com/pion/webrtc/v4" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil" - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -2180,8 +2178,7 @@ func (f *Forwarder) isEndOfLayerFrame(extPkt *buffer.ExtPacket) bool { } if f.mime == mime.MimeTypeVP9 { - vp9, ok := extPkt.Payload.(codecs.VP9Packet) - return ok && vp9.E + return extPkt.VP9 != nil && extPkt.VP9.E } return false @@ -2204,7 +2201,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) } if _, ok := f.codecMunger.(*codecmunger.VP8DD); ok && extPkt.DependencyDescriptor != nil { - if vp8, ok := extPkt.Payload.(codec.VP8); ok && vp8.I { + if extPkt.VP8 != nil && extPkt.VP8.I { // The publisher sends picture IDs with the dependency descriptor, so they need // rewriting. The VP8 munger does that, the temporal layers still switch with the // dependency descriptor. diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 0ee950ae6..83682329a 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -2106,7 +2106,7 @@ func TestForwarderVP8DDWithPictureID(t *testing.T) { &testutils.TestDDParams{FirstPacketInFrame: true, ExtFrameNum: 2, TemporalID: 1, DTIs: "-SS", FrameDiffs: []int{1}, DecodeTargets: testutils.TestVP8DecodeTargets}, ) require.NoError(t, err) - extPkt.Payload = codec.VP8{S: true, I: true, PictureID: 2} + extPkt.VP8 = &codec.VP8{S: true, I: true, PictureID: 2} _, _ = f.GetTranslationParams(extPkt, 0) require.IsType(t, &codecmunger.VP8{}, f.codecMunger) require.True(t, f.codecMunger.GetState().(*livekit.VP8MungerState).PictureIdUsed, "seeded from the packet") @@ -2463,24 +2463,24 @@ func TestForwarderIsEndOfLayerFrame(t *testing.T) { vp9Codec := webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP9.String(), ClockRate: 90000} f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}})) f = newForwarder(vp9Codec, webrtc.RTPCodecTypeVideo) - require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) - require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: false}})) + require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: false}})) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{})) require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: true}, }, - Payload: codecs.VP9Packet{E: false}, + VP9: &codecs.VP9Packet{E: false}, })) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: false}, }, - Payload: codecs.VP9Packet{E: true}, + VP9: &codecs.VP9Packet{E: true}, })) require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ DependencyDescriptor: &buffer.ExtDependencyDescriptor{}, diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index f4a89e49d..1056e4119 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -88,7 +88,8 @@ func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.E } ep.IsKeyFrame = vp8.IsKeyFrame - ep.Payload = *vp8 + vp8Copy := *vp8 // keep the packet independent of later changes to vp8 + ep.VP8 = &vp8Copy if ep.DependencyDescriptor == nil { ep.Temporal = int32(vp8.TID) } diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go index bc5679c58..57162b2f0 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -16,7 +16,6 @@ package temporallayerselector import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -37,8 +36,8 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi return } - vp8, ok := extPkt.Payload.(codec.VP8) - if !ok { + vp8 := extPkt.VP8 + if vp8 == nil { return } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 4b11d37ab..7cb5de7cc 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -15,8 +15,6 @@ package videolayerselector import ( - "github.com/pion/rtp/codecs" - "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/logger" ) @@ -40,8 +38,8 @@ func (v *VP9) IsOvershootOkay() bool { } func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { - vp9, ok := extPkt.Payload.(codecs.VP9Packet) - if !ok { + vp9 := extPkt.VP9 + if vp9 == nil { return }