dandri/livekit
1
0
Fork 0
mirror of https://github.com/livekit/livekit.git synced 2026-10-15 13:20:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

sfu: parse VP8 and VP9 payload descriptors without an allocation (#4981)

Browse Source 
* sfu: parse VP8 and VP9 payload descriptors without an allocation

Replace ExtPacket.Payload with typed VP8 and VP9 pointer fields that point
at backing structs inside the ExtPacket, so parsing does not allocate and a
wrong codec type is a compile error.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* sfu: copy the VP8 descriptor in the test packet helper

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Raja Subramanian
2026-10-11 20:43:17 +05:30
committed by GitHub
co-authored by Claude Opus 5.5
parent c2de273198
commit 9ddcc50767
9 changed files with 37 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sfu/buffer/buffer_base.go
+12 -7
View File
@@ -79,7 +79,8 @@ type ExtPacket struct {
ExtSequenceNumber uint64
ExtTimestamp uint64
Packet *rtp.Packet
Payload any
VP8 *codec.VP8 // nil when not parsed
VP9 *codecs.VP9Packet // nil when not parsed
IsKeyFrame bool
RawPacket []byte
DependencyDescriptor *ExtDependencyDescriptor
@@ -91,6 +92,10 @@ type ExtPacket struct {
// extension and CSRC slices point into the write input until ReadExtended re-points them into RawPacket
pkt rtp.Packet
extensions [8]rtp.Extension
// backing storage for VP8 and VP9, so parsing does not allocate
vp8 codec.VP8
vp9 codecs.VP9Packet
}
type BufferProvider interface {
@@ -1106,7 +1111,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error {
switch b.mime {
case mime.MimeTypeVP8:
vp8Packet := codec.VP8{}
vp8Packet := &ep.vp8
if err := vp8Packet.Unmarshal(ep.Packet.Payload); err != nil {
b.logger.Warnw("could not unmarshal VP8 packet", err)
return err
@@ -1116,7 +1121,7 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error {
ep.Temporal = int32(vp8Packet.TID)
if ep.IsKeyFrame {
if sz := codec.ExtractVP8VideoSize(&vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 {
if sz := codec.ExtractVP8VideoSize(vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 {
videoSize = append(videoSize, sz)
}
}
@@ -1124,12 +1129,12 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error {
// vp8 with DependencyDescriptor enabled, use the TID from the descriptor
vp8Packet.TID = uint8(ep.Temporal)
}
ep.Payload = vp8Packet
ep.VP8 = vp8Packet
ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid
case mime.MimeTypeVP9:
if ep.DependencyDescriptor == nil {
var vp9Packet codecs.VP9Packet
vp9Packet := &ep.vp9
_, err := vp9Packet.Unmarshal(ep.Packet.Payload)
if err != nil {
b.logger.Warnw("could not unmarshal VP9 packet", err)
@@ -1139,8 +1144,8 @@ func (b *BufferBase) processVideoPacket(ep *ExtPacket) error {
Spatial: int32(vp9Packet.SID),
Temporal: int32(vp9Packet.TID),
}
ep.Payload = vp9Packet
ep.IsKeyFrame = codec.IsVP9KeyFrame(&vp9Packet, ep.Packet.Payload)
ep.VP9 = vp9Packet
ep.IsKeyFrame = codec.IsVP9KeyFrame(vp9Packet, ep.Packet.Payload)
if ep.IsKeyFrame {
for i := 0; i < len(vp9Packet.Width); i++ {
pkg/sfu/buffer/fps.go
+4 -7
View File
@@ -17,9 +17,6 @@ package buffer
import (
"container/list"
"github.com/pion/rtp/codecs"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
)
@@ -209,8 +206,8 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
return true
}
vp8, ok := ep.Payload.(codec.VP8)
if !ok {
vp8 := ep.VP8
if vp8 == nil {
f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
return false
}
@@ -260,8 +257,8 @@ func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool {
return true
}
vp9, ok := ep.Payload.(codecs.VP9Packet)
if !ok {
vp9 := ep.VP9
if vp9 == nil {
f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber)
return false
}
pkg/sfu/buffer/fps_test.go
+1 -1
View File
@@ -36,7 +36,7 @@ type testFrameInfo struct {
func (f *testFrameInfo) toVP8() *ExtPacket {
return &ExtPacket{
Packet: &rtp.Packet{Header: f.header},
Payload: codec.VP8{
VP8: &codec.VP8{
I: true,
PictureID: f.framenumber,
},
pkg/sfu/codecmunger/vp8.go
+6 -6
View File
@@ -104,8 +104,8 @@ func (v *VP8) SeedState(seed any) {
}
func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
vp8 := extPkt.VP8
if vp8 == nil {
return
}
@@ -129,8 +129,8 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
}
func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
vp8 := extPkt.VP8
if vp8 == nil {
return
}
@@ -155,8 +155,8 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, [MaxHeaderSize]byte, int, error) {
var hdr [MaxHeaderSize]byte
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
vp8 := extPkt.VP8
if vp8 == nil {
return 0, hdr, 0, ErrNotVP8
}
pkg/sfu/forwarder.go
+2 -5
View File
@@ -23,13 +23,11 @@ import (
"time"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v4"
"go.uber.org/zap/zapcore"
"google.golang.org/protobuf/proto"
"github.com/livekit/mediatransportutil"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
@@ -2180,8 +2178,7 @@ func (f *Forwarder) isEndOfLayerFrame(extPkt *buffer.ExtPacket) bool {
}
if f.mime == mime.MimeTypeVP9 {
vp9, ok := extPkt.Payload.(codecs.VP9Packet)
return ok && vp9.E
return extPkt.VP9 != nil && extPkt.VP9.E
}
return false
@@ -2204,7 +2201,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger))
}
if _, ok := f.codecMunger.(*codecmunger.VP8DD); ok && extPkt.DependencyDescriptor != nil {
if vp8, ok := extPkt.Payload.(codec.VP8); ok && vp8.I {
if extPkt.VP8 != nil && extPkt.VP8.I {
// The publisher sends picture IDs with the dependency descriptor, so they need
// rewriting. The VP8 munger does that, the temporal layers still switch with the
// dependency descriptor.
pkg/sfu/forwarder_test.go
+6 -6
View File
@@ -2106,7 +2106,7 @@ func TestForwarderVP8DDWithPictureID(t *testing.T) {
&testutils.TestDDParams{FirstPacketInFrame: true, ExtFrameNum: 2, TemporalID: 1, DTIs: "-SS", FrameDiffs: []int{1}, DecodeTargets: testutils.TestVP8DecodeTargets},
)
require.NoError(t, err)
extPkt.Payload = codec.VP8{S: true, I: true, PictureID: 2}
extPkt.VP8 = &codec.VP8{S: true, I: true, PictureID: 2}
_, _ = f.GetTranslationParams(extPkt, 0)
require.IsType(t, &codecmunger.VP8{}, f.codecMunger)
require.True(t, f.codecMunger.GetState().(*livekit.VP8MungerState).PictureIdUsed, "seeded from the packet")
@@ -2463,24 +2463,24 @@ func TestForwarderIsEndOfLayerFrame(t *testing.T) {
vp9Codec := webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP9.String(), ClockRate: 90000}
f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo)
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}}))
f = newForwarder(vp9Codec, webrtc.RTPCodecTypeVideo)
require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: false}}))
require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: true}}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{VP9: &codecs.VP9Packet{E: false}}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{}))
require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{
DependencyDescriptor: &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: true},
},
Payload: codecs.VP9Packet{E: false},
VP9: &codecs.VP9Packet{E: false},
}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{
DependencyDescriptor: &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: false},
},
Payload: codecs.VP9Packet{E: true},
VP9: &codecs.VP9Packet{E: true},
}))
require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{
DependencyDescriptor: &buffer.ExtDependencyDescriptor{},
pkg/sfu/testutils/data.go
+2 -1
View File
@@ -88,7 +88,8 @@ func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.E
}
ep.IsKeyFrame = vp8.IsKeyFrame
ep.Payload = *vp8
vp8Copy := *vp8 // keep the packet independent of later changes to vp8
ep.VP8 = &vp8Copy
if ep.DependencyDescriptor == nil {
ep.Temporal = int32(vp8.TID)
}
pkg/sfu/videolayerselector/temporallayerselector/vp8.go
+2 -3
View File
@@ -16,7 +16,6 @@ package temporallayerselector
import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
)
@@ -37,8 +36,8 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi
return
}
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
vp8 := extPkt.VP8
if vp8 == nil {
return
}
pkg/sfu/videolayerselector/vp9.go
+2 -4
View File
@@ -15,8 +15,6 @@
package videolayerselector
import (
"github.com/pion/rtp/codecs"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/logger"
)
@@ -40,8 +38,8 @@ func (v *VP9) IsOvershootOkay() bool {
}
func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) {
vp9, ok := extPkt.Payload.(codecs.VP9Packet)
if !ok {
vp9 := extPkt.VP9
if vp9 == nil {
return
}