dandri/livekit
1
0
Fork 0
mirror of https://github.com/livekit/livekit.git synced 2026-08-27 22:34:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

H265 supoort and codec regression (#3358)

Browse Source 
* H265 supoort and codec regression

Support H265 codec.
Add optional codec regression for subscribers don't
support advanced codecs like H265, AV1, VP9.

* restart forwarder on upstream codec change

* tests

* Reneogitate new codec if client doesn't support change

* Add option to disable codec regression

---------

Co-authored-by: boks1971 <raja.gobi@tutanota.com>
This commit is contained in:
cnderrauber
2025-02-06 11:56:49 +08:00
committed by GitHub
co-authored by boks1971
parent abcaa9b8ab
commit aeec75edeb
27 changed files with 1424 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sfu/buffer/buffer.go
+127 -26
View File
@@ -17,6 +17,7 @@ package buffer
import (
"encoding/binary"
"errors"
"fmt"
"io"
"strings"
"sync"
@@ -79,7 +80,6 @@ type Buffer struct {
maxVideoPkts int
maxAudioPkts int
codecType webrtc.RTPCodecType
payloadType uint8
extPackets deque.Deque[*ExtPacket]
pPackets []pendingPacket
closeOnce sync.Once
@@ -90,7 +90,11 @@ type Buffer struct {
audioLevelExtID uint8
bound bool
closed atomic.Bool
mime string
rtpParameters webrtc.RTPParameters
payloadType uint8
rtxPayloadType uint8
mime string
snRangeMap *utils.RangeMap[uint64, uint64]
@@ -119,6 +123,7 @@ type Buffer struct {
onRtcpSenderReport func()
onFpsChanged func()
onFinalRtpStats func(*livekit.RTPStats)
onCodecChange func(webrtc.RTPCodecParameters)
// logger
logger logger.Logger
@@ -214,6 +219,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
b.clockRate = codec.ClockRate
b.lastReport = mono.UnixNano()
b.mime = strings.ToLower(codec.MimeType)
b.rtpParameters = params
for _, codecParameter := range params.Codecs {
if strings.EqualFold(codecParameter.MimeType, codec.MimeType) {
b.payloadType = uint8(codecParameter.PayloadType)
@@ -226,23 +232,23 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
b.payloadType = uint8(params.Codecs[0].PayloadType)
}
// find RTX payload type
for _, codec := range params.Codecs {
if strings.EqualFold(codec.MimeType, "video/rtx") && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", b.payloadType)) {
b.rtxPayloadType = uint8(codec.PayloadType)
break
}
}
for _, ext := range params.HeaderExtensions {
switch ext.URI {
case dd.ExtensionURI:
if IsSvcCodec(codec.MimeType) || strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) {
if b.ddExtID != 0 {
b.logger.Warnw("multiple dependency descriptor extensions found", nil, "id", ext.ID, "previous", b.ddExtID)
continue
}
b.ddExtID = uint8(ext.ID)
frc := NewFrameRateCalculatorDD(b.clockRate, b.logger)
for i := range b.frameRateCalculator {
b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i))
}
b.ddParser = NewDependencyDescriptorParser(b.ddExtID, b.logger, func(spatial, temporal int32) {
frc.SetMaxLayer(spatial, temporal)
})
if b.ddExtID != 0 {
b.logger.Warnw("multiple dependency descriptor extensions found", nil, "id", ext.ID, "previous", b.ddExtID)
continue
}
b.ddExtID = uint8(ext.ID)
b.createDDParserAndFrameRateCalculator(codec.MimeType)
case sdp.AudioLevelURI:
b.audioLevelExtID = uint8(ext.ID)
@@ -262,16 +268,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
b.codecType = webrtc.RTPCodecTypeVideo
b.bucket = bucket.NewBucket[uint64](InitPacketBufferSizeVideo)
if b.frameRateCalculator[0] == nil {
if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) {
b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger)
}
if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP9) {
frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger)
for i := range b.frameRateCalculator {
b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i))
}
}
b.createFrameRateCalculator(codec.MimeType)
}
if bitrates > 0 {
pps := bitrates / 8 / 1200
@@ -310,6 +307,40 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
b.bound = true
}
func (b *Buffer) OnCodecChange(fn func(webrtc.RTPCodecParameters)) {
b.Lock()
b.onCodecChange = fn
b.Unlock()
}
func (b *Buffer) createDDParserAndFrameRateCalculator(mime string) {
if IsSvcCodec(mime) || strings.EqualFold(mime, webrtc.MimeTypeVP8) {
frc := NewFrameRateCalculatorDD(b.clockRate, b.logger)
for i := range b.frameRateCalculator {
b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i))
}
b.ddParser = NewDependencyDescriptorParser(b.ddExtID, b.logger, func(spatial, temporal int32) {
frc.SetMaxLayer(spatial, temporal)
})
}
}
func (b *Buffer) createFrameRateCalculator(mime string) {
switch {
case strings.EqualFold(mime, webrtc.MimeTypeVP8):
b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger)
case strings.EqualFold(mime, webrtc.MimeTypeVP9):
frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger)
for i := range b.frameRateCalculator {
b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i))
}
case strings.EqualFold(mime, webrtc.MimeTypeH265):
b.frameRateCalculator[0] = NewFrameRateCalculatorH26x(b.clockRate, b.logger)
}
}
// Write adds an RTP Packet, ordering is not guaranteed, newer packets may arrive later
func (b *Buffer) Write(pkt []byte) (n int, err error) {
var rtpPacket rtp.Packet
@@ -364,7 +395,6 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) {
return
}
b.payloadType = rtpPacket.PayloadType
b.calc(pkt, &rtpPacket, now, false)
b.Unlock()
b.readCond.Broadcast()
@@ -397,6 +427,11 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) (n int, err err
return
}
if rtxPkt.PayloadType != b.rtxPayloadType {
b.logger.Debugw("unexpected rtx payload type", "expected", b.rtxPayloadType, "actual", rtxPkt.PayloadType)
return
}
if b.rtxPktBuf == nil {
b.rtxPktBuf = make([]byte, bucket.MaxPktSize)
}
@@ -593,6 +628,10 @@ func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime int64, i
return
}
if !flowState.IsOutOfOrder && rtpPacket.PayloadType != b.payloadType && b.codecType == webrtc.RTPCodecTypeVideo {
b.handleCodecChange(rtpPacket.PayloadType)
}
// add to RTX buffer using sequence number after accounting for dropped padding only packets
snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber)
if err != nil {
@@ -712,6 +751,55 @@ func (b *Buffer) doFpsCalc(ep *ExtPacket) {
}
}
func (b *Buffer) handleCodecChange(newPT uint8) {
var (
codecFound, rtxFound bool
rtxPt uint8
newCodec webrtc.RTPCodecParameters
)
for _, codec := range b.rtpParameters.Codecs {
if !codecFound && uint8(codec.PayloadType) == newPT {
newCodec = codec
codecFound = true
}
if strings.EqualFold(codec.MimeType, "video/rtx") && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", newPT)) {
rtxFound = true
rtxPt = uint8(codec.PayloadType)
}
if codecFound && rtxFound {
break
}
}
if !codecFound {
b.logger.Errorw("could not find codec for new payload type", nil, "pt", newPT, "rtpParameters", b.rtpParameters)
return
}
b.logger.Infow("codec changed",
"oldPayload", b.payloadType, "newPayload", newPT,
"oldRtxPayload", b.rtxPayloadType, "newRtxPayload", rtxPt,
"oldMime", b.mime, "newMime", newCodec.MimeType)
b.payloadType = newPT
b.rtxPayloadType = rtxPt
b.mime = strings.ToLower(newCodec.MimeType)
b.frameRateCalculated = false
if b.ddExtID != 0 {
b.createDDParserAndFrameRateCalculator(b.mime)
}
if b.frameRateCalculator[0] == nil {
b.createFrameRateCalculator(b.mime)
}
b.bucket.ResyncOnNextPacket()
if f := b.onCodecChange; f != nil {
go f(newCodec)
}
}
func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) rtpstats.RTPFlowState {
flowState := b.rtpStats.Update(
arrivalTime,
@@ -825,6 +913,19 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
case utils.MimeTypeAV1:
ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload)
case utils.MimeTypeH265:
if ep.DependencyDescriptor == nil {
if len(rtpPacket.Payload) < 2 {
b.logger.Warnw("invalid H265 packet", nil)
return nil
}
ep.VideoLayer = VideoLayer{
Temporal: int32(rtpPacket.Payload[1]&0x07) - 1,
}
ep.Spatial = InvalidLayerSpatial
}
ep.KeyFrame = IsH265KeyFrame(rtpPacket.Payload)
}
if ep.KeyFrame {