diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index de637e9df..1b01b8874 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -40,16 +40,14 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, PayloadType: 96, }, - /* - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 98, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 100, - }, - */ + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 98, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 100, + }, { RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, PayloadType: 125, diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 6ae4b8715..3edb71b9e 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -132,7 +132,7 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess mime = strings.ToUpper(mime) // remove dd extension if av1 not preferred - if !strings.Contains(mime, "AV1") { + if !strings.Contains(mime, "AV1") && !strings.Contains(mime, "VP9") { for i, attr := range unmatchVideo.Attributes { if strings.Contains(attr.Value, dd.ExtensionUrl) { unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 5a04d66c1..1cfc7b2de 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -564,6 +564,8 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload) case "video/av1": ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload) + case "video/vp9": + ep.KeyFrame = IsVP9Keyframe(rtpPacket.Payload) } if ep.KeyFrame { diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index cacc7ad1f..9545ab049 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -4,6 +4,8 @@ import ( "encoding/binary" "errors" + "github.com/pion/rtp/codecs" + "github.com/livekit/protocol/logger" ) @@ -351,4 +353,28 @@ func IsAV1Keyframe(payload []byte) bool { } } +// IsVP9Keyframe detects if vp9 payload is a keyframe +// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go +// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU +func IsVP9Keyframe(payload []byte) bool { + var vp9 codecs.VP9Packet + _, err := vp9.Unmarshal(payload) + if err != nil || len(vp9.Payload) < 1 { + return false + } + if !vp9.B { + return false + } + + if (vp9.Payload[0] & 0xc0) != 0x80 { + return false + } + + profile := (vp9.Payload[0] >> 4) & 0x3 + if profile != 3 { + return (vp9.Payload[0] & 0xC) == 0 + } + return (vp9.Payload[0] & 0x6) == 0 +} + // ------------------------------------- diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1ef7674f2..1104fb6d0 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -273,8 +273,8 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { case "video/vp8": f.isTemporalSupported = true f.vp8Munger = NewVP8Munger(f.logger) - case "video/av1": - // TODO : we only enable dd layer selector for av1 now, at future we can + case "video/av1", "video/vp9": + // TODO : we only enable dd layer selector for av1 and vp9 now, at future we can // enable it for vp8 too f.ddLayerSelector = NewDDVideoLayerSelector(f.logger) } @@ -515,7 +515,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow } alloc.TargetLayers = buffer.VideoLayer{ Spatial: int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))), - Temporal: buffer.DefaultMaxLayerTemporal, + Temporal: f.maxLayers.Temporal, } } @@ -570,14 +570,14 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.TargetLayers.Spatial = l } } - alloc.TargetLayers.Temporal = buffer.DefaultMaxLayerTemporal + alloc.TargetLayers.Temporal = f.maxLayers.Temporal alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial { // current is locked to desired, stay there - alloc.TargetLayers = f.currentLayers + alloc.TargetLayers = buffer.VideoLayer{Spatial: f.requestLayerSpatial, Temporal: f.maxLayers.Temporal} alloc.RequestLayerSpatial = f.requestLayerSpatial } else { // opportunistically latch on to anything diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 81778adae..c543824ad 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -387,7 +387,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.requestLayerSpatial = 0 expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, - Temporal: 3, + Temporal: 1, } expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, @@ -397,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { TargetLayers: expectedTargetLayers, RequestLayerSpatial: 2, MaxLayers: f.maxLayers, - DistanceToDesired: -1.5, + DistanceToDesired: -1, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result)