An attempt to use consistent layer mapping (#986)

* WIP commit

* Consistent layers.

* slight re-arrangement of code

* log mime

* fix tests

* map -> array
This commit is contained in:
Raja Subramanian
2022-09-07 09:57:31 +05:30
committed by GitHub
parent aaeba74402
commit d76f7811e9
14 changed files with 832 additions and 151 deletions

View File

@@ -58,7 +58,13 @@ func (d *DynacastQuality) OnSubscribedMaxQualityChange(f func(maxSubscribedQuali
}
func (d *DynacastQuality) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) {
d.params.Logger.Infow("setting subscriber max quality", "mime", d.params.MimeType, "subscriberID", subscriberID, "quality", quality.String())
d.params.Logger.Infow(
"setting subscriber max quality",
"mime", d.params.MimeType,
"subscriberID", subscriberID,
"quality", quality.String(),
)
d.lock.Lock()
if quality == livekit.VideoQuality_OFF {
delete(d.maxSubscriberQuality, subscriberID)
@@ -71,7 +77,13 @@ func (d *DynacastQuality) NotifySubscriberMaxQuality(subscriberID livekit.Partic
}
func (d *DynacastQuality) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) {
d.params.Logger.Infow("setting subscriber node max quality", "mime", d.params.MimeType, "subscriberNodeID", nodeID, "quality", quality.String())
d.params.Logger.Infow(
"setting subscriber node max quality",
"mime", d.params.MimeType,
"subscriberNodeID", nodeID,
"quality", quality.String(),
)
d.lock.Lock()
if quality == livekit.VideoQuality_OFF {
delete(d.maxSubscriberNodeQuality, nodeID)

View File

@@ -19,7 +19,6 @@ import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/twcc"
"github.com/livekit/livekit-server/pkg/telemetry"
"github.com/livekit/livekit-server/pkg/utils"
)
// MediaTrack represents a WebRTC track that needs to be forwarded
@@ -106,28 +105,45 @@ func NewMediaTrack(params MediaTrackParams) *MediaTrack {
t.MediaTrackReceiver.OnSetupReceiver(func(mime string) {
t.dynacastManager.AddCodec(mime)
})
t.MediaTrackReceiver.OnSubscriberMaxQualityChange(func(subscriberID livekit.ParticipantID, codec webrtc.RTPCodecCapability, layer int32) {
t.dynacastManager.NotifySubscriberMaxQuality(subscriberID, codec.MimeType, utils.QualityForSpatialLayer(layer))
})
t.MediaTrackReceiver.OnSubscriberMaxQualityChange(
func(subscriberID livekit.ParticipantID, codec webrtc.RTPCodecCapability, layer int32) {
t.dynacastManager.NotifySubscriberMaxQuality(
subscriberID,
codec.MimeType,
buffer.SpatialLayerToVideoQuality(layer, t.params.TrackInfo),
)
},
)
}
return t
}
func (t *MediaTrack) OnSubscribedMaxQualityChange(f func(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) error) {
if t.dynacastManager != nil {
t.dynacastManager.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) {
if f != nil && !t.IsMuted() {
_ = f(t.ID(), subscribedQualities, maxSubscribedQualities)
}
for _, q := range maxSubscribedQualities {
receiver := t.Receiver(q.CodecMime)
if receiver != nil {
receiver.SetMaxExpectedSpatialLayer(utils.SpatialLayerForQuality(q.Quality))
}
}
})
func (t *MediaTrack) OnSubscribedMaxQualityChange(
f func(
trackID livekit.TrackID,
subscribedQualities []*livekit.SubscribedCodec,
maxSubscribedQualities []types.SubscribedCodecQuality,
) error,
) {
if t.dynacastManager == nil {
return
}
handler := func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) {
if f != nil && !t.IsMuted() {
_ = f(t.ID(), subscribedQualities, maxSubscribedQualities)
}
for _, q := range maxSubscribedQualities {
receiver := t.Receiver(q.CodecMime)
if receiver != nil {
receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(q.Quality, t.params.TrackInfo))
}
}
}
t.dynacastManager.OnSubscribedMaxQualityChange(handler)
}
func (t *MediaTrack) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) {
@@ -267,7 +283,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra
)
}
newWR.OnMaxLayerChange(t.OnMaxLayerChange)
newWR.OnMaxLayerChange(t.onMaxLayerChange)
t.buffer = buff
@@ -315,7 +331,7 @@ func (t *MediaTrack) HasPendingCodec() bool {
return t.MediaTrackReceiver.PrimaryReceiver() == nil
}
func (t *MediaTrack) OnMaxLayerChange(maxLayer int32) {
func (t *MediaTrack) onMaxLayerChange(maxLayer int32) {
ti := &livekit.TrackInfo{
Sid: t.trackInfo.Sid,
Type: t.trackInfo.Type,

View File

@@ -18,7 +18,6 @@ import (
"github.com/livekit/livekit-server/pkg/sfu"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/telemetry"
"github.com/livekit/livekit-server/pkg/utils"
)
const (
@@ -116,7 +115,7 @@ func (t *MediaTrackReceiver) Restart() {
t.lock.Unlock()
for _, receiver := range receivers {
receiver.SetMaxExpectedSpatialLayer(utils.SpatialLayerForQuality(livekit.VideoQuality_HIGH))
receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.TrackInfo))
}
}
@@ -219,7 +218,7 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32)
t.lock.Lock()
defer t.lock.Unlock()
layer := sfu.RidToLayer(rid)
layer := buffer.RidToSpatialLayer(rid, t.params.TrackInfo)
if layer == sfu.InvalidLayerSpatial {
// non-simulcast case will not have `rid`
layer = 0
@@ -774,14 +773,3 @@ func (t *MediaTrackReceiver) SetRTT(rtt uint32) {
}
// ---------------------------
func VideoQualityToRID(q livekit.VideoQuality) string {
switch q {
case livekit.VideoQuality_HIGH:
return sfu.FullResolution
case livekit.VideoQuality_MEDIUM:
return sfu.HalfResolution
default:
return sfu.QuarterResolution
}
}

View File

@@ -1226,13 +1226,15 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w
"kind", track.Kind().String(),
"trackID", publishedTrack.ID(),
"rid", track.RID(),
"SSRC", track.SSRC())
"SSRC", track.SSRC(),
"mime", track.Codec().MimeType)
} else {
p.params.Logger.Warnw("webrtc Track published but can't find MediaTrack", nil,
"kind", track.Kind().String(),
"webrtcTrackID", track.ID(),
"rid", track.RID(),
"SSRC", track.SSRC())
"SSRC", track.SSRC(),
"mime", track.Codec().MimeType)
}
if !isNewTrack && publishedTrack != nil && !publishedTrack.HasPendingCodec() && p.IsReady() {

View File

@@ -11,7 +11,7 @@ import (
"github.com/livekit/livekit-server/pkg/rtc/types"
"github.com/livekit/livekit-server/pkg/sfu"
"github.com/livekit/livekit-server/pkg/utils"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
const (
@@ -58,7 +58,9 @@ func (t *SubscribedTrack) OnBind(f func()) {
func (t *SubscribedTrack) Bound() {
t.bound.Store(true)
if !t.params.AdaptiveStream {
t.params.DownTrack.SetMaxSpatialLayer(utils.SpatialLayerForQuality(livekit.VideoQuality_HIGH))
t.params.DownTrack.SetMaxSpatialLayer(
buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.MediaTrack.ToProto()),
)
}
t.maybeOnBind()
}
@@ -141,7 +143,7 @@ func (t *SubscribedTrack) UpdateVideoLayer() {
if settings.Width > 0 {
quality = t.MediaTrack().GetQualityForDimension(settings.Width, settings.Height)
}
t.DownTrack().SetMaxSpatialLayer(utils.SpatialLayerForQuality(quality))
t.DownTrack().SetMaxSpatialLayer(buffer.VideoQualityToSpatialLayer(quality, t.params.MediaTrack.ToProto()))
}
func (t *SubscribedTrack) updateDownTrackMute() {

View File

@@ -476,8 +476,6 @@ func (b *Buffer) getExtPacket(rawPacket []byte, rtpPacket *rtp.Packet, arrivalTi
}
case "video/h264":
ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload)
case "video/vp9":
ep.KeyFrame = IsVp9Keyframe(rtpPacket.Payload)
case "video/av1":
ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload)
}

View File

@@ -5,8 +5,6 @@ import (
"errors"
"time"
"github.com/pion/rtp/codecs"
"github.com/livekit/protocol/logger"
)
@@ -279,30 +277,6 @@ func IsH264Keyframe(payload []byte) bool {
return false
}
// IsAV1Keyframe detects if vp9 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsVp9Keyframe(payload []byte) bool {
var vp9 codecs.VP9Packet
_, err := vp9.Unmarshal(payload)
if err != nil || len(vp9.Payload) < 1 {
return false
}
if !vp9.B {
return false
}
if (vp9.Payload[0] & 0xc0) != 0x80 {
return false
}
profile := (vp9.Payload[0] >> 4) & 0x3
if profile != 3 {
return (vp9.Payload[0] & 0xC) == 0
}
return (vp9.Payload[0] & 0x6) == 0
}
// IsAV1Keyframe detects if av1 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU

View File

@@ -0,0 +1,306 @@
package buffer
import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
const (
QuarterResolution = "q"
HalfResolution = "h"
FullResolution = "f"
)
func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
if trackInfo == nil || len(trackInfo.Layers) == 0 {
return nil
}
var layerPresence [livekit.VideoQuality_HIGH + 1]bool
for _, layer := range trackInfo.Layers {
layerPresence[layer.Quality] = true
}
return &layerPresence
}
func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch rid {
case QuarterResolution:
return 0
case HalfResolution:
return 1
case FullResolution:
return 2
default:
return 0
}
}
switch rid {
case QuarterResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 0
default:
// only one quality published, could be any
return 0
}
case HalfResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 1
default:
// only one quality published, could be any
return 0
}
case FullResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 2
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected rid f with only two qualities, low and medium", nil)
return 1
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, low and high", nil)
return 1
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, medum and high", nil)
return 1
default:
// only one quality published, could be any
return 0
}
default:
// no rid, should be single layer
return 0
}
}
func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch layer {
case 0:
return QuarterResolution
case 1:
return HalfResolution
case 2:
return FullResolution
default:
return QuarterResolution
}
}
switch layer {
case 0:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return QuarterResolution
default:
return QuarterResolution
}
case 1:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return HalfResolution
default:
return QuarterResolution
}
case 2:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return FullResolution
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil)
return HalfResolution
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil)
return HalfResolution
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, medum and high", nil)
return HalfResolution
default:
return QuarterResolution
}
default:
return QuarterResolution
}
}
func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string {
return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo)
}
func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch layer {
case 0:
return livekit.VideoQuality_LOW
case 1:
return livekit.VideoQuality_MEDIUM
case 2:
return livekit.VideoQuality_HIGH
default:
return livekit.VideoQuality_OFF
}
}
switch layer {
case 0:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW]:
return livekit.VideoQuality_LOW
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM]:
return livekit.VideoQuality_MEDIUM
default:
return livekit.VideoQuality_HIGH
}
case 1:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
return livekit.VideoQuality_MEDIUM
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return livekit.VideoQuality_HIGH
default:
logger.Errorw("invalid layer", nil, "layer", layer, "trackInfo", trackInfo)
return livekit.VideoQuality_HIGH
}
case 2:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return livekit.VideoQuality_HIGH
default:
logger.Errorw("invalid layer", nil, "layer", layer, "trackInfo", trackInfo)
return livekit.VideoQuality_HIGH
}
}
return livekit.VideoQuality_OFF
}
func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch quality {
case livekit.VideoQuality_LOW:
return 0
case livekit.VideoQuality_MEDIUM:
return 1
case livekit.VideoQuality_HIGH:
return 2
default:
return InvalidLayerSpatial
}
}
switch quality {
case livekit.VideoQuality_LOW:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
default: // only one quality published, could be any
return 0
}
case livekit.VideoQuality_MEDIUM:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
return 1
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 0
default: // only one quality published, could be any
return 0
}
case livekit.VideoQuality_HIGH:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 2
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 1
default: // only one quality published, could be any
return 0
}
}
return InvalidLayerSpatial
}

View File

@@ -0,0 +1,427 @@
package buffer
import (
"testing"
"github.com/livekit/protocol/livekit"
"github.com/stretchr/testify/require"
)
func TestRidConversion(t *testing.T) {
type RidAndLayer struct {
rid string
layer int32
}
tests := []struct {
name string
trackInfo *livekit.TrackInfo
ridToLayer map[string]RidAndLayer
}{
{
"no track info",
nil,
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: FullResolution, layer: 2},
},
},
{
"no layers",
&livekit.TrackInfo{},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: FullResolution, layer: 2},
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: HalfResolution, layer: 1},
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: HalfResolution, layer: 1},
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: HalfResolution, layer: 1},
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": RidAndLayer{rid: QuarterResolution, layer: 0},
QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0},
HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1},
FullResolution: RidAndLayer{rid: FullResolution, layer: 2},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testRid, expectedResult := range test.ridToLayer {
actualLayer := RidToSpatialLayer(testRid, test.trackInfo)
require.Equal(t, expectedResult.layer, actualLayer)
actualRid := SpatialLayerToRid(actualLayer, test.trackInfo)
require.Equal(t, expectedResult.rid, actualRid)
}
})
}
}
func TestQualityConversion(t *testing.T) {
type QualityAndLayer struct {
quality livekit.VideoQuality
layer int32
}
tests := []struct {
name string
trackInfo *livekit.TrackInfo
qualityToLayer map[livekit.VideoQuality]QualityAndLayer
}{
{
"no track info",
nil,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
{
"no layers",
&livekit.TrackInfo{},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0},
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0},
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1},
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1},
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1},
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedResult := range test.qualityToLayer {
actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo)
require.Equal(t, expectedResult.layer, actualLayer)
actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo)
require.Equal(t, expectedResult.quality, actualQuality)
}
})
}
}
func TestVideoQualityToRidConversion(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
qualityToRid map[livekit.VideoQuality]string
}{
{
"no track info",
nil,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
{
"no layers",
&livekit.TrackInfo{},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
&livekit.VideoLayer{Quality: livekit.VideoQuality_LOW},
&livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM},
&livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedRid := range test.qualityToRid {
actualRid := VideoQualityToRid(testQuality, test.trackInfo)
require.Equal(t, expectedRid, actualRid)
}
})
}
}

View File

@@ -11,7 +11,6 @@ import (
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/utils"
)
const (
@@ -82,7 +81,7 @@ func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo) {
cs.normFactors[0] = MaxScore / AudioTrackScore(params, 1)
} else {
for _, layer := range cs.trackInfo.Layers {
spatial := utils.SpatialLayerForQuality(layer.Quality)
spatial := buffer.VideoQualityToSpatialLayer(layer.Quality, cs.trackInfo)
// LK-TODO: would be good to have expected frame rate in Trackinfo
frameRate := uint32(30)
switch spatial {
@@ -99,7 +98,7 @@ func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo) {
Height: layer.Height,
Frames: frameRate,
}
cs.normFactors[utils.SpatialLayerForQuality(layer.Quality)] = MaxScore / VideoTrackScore(params, 1)
cs.normFactors[spatial] = MaxScore / VideoTrackScore(params, 1)
}
}
cs.lock.Unlock()
@@ -132,7 +131,7 @@ func (cs *ConnectionStats) getLayerDimensions(layer int32) (uint32, uint32) {
}
for _, l := range cs.trackInfo.Layers {
if layer == utils.SpatialLayerForQuality(l.Quality) {
if layer == buffer.VideoQualityToSpatialLayer(l.Quality, cs.trackInfo) {
return l.Width, l.Height
}
}

View File

@@ -11,12 +11,6 @@ import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
const (
QuarterResolution = "q"
HalfResolution = "h"
FullResolution = "f"
)
// Do a fuzzy find for a codec in the list of codecs
// Used for lookup up a codec in an existing list to find a match
func codecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []webrtc.RTPCodecParameters) (webrtc.RTPCodecParameters, error) {

View File

@@ -12,7 +12,6 @@ import (
"github.com/pion/webrtc/v3"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
@@ -109,19 +108,6 @@ type WebRTCReceiver struct {
primaryReceiver atomic.Value // *RedPrimaryReceiver
}
func RidToLayer(rid string) int32 {
switch rid {
case FullResolution:
return 2
case HalfResolution:
return 1
case QuarterResolution:
return 0
default:
return InvalidLayerSpatial
}
}
func IsSvcCodec(mime string) bool {
switch strings.ToLower(mime) {
case "video/av1":
@@ -312,14 +298,9 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
return
}
layer := RidToLayer(track.RID())
if layer == InvalidLayerSpatial {
// check if there is only one layer and if so, assign it to that
if len(w.trackInfo.Layers) == 1 {
layer = utils.SpatialLayerForQuality(w.trackInfo.Layers[0].Quality)
} else {
layer = 0
}
layer := int32(0)
if w.Kind() == webrtc.RTPCodecTypeVideo {
layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo)
}
buff.SetLogger(logger.Logger(logr.Logger(w.logger).WithValues("layer", layer)))
buff.SetTWCC(w.twcc)
@@ -332,12 +313,12 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
buff.OnRtcpFeedback(w.sendRTCP)
var duration time.Duration
switch track.RID() {
case FullResolution:
switch layer {
case 2:
duration = w.pliThrottleConfig.HighQuality
case HalfResolution:
case 1:
duration = w.pliThrottleConfig.MidQuality
case QuarterResolution:
case 0:
duration = w.pliThrottleConfig.LowQuality
default:
duration = w.pliThrottleConfig.MidQuality
@@ -549,6 +530,9 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) {
})
w.streamTrackerManager.RemoveTracker(layer)
if w.isSVC {
w.streamTrackerManager.RemoveAllTrackers()
}
}()
for {

View File

@@ -7,7 +7,7 @@ import (
"github.com/go-logr/logr"
"github.com/livekit/livekit-server/pkg/utils"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
@@ -89,7 +89,7 @@ func NewStreamTrackerManager(logger logger.Logger, trackInfo *livekit.TrackInfo)
}
for _, layer := range s.trackInfo.Layers {
spatialLayer := utils.SpatialLayerForQuality(layer.Quality)
spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, trackInfo)
if spatialLayer > s.maxPublishedLayer {
s.maxPublishedLayer = spatialLayer
}
@@ -263,7 +263,7 @@ func (s *StreamTrackerManager) GetLayerDimension(layer int32) (uint32, uint32) {
height := uint32(0)
width := uint32(0)
if len(s.trackInfo.Layers) > 0 {
quality := utils.QualityForSpatialLayer(layer)
quality := buffer.SpatialLayerToVideoQuality(layer, s.trackInfo)
for _, layer := range s.trackInfo.Layers {
if layer.Quality == quality {
height = layer.Height
@@ -376,7 +376,12 @@ func (s *StreamTrackerManager) addAvailableLayer(layer int32) {
exemptedLayers = append(exemptedLayers, s.exemptedLayers...)
s.lock.Unlock()
s.logger.Debugw("available layers changed - layer seen", "added", layer, "availableLayers", availableLayers, "exemptedLayers", exemptedLayers)
s.logger.Debugw(
"available layers changed - layer seen",
"added", layer,
"availableLayers", availableLayers,
"exemptedLayers", exemptedLayers,
)
if s.onAvailableLayersChanged != nil {
s.onAvailableLayersChanged(availableLayers, exemptedLayers)
@@ -430,16 +435,18 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
//
// add to exempt if not already present
//
found := false
for _, el := range s.exemptedLayers {
if el == layer {
found = true
break
if exempt {
found := false
for _, el := range s.exemptedLayers {
if el == layer {
found = true
break
}
}
if !found {
s.exemptedLayers = append(s.exemptedLayers, layer)
sort.Slice(s.exemptedLayers, func(i, j int) bool { return s.exemptedLayers[i] < s.exemptedLayers[j] })
}
}
if !found && exempt {
s.exemptedLayers = append(s.exemptedLayers, layer)
sort.Slice(s.exemptedLayers, func(i, j int) bool { return s.exemptedLayers[i] < s.exemptedLayers[j] })
}
var exemptedLayers []int32
@@ -451,7 +458,12 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
}
s.lock.Unlock()
s.logger.Debugw("available layers changed - layer gone", "removed", layer, "layers", newLayers)
s.logger.Debugw(
"available layers changed - layer gone",
"removed", layer,
"availableLayers", newLayers,
"exeptedLayers", exemptedLayers,
)
// need to immediately switch off unavailable layers
if s.onAvailableLayersChanged != nil {

View File

@@ -1,33 +0,0 @@
package utils
import (
"github.com/livekit/protocol/livekit"
)
func SpatialLayerForQuality(quality livekit.VideoQuality) int32 {
switch quality {
case livekit.VideoQuality_LOW:
return 0
case livekit.VideoQuality_MEDIUM:
return 1
case livekit.VideoQuality_HIGH:
return 2
case livekit.VideoQuality_OFF:
return -1
default:
return -1
}
}
func QualityForSpatialLayer(layer int32) livekit.VideoQuality {
switch layer {
case 0:
return livekit.VideoQuality_LOW
case 1:
return livekit.VideoQuality_MEDIUM
case 2:
return livekit.VideoQuality_HIGH
default:
return livekit.VideoQuality_OFF
}
}