diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index c42cb5bb7..a68e1e4a8 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -993,10 +993,6 @@ func (p *ParticipantImpl) synthesizeAddTrackRequests(offer webrtc.SessionDescrip } func (p *ParticipantImpl) updateRidsFromSDP(offer *webrtc.SessionDescription) { - if !p.params.UseOneShotSignallingMode { - return - } - if offer == nil { return } @@ -1033,6 +1029,7 @@ func (p *ParticipantImpl) updateRidsFromSDP(offer *webrtc.SessionDescription) { for i := n; i < len(pti.sdpRids); i++ { pti.sdpRids[i] = "" } + pti.sdpRids = buffer.NormalizeVideoLayersRid(pti.sdpRids) p.pubLogger.Debugw( "pending track rids updated", diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 5b1ff5e32..2da9d0aad 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -15,20 +15,28 @@ package buffer import ( + "slices" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) const ( - QuarterResolution = "q" - HalfResolution = "h" - FullResolution = "f" + quarterResolutionQ = "q" + halfResolutionH = "h" + fullResolutionF = "f" + + quarterResolution0 = "0" + halfResolution1 = "1" + fullResolution2 = "2" ) type VideoLayersRid [DefaultMaxLayerSpatial + 1]string var ( - DefaultVideoLayersRid = VideoLayersRid{QuarterResolution, HalfResolution, FullResolution} + videoLayersRidQHF = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF} + videoLayersRid012 = VideoLayersRid{quarterResolution0, halfResolution1, fullResolution2} + DefaultVideoLayersRid = videoLayersRidQHF ) // SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers @@ -54,11 +62,11 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo, ridSpace VideoL lp := LayerPresenceFromTrackInfo(trackInfo) if lp == nil { switch rid { - case QuarterResolution: + case quarterResolutionQ: return 0 - case HalfResolution: + case halfResolutionH: return 1 - case FullResolution: + case fullResolutionF: return 2 default: return 0 @@ -129,13 +137,13 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo, ridSpace Video if lp == nil { switch layer { case 0: - return QuarterResolution + return quarterResolutionQ case 1: - return HalfResolution + return halfResolutionH case 2: - return FullResolution + return fullResolutionF default: - return QuarterResolution + return quarterResolutionQ } } @@ -435,3 +443,41 @@ func GetVideoQualityForSpatialLayer(spatialLayer int32, ti *livekit.TrackInfo) l return livekit.VideoQuality_OFF } + +func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool { + for _, rid := range rids { + if rid == "" { + continue + } + + if !slices.Contains(knownRids[:], rid) { + return false + } + } + + return true +} + +func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid { + out := rids + + normalize := func(knownRids VideoLayersRid) { + idx := 0 + for _, known := range knownRids { + if slices.Contains(rids[:], known) { + out[idx] = known + idx++ + } + } + } + + if isVideoLayersRidKnown(rids, videoLayersRidQHF) { + normalize(videoLayersRidQHF) + } + + if isVideoLayersRidKnown(rids, videoLayersRid012) { + normalize(videoLayersRid012) + } + + return out +} diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index 386983064..a47f0c0ee 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -36,20 +36,20 @@ func TestRidConversion(t *testing.T) { "no track info", nil, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, { @@ -60,10 +60,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { @@ -74,10 +74,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { @@ -88,10 +88,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { @@ -103,10 +103,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { @@ -118,10 +118,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { @@ -133,10 +133,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { @@ -149,10 +149,10 @@ func TestRidConversion(t *testing.T) { }, }, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, } @@ -319,18 +319,18 @@ func TestVideoQualityToRidConversion(t *testing.T) { "no track info", nil, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, { "no layers", &livekit.TrackInfo{}, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, { @@ -341,9 +341,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { @@ -354,9 +354,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { @@ -367,9 +367,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { @@ -381,9 +381,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { @@ -395,9 +395,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { @@ -409,9 +409,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { @@ -424,9 +424,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { }, }, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, } @@ -451,9 +451,9 @@ func TestGetSpatialLayerForRid(t *testing.T) { "no track info", nil, map[string]int32{ - QuarterResolution: InvalidLayerSpatial, - HalfResolution: InvalidLayerSpatial, - FullResolution: InvalidLayerSpatial, + quarterResolutionQ: InvalidLayerSpatial, + halfResolutionH: InvalidLayerSpatial, + fullResolutionF: InvalidLayerSpatial, }, }, { @@ -461,12 +461,12 @@ func TestGetSpatialLayerForRid(t *testing.T) { &livekit.TrackInfo{}, map[string]int32{ // SIMULCAST-CODEC-TODO - // QuarterResolution: InvalidLayerSpatial, - // HalfResolution: InvalidLayerSpatial, - // FullResolution: InvalidLayerSpatial, - QuarterResolution: 0, - HalfResolution: 0, - FullResolution: 0, + // quarterResolutionQ: InvalidLayerSpatial, + // halfResolutionH: InvalidLayerSpatial, + // fullResolutionF: InvalidLayerSpatial, + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, }, }, { @@ -484,25 +484,25 @@ func TestGetSpatialLayerForRid(t *testing.T) { }, }, map[string]int32{ - QuarterResolution: 0, - HalfResolution: 0, - FullResolution: 0, + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, }, }, { "layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: QuarterResolution}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: HalfResolution}, + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, }, }, map[string]int32{ - QuarterResolution: 0, - HalfResolution: 1, + quarterResolutionQ: 0, + halfResolutionH: 1, // SIMULCAST-CODEC-TODO - // FullResolution: InvalidLayerSpatial, - FullResolution: 0, + // fullResolutionF: InvalidLayerSpatial, + fullResolutionF: 0, }, }, { @@ -514,9 +514,9 @@ func TestGetSpatialLayerForRid(t *testing.T) { }, }, map[string]int32{ - QuarterResolution: 0, - HalfResolution: 0, - FullResolution: 0, + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, }, }, } @@ -561,8 +561,8 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { "not all layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: QuarterResolution}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: HalfResolution}, + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, }, }, map[livekit.VideoQuality]int32{ @@ -576,9 +576,9 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { "all layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: QuarterResolution}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: HalfResolution}, - {Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: FullResolution}, + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + {Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF}, }, }, map[livekit.VideoQuality]int32{ @@ -630,8 +630,8 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { "layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: QuarterResolution}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: HalfResolution}, + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, }, }, map[int32]livekit.VideoQuality{ @@ -652,3 +652,59 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { }) } } + +func TestNormalizeVideoLayersRid(t *testing.T) { + tests := []struct { + name string + rids VideoLayersRid + normalized VideoLayersRid + }{ + { + "empty", + VideoLayersRid{}, + VideoLayersRid{}, + }, + { + "unknown pattern", + VideoLayersRid{"3", "2", "1"}, + VideoLayersRid{"3", "2", "1"}, + }, + { + "qhf", + videoLayersRidQHF, + videoLayersRidQHF, + }, + { + "scrambled qhf", + VideoLayersRid{"f", "h", "q"}, + videoLayersRidQHF, + }, + { + "partial qhf", + VideoLayersRid{"h", "q"}, + VideoLayersRid{"q", "h", ""}, + }, + { + "012", + videoLayersRid012, + videoLayersRid012, + }, + { + "scrambled 012", + VideoLayersRid{"2", "0", "1"}, + videoLayersRid012, + }, + { + "partial 012", + VideoLayersRid{"2", "1"}, + VideoLayersRid{"1", "2", ""}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + normalizedRids := NormalizeVideoLayersRid(test.rids) + require.Equal(t, test.normalized, normalizedRids) + }) + } +}