From 1b2289137d4cc36e9239b3fa879baabf17cfef71 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 9 Aug 2025 21:26:11 +0530 Subject: [PATCH] Support video layer mode from client and make most of the code mime aware (#3843) --- go.mod | 8 +- go.sum | 16 +- pkg/rtc/mediatrack.go | 26 +- pkg/rtc/mediatrack_test.go | 150 ++++---- pkg/rtc/mediatrackreceiver.go | 47 +-- pkg/rtc/mediatracksubscriptions.go | 9 +- pkg/rtc/participant.go | 53 ++- pkg/rtc/subscribedtrack.go | 7 +- pkg/rtc/types/interfaces.go | 6 +- .../typesfakes/fake_local_media_track.go | 107 ++---- pkg/rtc/types/typesfakes/fake_media_track.go | 107 ++---- pkg/rtc/wrappedreceiver.go | 8 + pkg/sfu/buffer/videolayerutils.go | 110 ++++-- pkg/sfu/buffer/videolayerutils_test.go | 348 +++++++++++++----- pkg/sfu/downtrack.go | 28 +- pkg/sfu/forwarder.go | 74 ++-- pkg/sfu/forwarder_test.go | 3 +- pkg/sfu/receiver.go | 65 ++-- pkg/sfu/redprimaryreceiver.go | 3 +- pkg/sfu/redreceiver.go | 3 +- pkg/sfu/streamtrackermanager.go | 19 +- 21 files changed, 673 insertions(+), 524 deletions(-) diff --git a/go.mod b/go.mod index ed02dcdd3..c0add4590 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded - github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a + github.com/livekit/protocol v1.39.4-0.20250809061103-746c9d68529a github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741 github.com/mackerelio/go-osstat v0.2.5 github.com/magefile/mage v1.15.0 @@ -55,7 +55,7 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 + golang.org/x/exp v0.0.0-20250808145144-a408d31f581a golang.org/x/mod v0.27.0 golang.org/x/sync v0.16.0 google.golang.org/protobuf v1.36.7 @@ -63,7 +63,7 @@ require ( ) require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250717185734-6c6e0d3c608e.1 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.7-20250717185734-6c6e0d3c608e.1 // indirect buf.build/go/protovalidate v0.14.0 // indirect buf.build/go/protoyaml v0.6.0 // indirect cel.dev/expr v0.24.0 // indirect @@ -134,7 +134,7 @@ require ( golang.org/x/net v0.43.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/tools v0.36.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect google.golang.org/grpc v1.74.2 // indirect diff --git a/go.sum b/go.sum index ec456cc08..c617a449e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250717185734-6c6e0d3c608e.1 h1:Lg6klmCi3v7VvpqeeLEER9/m5S8y9e9DjhqQnSCNy4k= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250717185734-6c6e0d3c608e.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.7-20250717185734-6c6e0d3c608e.1 h1:/AZH8sVB6LHv8G+hZlAMCP31NevnesHwYgnlgS5Vt14= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.7-20250717185734-6c6e0d3c608e.1/go.mod h1:eva/VCrd8X7xuJw+JtwCEyrCKiRRASukFqmirnWBvFU= buf.build/go/protovalidate v0.14.0 h1:kr/rC/no+DtRyYX+8KXLDxNnI1rINz0imk5K44ZpZ3A= buf.build/go/protovalidate v0.14.0/go.mod h1:+F/oISho9MO7gJQNYC2VWLzcO1fTPmaTA08SDYJZncA= buf.build/go/protoyaml v0.6.0 h1:Nzz1lvcXF8YgNZXk+voPPwdU8FjDPTUV4ndNTXN0n2w= @@ -167,8 +167,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded h1:ylZPdnlX1RW9Z15SD4mp87vT2D2shsk0hpLJwSPcq3g= github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded/go.mod h1:mSNtYzSf6iY9xM3UX42VEI+STHvMgHmrYzEHPcdhB8A= -github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a h1:hqju+FP5ly5F1zg25DpP1jqgSdv9sIglDM03jgw1864= -github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU= +github.com/livekit/protocol v1.39.4-0.20250809061103-746c9d68529a h1:Hk8t/FBNu6pFu196nXoBLoIsvUcbV4JCW4eAPHxHnfY= +github.com/livekit/protocol v1.39.4-0.20250809061103-746c9d68529a/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU= github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741 h1:KKL1u94l6dF9u4cBwnnfozk27GH1txWy2SlvkfgmzoY= github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741/go.mod h1:AuDC5uOoEjQJEc69v4Li3t77Ocz0e0NdjQEuFfO+vfk= github.com/mackerelio/go-osstat v0.2.5 h1:+MqTbZUhoIt4m8qzkVoXUJg1EuifwlAJSk4Yl2GXh+o= @@ -361,8 +361,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= -golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90= +golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -465,8 +465,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 453516e45..b858e2f7c 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -39,6 +39,8 @@ import ( util "github.com/livekit/mediatransportutil" ) +var _ types.LocalMediaTrack = (*MediaTrack)(nil) + // MediaTrack represents a WebRTC track that needs to be forwarded // Implements MediaTrack and PublishedTrack interface type MediaTrack struct { @@ -134,7 +136,7 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { t.dynacastManager.NotifySubscriberMaxQuality( subscriberID, mimeType, - buffer.GetVideoQualityForSpatialLayer(layer, t.MediaTrackReceiver.TrackInfo()), + buffer.GetVideoQualityForSpatialLayer(mimeType, layer, t.MediaTrackReceiver.TrackInfo()), ) }, ) @@ -143,6 +145,7 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { }) } + t.SetMuted(ti.Muted) return t } @@ -166,7 +169,9 @@ func (t *MediaTrack) OnSubscribedMaxQualityChange( for _, q := range maxSubscribedQualities { receiver := t.Receiver(q.CodecMime) if receiver != nil { - receiver.SetMaxExpectedSpatialLayer(buffer.GetSpatialLayerForVideoQuality(q.Quality, t.MediaTrackReceiver.TrackInfo())) + receiver.SetMaxExpectedSpatialLayer( + buffer.GetSpatialLayerForVideoQuality(q.CodecMime, q.Quality, t.MediaTrackReceiver.TrackInfo()), + ) } } } @@ -261,7 +266,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe t.lock.Lock() var regressCodec bool mimeType := mime.NormalizeMimeType(track.Codec().MimeType) - layer := buffer.GetSpatialLayerForRid(track.RID(), ti) + layer := buffer.GetSpatialLayerForRid(mimeType, track.RID(), ti) if layer < 0 { t.params.Logger.Warnw( "AddReceiver failed due to negative layer", nil, @@ -367,13 +372,13 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } }) - newWR.OnMaxLayerChange(func(maxLayer int32) { + newWR.OnMaxLayerChange(func(mimeType mime.MimeType, maxLayer int32) { // send for only one codec, either primary (priority == 0) OR regressed codec t.lock.RLock() regressionTargetCodecReceived := t.regressionTargetCodecReceived t.lock.RUnlock() if priority == 0 || regressionTargetCodecReceived { - t.MediaTrackReceiver.NotifyMaxLayerChange(maxLayer) + t.MediaTrackReceiver.NotifyMaxLayerChange(mimeType, maxLayer) } }) // SIMULCAST-CODEC-TODO END: these need to be receiver/mime aware, setting it up only for primary now @@ -437,15 +442,10 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe return newCodec, false } - // LK-TODO: can remove this completely when VideoLayers protocol becomes the default as it has info from client or if we decide to use TrackInfo.Simulcast - if t.numUpTracks.Inc() > 1 || track.RID() != "" { - // cannot only rely on numUpTracks since we fire metadata events immediately after the first layer - t.SetSimulcast(true) - } - var bitrates int - if layer >= 0 && len(ti.Layers) > int(layer) { - bitrates = int(ti.Layers[layer].GetBitrate()) + layers := buffer.GetVideoLayersForMimeType(mimeType, ti) + if layer >= 0 && len(layers) > int(layer) { + bitrates = int(layers[layer].GetBitrate()) } t.MediaTrackReceiver.SetLayerSsrc(mimeType, track.RID(), uint32(track.SSRC())) diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 8d96bbae4..f9d5c03e6 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/protocol/livekit" ) @@ -47,10 +48,6 @@ func TestTrackInfo(t *testing.T) { require.Equal(t, ti.Width, outInfo.Width) require.Equal(t, ti.Height, outInfo.Height) require.Equal(t, ti.Simulcast, outInfo.Simulcast) - - // make it simulcasted - mt.SetSimulcast(true) - require.True(t, mt.ToProto().Simulcast) } func TestGetQualityForDimension(t *testing.T) { @@ -61,11 +58,11 @@ func TestGetQualityForDimension(t *testing.T) { Height: 720, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 200)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(200, 250)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(700, 480)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(500, 1000)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 300, 200)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 200, 250)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 700, 480)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 500, 1000)) }) t.Run("portrait source", func(t *testing.T) { @@ -75,10 +72,10 @@ func TestGetQualityForDimension(t *testing.T) { Height: 960, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(200, 400)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 400)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(600, 900)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 200, 400)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 400, 400)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 400, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 600, 900)) }) t.Run("layers provided", func(t *testing.T) { @@ -86,29 +83,34 @@ func TestGetQualityForDimension(t *testing.T) { Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 960, - Height: 540, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 960, + Height: 540, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) }) t.Run("highest layer with smallest dimensions", func(t *testing.T) { @@ -116,59 +118,69 @@ func TestGetQualityForDimension(t *testing.T) { Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 1080, - Height: 720, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 1080, + Height: 720, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1200, 800)) mt = NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1200, 800)) }) } diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 4ed7c08cc..7d6079a4b 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -166,17 +166,12 @@ func NewMediaTrackReceiver(params MediaTrackReceiverParams, ti *livekit.TrackInf Logger: params.Logger, }) t.MediaTrackSubscriptions.OnDownTrackCreated(t.onDownTrackCreated) - - if ti.Muted { - t.SetMuted(true) - } return t } func (t *MediaTrackReceiver) Restart() { - hq := buffer.GetSpatialLayerForVideoQuality(livekit.VideoQuality_HIGH, t.TrackInfo()) - for _, receiver := range t.loadReceivers() { + hq := buffer.GetSpatialLayerForVideoQuality(receiver.Mime(), livekit.VideoQuality_HIGH, t.TrackInfo()) receiver.SetMaxExpectedSpatialLayer(hq) } } @@ -493,19 +488,6 @@ func (t *MediaTrackReceiver) PublisherVersion() uint32 { return t.params.ParticipantVersion } -func (t *MediaTrackReceiver) IsSimulcast() bool { - return t.TrackInfo().Simulcast -} - -func (t *MediaTrackReceiver) SetSimulcast(simulcast bool) { - t.lock.Lock() - defer t.lock.Unlock() - - trackInfo := t.TrackInfoClone() - trackInfo.Simulcast = simulcast - t.trackInfo.Store(trackInfo) -} - func (t *MediaTrackReceiver) Name() string { return t.TrackInfo().Name } @@ -671,12 +653,12 @@ func (t *MediaTrackReceiver) updateTrackInfoOfReceivers() { func (t *MediaTrackReceiver) SetLayerSsrc(mimeType mime.MimeType, rid string, ssrc uint32) { t.lock.Lock() trackInfo := t.TrackInfoClone() - layer := buffer.GetSpatialLayerForRid(rid, trackInfo) + layer := buffer.GetSpatialLayerForRid(mimeType, rid, trackInfo) if layer == buffer.InvalidLayerSpatial { // non-simulcast case will not have `rid` layer = 0 } - quality := buffer.GetVideoQualityForSpatialLayer(layer, trackInfo) + quality := buffer.GetVideoQualityForSpatialLayer(mimeType, layer, trackInfo) // set video layer ssrc info for i, ci := range trackInfo.Codecs { if mime.NormalizeMimeType(ci.MimeType) != mimeType { @@ -725,7 +707,15 @@ func (t *MediaTrackReceiver) UpdateCodecInfo(codecs []*livekit.SimulcastCodec) { clonedLayers = append(clonedLayers, utils.CloneProto(l)) } origin.Layers = clonedLayers + + mimeType := mime.NormalizeMimeType(origin.MimeType) + for _, layer := range origin.Layers { + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, trackInfo) + // SIMULCAST-CODEC-TODO: need to map RIDs from SDP -> SimulcastCodecInfo + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, trackInfo, buffer.DefaultVideoLayersRid) + } } + break } } @@ -852,16 +842,17 @@ func (t *MediaTrackReceiver) TrackInfoClone() *livekit.TrackInfo { return utils.CloneProto(t.TrackInfo()) } -func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { +func (t *MediaTrackReceiver) NotifyMaxLayerChange(mimeType mime.MimeType, maxLayer int32) { trackInfo := t.TrackInfo() - quality := buffer.GetVideoQualityForSpatialLayer(maxLayer, trackInfo) + quality := buffer.GetVideoQualityForSpatialLayer(mimeType, maxLayer, trackInfo) ti := &livekit.TrackInfo{ Sid: trackInfo.Sid, Type: trackInfo.Type, Layers: []*livekit.VideoLayer{{Quality: quality}}, } if quality != livekit.VideoQuality_OFF { - for _, layer := range trackInfo.Layers { + layers := buffer.GetVideoLayersForMimeType(mimeType, trackInfo) + for _, layer := range layers { if layer.Quality == quality { ti.Layers[0].Width = layer.Width ti.Layers[0].Height = layer.Height @@ -875,7 +866,7 @@ func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { // GetQualityForDimension finds the closest quality to use for desired dimensions // affords a 20% tolerance on dimension -func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) livekit.VideoQuality { +func (t *MediaTrackReceiver) GetQualityForDimension(mimeType mime.MimeType, width, height uint32) livekit.VideoQuality { quality := livekit.VideoQuality_HIGH if t.Kind() == livekit.TrackType_AUDIO { return quality @@ -897,7 +888,7 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki // default sizes representing qualities low - high layerSizes := []uint32{180, 360, origSize} var providedSizes []uint32 - for _, layer := range trackInfo.Layers { + for _, layer := range buffer.GetVideoLayersForMimeType(mimeType, trackInfo) { providedSizes = append(providedSizes, layer.Height) } if len(providedSizes) > 0 { @@ -1004,8 +995,8 @@ func (t *MediaTrackReceiver) SetRTT(rtt uint32) { } } -func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime mime.MimeType) int32 { - receiver := t.Receiver(mime) +func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(mimeType mime.MimeType, spatial int32, fps uint32) int32 { + receiver := t.Receiver(mimeType) if receiver == nil { return buffer.DefaultMaxLayerTemporal } diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index ce2aa037a..9026a5d84 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -181,8 +181,13 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * if !wr.DetermineReceiver(codec) { if t.onSubscriberMaxQualityChange != nil { go func() { - spatial := buffer.GetSpatialLayerForVideoQuality(livekit.VideoQuality_HIGH, t.params.MediaTrack.ToProto()) - t.onSubscriberMaxQualityChange(downTrack.SubscriberID(), mime.NormalizeMimeType(codec.MimeType), spatial) + mimeType := mime.NormalizeMimeType(codec.MimeType) + spatial := buffer.GetSpatialLayerForVideoQuality( + mimeType, + livekit.VideoQuality_HIGH, + t.params.MediaTrack.ToProto(), + ) + t.onSubscriberMaxQualityChange(downTrack.SubscriberID(), mimeType, spatial) }() } } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 58121423b..6cdbca123 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1050,7 +1050,6 @@ func (p *ParticipantImpl) synthesizeAddTrackRequests(offer webrtc.SessionDescrip Stereo: false, Stream: "camera", } - // ONE-SHOT-SIGNALLING-MODE-TODO: simulcsat layer mapping if strings.EqualFold(m.MediaName.Media, "video") { if ridsOk { // add simulcast layers, NOTE: only quality can be set as dimensions/fps is not available @@ -2583,7 +2582,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange( Sid: trackInfo.Sid, Type: trackInfo.Type, } - for _, layer := range trackInfo.Layers { + for _, layer := range buffer.GetVideoLayersForMimeType(maxSubscribedQuality.CodecMime, trackInfo) { if layer.Quality == maxSubscribedQuality.Quality { ti.Width = layer.Width ti.Height = layer.Height @@ -2637,6 +2636,14 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l backupCodecPolicy = livekit.BackupCodecPolicy_SIMULCAST } + cloneLayers := func(layers []*livekit.VideoLayer) []*livekit.VideoLayer { + clonedLayers := make([]*livekit.VideoLayer, 0, len(layers)) + for _, l := range layers { + clonedLayers = append(clonedLayers, utils.CloneProto(l)) + } + return clonedLayers + } + ti := &livekit.TrackInfo{ Type: req.Type, Name: req.Name, @@ -2645,7 +2652,7 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l Muted: req.Muted, DisableDtx: req.DisableDtx, Source: req.Source, - Layers: req.Layers, + Layers: cloneLayers(req.Layers), DisableRed: req.DisableRed, Stereo: req.Stereo, Encryption: req.Encryption, @@ -2664,14 +2671,6 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } p.setTrackID(req.Cid, ti) - cloneLayers := func(layers []*livekit.VideoLayer) []*livekit.VideoLayer { - clonedLayers := make([]*livekit.VideoLayer, 0, len(layers)) - for _, l := range layers { - clonedLayers = append(clonedLayers, utils.CloneProto(l)) - } - return clonedLayers - } - if len(req.SimulcastCodecs) == 0 { if req.Type == livekit.TrackType_VIDEO { // clients not supporting simulcast codecs, synthesise a codec @@ -2688,6 +2687,7 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } mimeType := codec.Codec + videoLayerMode := codec.VideoLayerMode if req.Type == livekit.TrackType_VIDEO { if !mime.IsMimeTypeStringVideo(mimeType) { mimeType = mime.MimeTypePrefixVideo + mimeType @@ -2703,6 +2703,13 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l // select an alternative MIME type that's generally supported mimeType = altCodec } + if videoLayerMode == livekit.VideoLayer_MODE_UNUSED { + if mime.IsMimeTypeStringSVC(mimeType) { + videoLayerMode = livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM + } else { + videoLayerMode = livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM + } + } } else if req.Type == livekit.TrackType_AUDIO && !mime.IsMimeTypeStringAudio(mimeType) { mimeType = mime.MimeTypePrefixAudio + mimeType } @@ -2713,14 +2720,15 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l seenCodecs[mimeType] = struct{}{} ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mimeType, - Cid: codec.Cid, + MimeType: mimeType, + Cid: codec.Cid, + VideoLayerMode: videoLayerMode, }) } // set up layers with codec specific layers, // fall back to common layers if codec specific layer is not available - for _, codec := range ti.Codecs { + for idx, codec := range ti.Codecs { found := false for _, simulcastCodec := range req.SimulcastCodecs { if mime.GetMimeTypeCodec(codec.MimeType) != mime.NormalizeMimeTypeCodec(simulcastCodec.Codec) { @@ -2740,6 +2748,11 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l // could happen if an alternate codec is selected and that is not in the simulcast codecs list codec.Layers = cloneLayers(req.Layers) } + + // populate simulcast flag for compatibility, true if primary codec is not SVC and has multiple layers + if idx == 0 && codec.VideoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && len(codec.Layers) > 1 { + ti.Simulcast = true + } } } @@ -2925,15 +2938,17 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver ti.Version = p.params.VersionGenerator.Next().ToProto() } + mimeType := mime.NormalizeMimeType(ti.MimeType) for _, layer := range ti.Layers { - layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(layer.Quality, ti) - layer.Rid = buffer.VideoQualityToRid(layer.Quality, ti, sdpRids) + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, ti) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, ti, sdpRids) } for _, codec := range ti.Codecs { + mimeType := mime.NormalizeMimeType(codec.MimeType) for _, layer := range codec.Layers { - layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(layer.Quality, ti) - layer.Rid = buffer.VideoQualityToRid(layer.Quality, ti, sdpRids) + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, ti) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, ti, sdpRids) } } @@ -3039,8 +3054,6 @@ func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *M } } } - mt.SetSimulcast(ti.Simulcast) - mt.SetMuted(ti.Muted) return mt } diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 35f0b4e85..36e2ed275 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -253,13 +253,14 @@ func (t *SubscribedTrack) applySettings() { if dt.Kind() == webrtc.RTPCodecTypeVideo { mt := t.MediaTrack() quality := t.settings.Quality + mimeType := dt.Mime() if t.settings.Width > 0 { - quality = mt.GetQualityForDimension(t.settings.Width, t.settings.Height) + quality = mt.GetQualityForDimension(mimeType, t.settings.Width, t.settings.Height) } - spatial = buffer.GetSpatialLayerForVideoQuality(quality, mt.ToProto()) + spatial = buffer.GetSpatialLayerForVideoQuality(mimeType, quality, mt.ToProto()) if t.settings.Fps > 0 { - temporal = mt.GetTemporalLayerForSpatialFps(spatial, t.settings.Fps, dt.Mime()) + temporal = mt.GetTemporalLayerForSpatialFps(mimeType, spatial, t.settings.Fps) } } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 2e299bcf0..95d4afd65 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -575,8 +575,6 @@ type MediaTrack interface { IsMuted() bool SetMuted(muted bool) - IsSimulcast() bool - GetAudioLevel() (level float64, active bool) Close(isExpectedToResume bool) @@ -595,10 +593,10 @@ type MediaTrack interface { OnTrackSubscribed() // returns quality information that's appropriate for width & height - GetQualityForDimension(width, height uint32) livekit.VideoQuality + GetQualityForDimension(mimeType mime.MimeType, width, height uint32) livekit.VideoQuality // returns temporal layer that's appropriate for fps - GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime mime.MimeType) int32 + GetTemporalLayerForSpatialFps(mimeType mime.MimeType, spatial int32, fps uint32) int32 Receivers() []sfu.TrackReceiver ClearAllReceivers(isExpectedToResume bool) diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index 39d18da94..cca91feed 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -88,11 +88,12 @@ type FakeLocalMediaTrack struct { getNumSubscribersReturnsOnCall map[int]struct { result1 int } - GetQualityForDimensionStub func(uint32, uint32) livekit.VideoQuality + GetQualityForDimensionStub func(mime.MimeType, uint32, uint32) livekit.VideoQuality getQualityForDimensionMutex sync.RWMutex getQualityForDimensionArgsForCall []struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 + arg3 uint32 } getQualityForDimensionReturns struct { result1 livekit.VideoQuality @@ -100,12 +101,12 @@ type FakeLocalMediaTrack struct { getQualityForDimensionReturnsOnCall map[int]struct { result1 livekit.VideoQuality } - GetTemporalLayerForSpatialFpsStub func(int32, uint32, mime.MimeType) int32 + GetTemporalLayerForSpatialFpsStub func(mime.MimeType, int32, uint32) int32 getTemporalLayerForSpatialFpsMutex sync.RWMutex getTemporalLayerForSpatialFpsArgsForCall []struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 } getTemporalLayerForSpatialFpsReturns struct { result1 int32 @@ -174,16 +175,6 @@ type FakeLocalMediaTrack struct { isOpenReturnsOnCall map[int]struct { result1 bool } - IsSimulcastStub func() bool - isSimulcastMutex sync.RWMutex - isSimulcastArgsForCall []struct { - } - isSimulcastReturns struct { - result1 bool - } - isSimulcastReturnsOnCall map[int]struct { - result1 bool - } IsSubscriberStub func(livekit.ParticipantID) bool isSubscriberMutex sync.RWMutex isSubscriberArgsForCall []struct { @@ -773,19 +764,20 @@ func (fake *FakeLocalMediaTrack) GetNumSubscribersReturnsOnCall(i int, result1 i }{result1} } -func (fake *FakeLocalMediaTrack) GetQualityForDimension(arg1 uint32, arg2 uint32) livekit.VideoQuality { +func (fake *FakeLocalMediaTrack) GetQualityForDimension(arg1 mime.MimeType, arg2 uint32, arg3 uint32) livekit.VideoQuality { fake.getQualityForDimensionMutex.Lock() ret, specificReturn := fake.getQualityForDimensionReturnsOnCall[len(fake.getQualityForDimensionArgsForCall)] fake.getQualityForDimensionArgsForCall = append(fake.getQualityForDimensionArgsForCall, struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 - }{arg1, arg2}) + arg3 uint32 + }{arg1, arg2, arg3}) stub := fake.GetQualityForDimensionStub fakeReturns := fake.getQualityForDimensionReturns - fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2}) + fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2, arg3}) fake.getQualityForDimensionMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -799,17 +791,17 @@ func (fake *FakeLocalMediaTrack) GetQualityForDimensionCallCount() int { return len(fake.getQualityForDimensionArgsForCall) } -func (fake *FakeLocalMediaTrack) GetQualityForDimensionCalls(stub func(uint32, uint32) livekit.VideoQuality) { +func (fake *FakeLocalMediaTrack) GetQualityForDimensionCalls(stub func(mime.MimeType, uint32, uint32) livekit.VideoQuality) { fake.getQualityForDimensionMutex.Lock() defer fake.getQualityForDimensionMutex.Unlock() fake.GetQualityForDimensionStub = stub } -func (fake *FakeLocalMediaTrack) GetQualityForDimensionArgsForCall(i int) (uint32, uint32) { +func (fake *FakeLocalMediaTrack) GetQualityForDimensionArgsForCall(i int) (mime.MimeType, uint32, uint32) { fake.getQualityForDimensionMutex.RLock() defer fake.getQualityForDimensionMutex.RUnlock() argsForCall := fake.getQualityForDimensionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalMediaTrack) GetQualityForDimensionReturns(result1 livekit.VideoQuality) { @@ -835,13 +827,13 @@ func (fake *FakeLocalMediaTrack) GetQualityForDimensionReturnsOnCall(i int, resu }{result1} } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFps(arg1 int32, arg2 uint32, arg3 mime.MimeType) int32 { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFps(arg1 mime.MimeType, arg2 int32, arg3 uint32) int32 { fake.getTemporalLayerForSpatialFpsMutex.Lock() ret, specificReturn := fake.getTemporalLayerForSpatialFpsReturnsOnCall[len(fake.getTemporalLayerForSpatialFpsArgsForCall)] fake.getTemporalLayerForSpatialFpsArgsForCall = append(fake.getTemporalLayerForSpatialFpsArgsForCall, struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 }{arg1, arg2, arg3}) stub := fake.GetTemporalLayerForSpatialFpsStub fakeReturns := fake.getTemporalLayerForSpatialFpsReturns @@ -862,13 +854,13 @@ func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCallCount() int { return len(fake.getTemporalLayerForSpatialFpsArgsForCall) } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(int32, uint32, mime.MimeType) int32) { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(mime.MimeType, int32, uint32) int32) { fake.getTemporalLayerForSpatialFpsMutex.Lock() defer fake.getTemporalLayerForSpatialFpsMutex.Unlock() fake.GetTemporalLayerForSpatialFpsStub = stub } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (int32, uint32, mime.MimeType) { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (mime.MimeType, int32, uint32) { fake.getTemporalLayerForSpatialFpsMutex.RLock() defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() argsForCall := fake.getTemporalLayerForSpatialFpsArgsForCall[i] @@ -1224,59 +1216,6 @@ func (fake *FakeLocalMediaTrack) IsOpenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeLocalMediaTrack) IsSimulcast() bool { - fake.isSimulcastMutex.Lock() - ret, specificReturn := fake.isSimulcastReturnsOnCall[len(fake.isSimulcastArgsForCall)] - fake.isSimulcastArgsForCall = append(fake.isSimulcastArgsForCall, struct { - }{}) - stub := fake.IsSimulcastStub - fakeReturns := fake.isSimulcastReturns - fake.recordInvocation("IsSimulcast", []interface{}{}) - fake.isSimulcastMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalMediaTrack) IsSimulcastCallCount() int { - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - return len(fake.isSimulcastArgsForCall) -} - -func (fake *FakeLocalMediaTrack) IsSimulcastCalls(stub func() bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = stub -} - -func (fake *FakeLocalMediaTrack) IsSimulcastReturns(result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - fake.isSimulcastReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalMediaTrack) IsSimulcastReturnsOnCall(i int, result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - if fake.isSimulcastReturnsOnCall == nil { - fake.isSimulcastReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isSimulcastReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalMediaTrack) IsSubscriber(arg1 livekit.ParticipantID) bool { fake.isSubscriberMutex.Lock() ret, specificReturn := fake.isSubscriberReturnsOnCall[len(fake.isSubscriberArgsForCall)] @@ -2336,8 +2275,6 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() defer fake.isOpenMutex.RUnlock() - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() fake.isSubscriberMutex.RLock() defer fake.isSubscriberMutex.RUnlock() fake.kindMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index 58415b3f2..c98b13b9b 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -72,11 +72,12 @@ type FakeMediaTrack struct { getNumSubscribersReturnsOnCall map[int]struct { result1 int } - GetQualityForDimensionStub func(uint32, uint32) livekit.VideoQuality + GetQualityForDimensionStub func(mime.MimeType, uint32, uint32) livekit.VideoQuality getQualityForDimensionMutex sync.RWMutex getQualityForDimensionArgsForCall []struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 + arg3 uint32 } getQualityForDimensionReturns struct { result1 livekit.VideoQuality @@ -84,12 +85,12 @@ type FakeMediaTrack struct { getQualityForDimensionReturnsOnCall map[int]struct { result1 livekit.VideoQuality } - GetTemporalLayerForSpatialFpsStub func(int32, uint32, mime.MimeType) int32 + GetTemporalLayerForSpatialFpsStub func(mime.MimeType, int32, uint32) int32 getTemporalLayerForSpatialFpsMutex sync.RWMutex getTemporalLayerForSpatialFpsArgsForCall []struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 } getTemporalLayerForSpatialFpsReturns struct { result1 int32 @@ -137,16 +138,6 @@ type FakeMediaTrack struct { isOpenReturnsOnCall map[int]struct { result1 bool } - IsSimulcastStub func() bool - isSimulcastMutex sync.RWMutex - isSimulcastArgsForCall []struct { - } - isSimulcastReturns struct { - result1 bool - } - isSimulcastReturnsOnCall map[int]struct { - result1 bool - } IsSubscriberStub func(livekit.ParticipantID) bool isSubscriberMutex sync.RWMutex isSubscriberArgsForCall []struct { @@ -625,19 +616,20 @@ func (fake *FakeMediaTrack) GetNumSubscribersReturnsOnCall(i int, result1 int) { }{result1} } -func (fake *FakeMediaTrack) GetQualityForDimension(arg1 uint32, arg2 uint32) livekit.VideoQuality { +func (fake *FakeMediaTrack) GetQualityForDimension(arg1 mime.MimeType, arg2 uint32, arg3 uint32) livekit.VideoQuality { fake.getQualityForDimensionMutex.Lock() ret, specificReturn := fake.getQualityForDimensionReturnsOnCall[len(fake.getQualityForDimensionArgsForCall)] fake.getQualityForDimensionArgsForCall = append(fake.getQualityForDimensionArgsForCall, struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 - }{arg1, arg2}) + arg3 uint32 + }{arg1, arg2, arg3}) stub := fake.GetQualityForDimensionStub fakeReturns := fake.getQualityForDimensionReturns - fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2}) + fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2, arg3}) fake.getQualityForDimensionMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -651,17 +643,17 @@ func (fake *FakeMediaTrack) GetQualityForDimensionCallCount() int { return len(fake.getQualityForDimensionArgsForCall) } -func (fake *FakeMediaTrack) GetQualityForDimensionCalls(stub func(uint32, uint32) livekit.VideoQuality) { +func (fake *FakeMediaTrack) GetQualityForDimensionCalls(stub func(mime.MimeType, uint32, uint32) livekit.VideoQuality) { fake.getQualityForDimensionMutex.Lock() defer fake.getQualityForDimensionMutex.Unlock() fake.GetQualityForDimensionStub = stub } -func (fake *FakeMediaTrack) GetQualityForDimensionArgsForCall(i int) (uint32, uint32) { +func (fake *FakeMediaTrack) GetQualityForDimensionArgsForCall(i int) (mime.MimeType, uint32, uint32) { fake.getQualityForDimensionMutex.RLock() defer fake.getQualityForDimensionMutex.RUnlock() argsForCall := fake.getQualityForDimensionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeMediaTrack) GetQualityForDimensionReturns(result1 livekit.VideoQuality) { @@ -687,13 +679,13 @@ func (fake *FakeMediaTrack) GetQualityForDimensionReturnsOnCall(i int, result1 l }{result1} } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFps(arg1 int32, arg2 uint32, arg3 mime.MimeType) int32 { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFps(arg1 mime.MimeType, arg2 int32, arg3 uint32) int32 { fake.getTemporalLayerForSpatialFpsMutex.Lock() ret, specificReturn := fake.getTemporalLayerForSpatialFpsReturnsOnCall[len(fake.getTemporalLayerForSpatialFpsArgsForCall)] fake.getTemporalLayerForSpatialFpsArgsForCall = append(fake.getTemporalLayerForSpatialFpsArgsForCall, struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 }{arg1, arg2, arg3}) stub := fake.GetTemporalLayerForSpatialFpsStub fakeReturns := fake.getTemporalLayerForSpatialFpsReturns @@ -714,13 +706,13 @@ func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCallCount() int { return len(fake.getTemporalLayerForSpatialFpsArgsForCall) } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(int32, uint32, mime.MimeType) int32) { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(mime.MimeType, int32, uint32) int32) { fake.getTemporalLayerForSpatialFpsMutex.Lock() defer fake.getTemporalLayerForSpatialFpsMutex.Unlock() fake.GetTemporalLayerForSpatialFpsStub = stub } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (int32, uint32, mime.MimeType) { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (mime.MimeType, int32, uint32) { fake.getTemporalLayerForSpatialFpsMutex.RLock() defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() argsForCall := fake.getTemporalLayerForSpatialFpsArgsForCall[i] @@ -962,59 +954,6 @@ func (fake *FakeMediaTrack) IsOpenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeMediaTrack) IsSimulcast() bool { - fake.isSimulcastMutex.Lock() - ret, specificReturn := fake.isSimulcastReturnsOnCall[len(fake.isSimulcastArgsForCall)] - fake.isSimulcastArgsForCall = append(fake.isSimulcastArgsForCall, struct { - }{}) - stub := fake.IsSimulcastStub - fakeReturns := fake.isSimulcastReturns - fake.recordInvocation("IsSimulcast", []interface{}{}) - fake.isSimulcastMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeMediaTrack) IsSimulcastCallCount() int { - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - return len(fake.isSimulcastArgsForCall) -} - -func (fake *FakeMediaTrack) IsSimulcastCalls(stub func() bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = stub -} - -func (fake *FakeMediaTrack) IsSimulcastReturns(result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - fake.isSimulcastReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeMediaTrack) IsSimulcastReturnsOnCall(i int, result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - if fake.isSimulcastReturnsOnCall == nil { - fake.isSimulcastReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isSimulcastReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeMediaTrack) IsSubscriber(arg1 livekit.ParticipantID) bool { fake.isSubscriberMutex.Lock() ret, specificReturn := fake.isSubscriberReturnsOnCall[len(fake.isSubscriberArgsForCall)] @@ -1886,8 +1825,6 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() defer fake.isOpenMutex.RUnlock() - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() fake.isSubscriberMutex.RLock() defer fake.isSubscriberMutex.RUnlock() fake.kindMutex.RLock() diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index a4d41a715..54e01be3a 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -27,6 +27,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/mime" ) @@ -273,6 +274,13 @@ func (d *DummyReceiver) Mime() mime.MimeType { return mime.NormalizeMimeType(d.codec.MimeType) } +func (d *DummyReceiver) VideoLayerMode() livekit.VideoLayer_Mode { + if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + return r.VideoLayerMode() + } + return buffer.GetVideoLayerModeForMimeType(d.Mime(), d.TrackInfo()) +} + func (d *DummyReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.HeaderExtensions() diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 2da9d0aad..e4b923744 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -17,6 +17,7 @@ package buffer import ( "slices" + "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -39,14 +40,18 @@ var ( DefaultVideoLayersRid = videoLayersRidQHF ) -// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers -func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool { - if trackInfo == nil || len(trackInfo.Layers) == 0 { +func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool { + if trackInfo == nil { + return nil + } + + layers := GetVideoLayersForMimeType(mimeType, trackInfo) + if len(layers) == 0 { return nil } var layerPresence [livekit.VideoQuality_HIGH + 1]bool - for _, layer := range trackInfo.Layers { + for _, layer := range layers { // WARNING: comparing protobuf enum if layer.Quality <= livekit.VideoQuality_HIGH { layerPresence[layer.Quality] = true @@ -58,8 +63,8 @@ func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQua return &layerPresence } -func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 { - lp := LayerPresenceFromTrackInfo(trackInfo) +func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch rid { case quarterResolutionQ: @@ -132,8 +137,8 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo, ridSpace VideoL } } -func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { - lp := LayerPresenceFromTrackInfo(trackInfo) +func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch layer { case 0: @@ -202,12 +207,12 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo, ridSpace Video } } -func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { - return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo, ridSpace) +func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { + return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace) } -func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality { - lp := LayerPresenceFromTrackInfo(trackInfo) +func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch layer { case 0: @@ -273,8 +278,8 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek return livekit.VideoQuality_OFF } -func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 { - lp := LayerPresenceFromTrackInfo(trackInfo) +func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch quality { case livekit.VideoQuality_LOW: @@ -339,28 +344,56 @@ func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit return InvalidLayerSpatial } -// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers -func GetSpatialLayerForRid(rid string, ti *livekit.TrackInfo) int32 { +func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode { + if ti != nil { + for _, codec := range ti.Codecs { + if mime.NormalizeMimeType(codec.MimeType) == mimeType { + return codec.VideoLayerMode + } + } + } + + return livekit.VideoLayer_MODE_UNUSED +} + +func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer { + var layers []*livekit.VideoLayer + if ti != nil { + for _, codec := range ti.Codecs { + if mime.NormalizeMimeType(codec.MimeType) == mimeType { + layers = codec.Layers + break + } + } + if len(layers) == 0 { + layers = ti.Layers + } + } + return layers +} + +func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 { + if ti == nil { + return InvalidLayerSpatial + } + if rid == "" { // single layer without RID return 0 } - if ti == nil { - return InvalidLayerSpatial - } - - for _, layer := range ti.Layers { + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { if layer.Rid == rid { return layer.SpatialLayer } } - if len(ti.Layers) != 0 { + if len(layers) != 0 { // RID present in codec, but may not be specified via signalling // (happens with older browsers setting a rid for SVC codecs) hasRid := false - for _, layer := range ti.Layers { + for _, layer := range layers { if layer.Rid != "" { hasRid = true break @@ -372,21 +405,22 @@ func GetSpatialLayerForRid(rid string, ti *livekit.TrackInfo) int32 { } // SIMULCAST-CODEC-TODO - ideally should return invalid, but there are - // VP9 publish using rid = f, if there are only two layers + // VP9 publishers using rid = f, if there are only two layers // in TrackInfo, that will be q;h and f will become invalid. // // Actually, there should be no rids for VP9 in SDP and hence // the above check should take effect. However, as simulcast - // codec does not update SDP, the default rids are use when - // vp9 (primary codec) is published and that bypasses the above - // check. + // codec/back up codec does not update rids from SDP, + // the default rids are used when vp9 (primary codec) + // is published. Due to that the above check gets bypassed. // // The full proper sequence would be // 1. For primary codec using SVC, there will be no rids. // The above check should take effect and it should // return 0 even if some publisher uses a rid like `f`. - // 2. When secondary codec is published, update rids in the - // codec section in `TrackInfo`. This is a bit tricky + // 2. When secondary codec is published, rids for the codec + // corresponding to the back up codec mime type should + // be updated in `TrackInfo`. This is a bit tricky // for a couple of cases // a. Browsers like Firefox use a different CID everytime. // So, it cannot be matched between `AddTrack` and SDP. @@ -397,8 +431,8 @@ func GetSpatialLayerForRid(rid string, ti *livekit.TrackInfo) int32 { // b. The back up codec publish SDP will have the full // codec list. It should be okay to assume that the // codec that will be published is the back up codec, - // but just something to be aware of.. - // 3. Use this function with proper mime so that proper + // but just something to be aware of. + // 3. Use of this function with proper mime so that proper // codec section can be looked up in `TrackInfo`. // return InvalidLayerSpatial logger.Infow( @@ -410,32 +444,34 @@ func GetSpatialLayerForRid(rid string, ti *livekit.TrackInfo) int32 { return 0 } -func GetSpatialLayerForVideoQuality(quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 { +func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 { if ti == nil || quality == livekit.VideoQuality_OFF { return InvalidLayerSpatial } - for _, layer := range ti.Layers { + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { if layer.Quality == quality { return layer.SpatialLayer } } - if len(ti.Layers) == 0 { + if len(layers) == 0 { // single layer return 0 } // requested quality is higher than available layers, return the highest available layer - return ti.Layers[len(ti.Layers)-1].SpatialLayer + return layers[len(layers)-1].SpatialLayer } -func GetVideoQualityForSpatialLayer(spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality { +func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality { if spatialLayer == InvalidLayerSpatial || ti == nil { return livekit.VideoQuality_OFF } - for _, layer := range ti.Layers { + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { if layer.SpatialLayer == spatialLayer { return layer.Quality } diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index a47f0c0ee..357234e25 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/protocol/livekit" ) @@ -30,11 +31,13 @@ func TestRidConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType ridToLayer map[string]RidAndLayer }{ { "no track info", nil, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -45,6 +48,7 @@ func TestRidConversion(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -55,10 +59,16 @@ func TestRidConversion(t *testing.T) { { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -69,10 +79,16 @@ func TestRidConversion(t *testing.T) { { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -83,10 +99,16 @@ func TestRidConversion(t *testing.T) { { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -97,11 +119,17 @@ func TestRidConversion(t *testing.T) { { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -112,11 +140,17 @@ func TestRidConversion(t *testing.T) { { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -127,11 +161,17 @@ func TestRidConversion(t *testing.T) { { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -142,12 +182,18 @@ func TestRidConversion(t *testing.T) { { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ "": {rid: quarterResolutionQ, layer: 0}, quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, @@ -160,10 +206,10 @@ func TestRidConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testRid, expectedResult := range test.ridToLayer { - actualLayer := RidToSpatialLayer(testRid, test.trackInfo, DefaultVideoLayersRid) + actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedResult.layer, actualLayer) - actualRid := SpatialLayerToRid(actualLayer, test.trackInfo, DefaultVideoLayersRid) + actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedResult.rid, actualRid) } }) @@ -178,11 +224,13 @@ func TestQualityConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType qualityToLayer map[livekit.VideoQuality]QualityAndLayer }{ { "no track info", nil, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -192,6 +240,7 @@ func TestQualityConversion(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -201,10 +250,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0}, @@ -214,10 +269,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, @@ -227,10 +288,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0}, @@ -240,11 +307,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -254,11 +327,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1}, @@ -268,11 +347,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, @@ -282,12 +367,18 @@ func TestQualityConversion(t *testing.T) { { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -299,10 +390,10 @@ func TestQualityConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testQuality, expectedResult := range test.qualityToLayer { - actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo) + actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo) require.Equal(t, expectedResult.layer, actualLayer) - actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo) + actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo) require.Equal(t, expectedResult.quality, actualQuality) } }) @@ -313,11 +404,13 @@ func TestVideoQualityToRidConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeTye mime.MimeType qualityToRid map[livekit.VideoQuality]string }{ { "no track info", nil, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: halfResolutionH, @@ -327,6 +420,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: halfResolutionH, @@ -336,10 +430,16 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: quarterResolutionQ, @@ -349,10 +449,16 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: quarterResolutionQ, @@ -362,10 +468,16 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: quarterResolutionQ, @@ -375,11 +487,17 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: halfResolutionH, @@ -389,11 +507,17 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: halfResolutionH, @@ -403,11 +527,17 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: quarterResolutionQ, @@ -417,12 +547,18 @@ func TestVideoQualityToRidConversion(t *testing.T) { { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ livekit.VideoQuality_LOW: quarterResolutionQ, livekit.VideoQuality_MEDIUM: halfResolutionH, @@ -434,7 +570,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testQuality, expectedRid := range test.qualityToRid { - actualRid := VideoQualityToRid(testQuality, test.trackInfo, DefaultVideoLayersRid) + actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedRid, actualRid) } }) @@ -445,11 +581,13 @@ func TestGetSpatialLayerForRid(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType ridToSpatialLayer map[string]int32 }{ { "no track info", nil, + mime.MimeTypeVP8, map[string]int32{ quarterResolutionQ: InvalidLayerSpatial, halfResolutionH: InvalidLayerSpatial, @@ -459,6 +597,7 @@ func TestGetSpatialLayerForRid(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[string]int32{ // SIMULCAST-CODEC-TODO // quarterResolutionQ: InvalidLayerSpatial, @@ -472,6 +611,7 @@ func TestGetSpatialLayerForRid(t *testing.T) { { "no rid", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[string]int32{ "": 0, }, @@ -479,10 +619,16 @@ func TestGetSpatialLayerForRid(t *testing.T) { { "single layer", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]int32{ quarterResolutionQ: 0, halfResolutionH: 0, @@ -492,11 +638,17 @@ func TestGetSpatialLayerForRid(t *testing.T) { { "layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]int32{ quarterResolutionQ: 0, halfResolutionH: 1, @@ -508,11 +660,17 @@ func TestGetSpatialLayerForRid(t *testing.T) { { "layers - no rid", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]int32{ quarterResolutionQ: 0, halfResolutionH: 0, @@ -524,7 +682,7 @@ func TestGetSpatialLayerForRid(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testRid, expectedSpatialLayer := range test.ridToSpatialLayer { - actualSpatialLayer := GetSpatialLayerForRid(testRid, test.trackInfo) + actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo) require.Equal(t, expectedSpatialLayer, actualSpatialLayer) } }) @@ -535,11 +693,13 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType videoQualityToSpatialLayer map[livekit.VideoQuality]int32 }{ { "no track info", nil, + mime.MimeTypeVP8, map[livekit.VideoQuality]int32{ livekit.VideoQuality_LOW: InvalidLayerSpatial, livekit.VideoQuality_MEDIUM: InvalidLayerSpatial, @@ -550,6 +710,7 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[livekit.VideoQuality]int32{ livekit.VideoQuality_LOW: 0, livekit.VideoQuality_MEDIUM: 0, @@ -560,11 +721,17 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { { "not all layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]int32{ livekit.VideoQuality_LOW: 0, livekit.VideoQuality_MEDIUM: 1, @@ -575,12 +742,18 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { { "all layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, - {Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + {Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]int32{ livekit.VideoQuality_LOW: 0, livekit.VideoQuality_MEDIUM: 1, @@ -593,7 +766,7 @@ func TestGetSpatialLayerForVideoQuality(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer { - actualSpatialLayer := GetSpatialLayerForVideoQuality(testVideoQuality, test.trackInfo) + actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo) require.Equal(t, expectedSpatialLayer, actualSpatialLayer) } }) @@ -604,11 +777,13 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType spatialLayerToVideoQuality map[int32]livekit.VideoQuality }{ { "no track info", nil, + mime.MimeTypeVP8, map[int32]livekit.VideoQuality{ InvalidLayerSpatial: livekit.VideoQuality_OFF, 0: livekit.VideoQuality_OFF, @@ -619,6 +794,7 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[int32]livekit.VideoQuality{ InvalidLayerSpatial: livekit.VideoQuality_OFF, 0: livekit.VideoQuality_OFF, @@ -629,11 +805,17 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { { "layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, - {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, }, }, + mime.MimeTypeVP8, map[int32]livekit.VideoQuality{ InvalidLayerSpatial: livekit.VideoQuality_OFF, 0: livekit.VideoQuality_LOW, @@ -646,7 +828,7 @@ func TestGetVideoQualityorSpatialLayer(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality { - actualVideoQuality := GetVideoQualityForSpatialLayer(testSpatialLayer, test.trackInfo) + actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo) require.Equal(t, expectedVideoQuality, actualVideoQuality) } }) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index c19614073..1bf366038 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -64,7 +64,6 @@ type TrackSender interface { SubscriberID() livekit.ParticipantID HandleRTCPSenderReportData( payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) error @@ -223,6 +222,8 @@ func (bs bindState) String() string { // ------------------------------------------------------------------- +var _ TrackSender = (*DownTrack)(nil) + type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) type DowntrackParams struct { @@ -580,7 +581,8 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.setBindStateLocked(bindStateBound) d.bindLock.Unlock() - d.forwarder.DetermineCodec(codec.RTPCodecCapability, d.Receiver().HeaderExtensions()) + receiver := d.Receiver() + d.forwarder.DetermineCodec(codec.RTPCodecCapability, receiver.HeaderExtensions(), receiver.VideoLayerMode()) d.connectionStats.Start(d.Mime(), isFECEnabled) d.params.Logger.Debugw("downtrack bound") } @@ -689,7 +691,8 @@ func (d *DownTrack) handleUpstreamCodecChange(mimeType string) { ) d.forwarder.Restart() - d.forwarder.DetermineCodec(codec.RTPCodecCapability, d.Receiver().HeaderExtensions()) + receiver := d.Receiver() + d.forwarder.DetermineCodec(codec.RTPCodecCapability, receiver.HeaderExtensions(), receiver.VideoLayerMode()) d.connectionStats.UpdateCodec(d.Mime(), isFECEnabled) } @@ -1002,7 +1005,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { // clock domain of publisher. SFU is normalising sender reports of publisher // to SFU clock before sending to subscribers. So, capture time should be // normalized to the same clock. Clear out any offset. - _, _, refSenderReport := d.forwarder.GetSenderReportParams() + _, _, _, refSenderReport := d.forwarder.GetSenderReportParams() if refSenderReport != nil { actExtCopy := *extPkt.AbsCaptureTimeExt if err = actExtCopy.Rewrite( @@ -1668,7 +1671,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + _, _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() return d.rtpStats.GetRtcpSenderReport(d.ssrc, refSenderReport, tsOffset, !d.params.DisableSenderReportPassThrough) // not sending RTCP Sender Report for RTX @@ -2175,6 +2178,8 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) + receiver := d.Receiver() + nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) @@ -2186,7 +2191,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { nackAcks++ pktBuff := *src - n, err := d.Receiver().ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) + n, err := receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) if err != nil { if err == io.EOF { break @@ -2276,6 +2281,8 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) + receiver := d.Receiver() + endExtHighestSequenceNumber := d.rtpStats.ExtHighestSequenceNumber() startExtHighestSequenceNumber := endExtHighestSequenceNumber - 5 for esn := startExtHighestSequenceNumber; esn <= endExtHighestSequenceNumber; esn++ { @@ -2285,7 +2292,7 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { } pktBuff := *src - n, err := d.Receiver().ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) + n, err := receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) if err != nil { if err == io.EOF { break @@ -2507,14 +2514,13 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { func (d *DownTrack) HandleRTCPSenderReportData( _payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) error { - d.forwarder.SetRefSenderReport(isSVC, layer, publisherSRData) + d.forwarder.SetRefSenderReport(layer, publisherSRData) - currentLayer, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() - if layer == currentLayer || (layer == 0 && isSVC) { + currentLayer, isSingleStream, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + if layer == currentLayer || (layer == 0 && isSingleStream) { d.handleRTCPSenderReportData(refSenderReport, tsOffset) } return nil diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 7318bcbaf..5d68ebc6a 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -230,7 +230,7 @@ type Forwarder struct { referenceLayerSpatial int32 dummyStartTSOffset uint64 refInfos [buffer.DefaultMaxLayerSpatial + 1]refInfo - refIsSVC bool + refVideoLayerMode livekit.VideoLayer_Mode isDDAvailable bool provisional *VideoAllocationProvisional @@ -298,7 +298,7 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool { return true } -func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) { +func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter, videoLayerMode livekit.VideoLayer_Mode) { f.lock.Lock() defer f.lock.Unlock() @@ -309,6 +309,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ } f.mime = toMimeType f.clockRate = codec.ClockRate + f.refVideoLayerMode = videoLayerMode ddAvailable := func(exts []webrtc.RTPHeaderExtensionParameter) bool { for _, ext := range exts { @@ -335,46 +336,62 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ case mime.MimeTypeH264, mime.MimeTypeH265: if f.vls != nil { - f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil { + f.vls = vls + } else { + f.logger.Errorw("failed to create simulcast on codec change", nil) + } } else { f.vls = videolayerselector.NewSimulcast(f.logger) } case mime.MimeTypeVP9: - // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - f.isDDAvailable = ddAvailable(extensions) - if f.isDDAvailable { + if videoLayerMode == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM { if f.vls != nil { - f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) } else { f.vls = videolayerselector.NewDependencyDescriptor(f.logger) } } else { - if f.vls != nil { - f.vls = videolayerselector.NewVP9FromOther(f.vls) + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } } else { - f.vls = videolayerselector.NewVP9(f.logger) + if f.vls != nil { + f.vls = videolayerselector.NewVP9FromOther(f.vls) + } else { + f.vls = videolayerselector.NewVP9(f.logger) + } } } - // SVC-TODO: Support for VP9 simulcast. When DD is not available, have to pick selector based on VP9 SVC or Simulcast case mime.MimeTypeAV1: - // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - f.isDDAvailable = ddAvailable(extensions) - if f.isDDAvailable { - if f.vls != nil { - f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) - } else { - f.vls = videolayerselector.NewDependencyDescriptor(f.logger) - } - } else { + if videoLayerMode == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM { if f.vls != nil { f.vls = videolayerselector.NewSimulcastFromOther(f.vls) } else { f.vls = videolayerselector.NewSimulcast(f.logger) } + } else { + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } + } else { + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + } } - // SVC-TODO: Support for AV1 Simulcast } } @@ -613,18 +630,17 @@ func (f *Forwarder) getRefLayer() (int32, int32) { return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial } - if f.refIsSVC { + if f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { return 0, currentLayerSpatial } return currentLayerSpatial, currentLayerSpatial } -func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.RTCPSenderReportState) { +func (f *Forwarder) SetRefSenderReport(layer int32, srData *livekit.RTCPSenderReportState) { f.lock.Lock() defer f.lock.Unlock() - f.refIsSVC = isSVC if layer >= 0 && int(layer) < len(f.refInfos) { if layer == f.referenceLayerSpatial && f.refInfos[layer].senderReport == nil { f.logger.Debugw("received RTCP sender report for reference layer spatial", "layer", layer) @@ -666,7 +682,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit. } } -func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderReportState) { +func (f *Forwarder) GetSenderReportParams() (int32, bool, uint64, *livekit.RTCPSenderReportState) { f.lock.RLock() defer f.lock.RUnlock() @@ -674,10 +690,10 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderR if refLayer == buffer.InvalidLayerSpatial || f.refInfos[refLayer].senderReport == nil || !f.refInfos[refLayer].isTSOffsetValid { - return buffer.InvalidLayerSpatial, 0, nil + return buffer.InvalidLayerSpatial, false, 0, nil } - return currentLayerSpatial, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport + return currentLayerSpatial, f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -1584,7 +1600,7 @@ func (f *Forwarder) Restart() { f.refInfos[layer] = refInfo{} } f.lastSwitchExtIncomingTS = 0 - f.refIsSVC = false + f.refVideoLayerMode = livekit.VideoLayer_MODE_UNUSED } func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) { @@ -1645,7 +1661,7 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) } - if refLayer == targetLayer || f.refIsSVC { + if refLayer == targetLayer || f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { return ts, nil } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 722c76769..54bdbbd14 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -20,6 +20,7 @@ import ( "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -33,7 +34,7 @@ func disable(f *Forwarder) { func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { f := NewForwarder(kind, logger.GetLogger(), true, nil) - f.DetermineCodec(codec, nil) + f.DetermineCodec(codec, nil, livekit.VideoLayer_MODE_UNUSED) return f } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index f4cbd1a9b..4c1773610 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -102,6 +102,7 @@ type TrackReceiver interface { // and will not change if the codec changes during the session (publisher changes codec) Codec() webrtc.RTPCodecParameters Mime() mime.MimeType + VideoLayerMode() livekit.VideoLayer_Mode HeaderExtensions() []webrtc.RTPHeaderExtensionParameter IsClosed() bool @@ -146,7 +147,6 @@ type REDTransformer interface { ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int ForwardRTCPSenderReport( payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) @@ -155,6 +155,8 @@ type REDTransformer interface { Close() } +var _ TrackReceiver = (*WebRTCReceiver)(nil) + // WebRTCReceiver receives a media track type WebRTCReceiver struct { logger logger.Logger @@ -170,12 +172,12 @@ type WebRTCReceiver struct { codecState ReceiverCodecState codecStateLock sync.Mutex onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState) - isSVC bool isRED bool onCloseHandler func() closeOnce sync.Once closed atomic.Bool trackInfo atomic.Pointer[livekit.TrackInfo] + videoLayerMode livekit.VideoLayer_Mode onRTCP func([]rtcp.Packet) @@ -193,7 +195,7 @@ type WebRTCReceiver struct { connectionStats *connectionquality.ConnectionStats onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat) - onMaxLayerChange func(maxLayer int32) + onMaxLayerChange func(mimeType mime.MimeType, maxLayer int32) redTransformer atomic.Value // redTransformer interface @@ -248,16 +250,16 @@ func NewWebRTCReceiver( opts ...ReceiverOpts, ) *WebRTCReceiver { w := &WebRTCReceiver{ - logger: logger, - receiver: receiver, - trackID: livekit.TrackID(track.ID()), - streamID: track.StreamID(), - codec: track.Codec(), - codecState: ReceiverCodecStateNormal, - kind: track.Kind(), - onRTCP: onRTCP, - isSVC: mime.IsMimeTypeStringSVC(track.Codec().MimeType), - isRED: mime.IsMimeTypeStringRED(track.Codec().MimeType), + logger: logger, + receiver: receiver, + trackID: livekit.TrackID(track.ID()), + streamID: track.StreamID(), + codec: track.Codec(), + codecState: ReceiverCodecStateNormal, + kind: track.Kind(), + onRTCP: onRTCP, + isRED: mime.IsMimeTypeStringRED(track.Codec().MimeType), + videoLayerMode: buffer.GetVideoLayerModeForMimeType(mime.NormalizeMimeType(track.Codec().MimeType), trackInfo), } for _, opt := range opts { @@ -285,7 +287,7 @@ func NewWebRTCReceiver( mime.IsMimeTypeStringRED(w.codec.MimeType) || strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "useinbandfec=1"), ) - w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig) + w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.Mime(), w.codec.ClockRate, streamTrackerManagerConfig) w.streamTrackerManager.SetListener(w) return w @@ -304,13 +306,13 @@ func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit. w.onStatsUpdate = fn } -func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) { +func (w *WebRTCReceiver) OnMaxLayerChange(fn func(mimeType mime.MimeType, maxLayer int32)) { w.bufferMu.Lock() w.onMaxLayerChange = fn w.bufferMu.Unlock() } -func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) { +func (w *WebRTCReceiver) getOnMaxLayerChange() func(mimeType mime.MimeType, maxLayer int32) { w.bufferMu.RLock() defer w.bufferMu.RUnlock() @@ -368,6 +370,10 @@ func (w *WebRTCReceiver) Mime() mime.MimeType { return mime.NormalizeMimeType(w.codec.MimeType) } +func (w *WebRTCReceiver) VideoLayerMode() livekit.VideoLayer_Mode { + return w.videoLayerMode +} + func (w *WebRTCReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { return w.receiver.GetParameters().HeaderExtensions } @@ -382,8 +388,8 @@ func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) erro } layer := int32(0) - if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { - layer = buffer.GetSpatialLayerForRid(track.RID(), w.trackInfo.Load()) + if w.Kind() == webrtc.RTPCodecTypeVideo && w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + layer = buffer.GetSpatialLayerForRid(w.Mime(), track.RID(), w.trackInfo.Load()) } if layer < 0 { w.logger.Warnw( @@ -402,11 +408,11 @@ func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) erro buff.OnRtcpSenderReport(func() { srData := buff.GetSenderReportData() w.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData) + _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, layer, srData) }) if rt := w.redTransformer.Load(); rt != nil { - rt.(REDTransformer).ForwardRTCPSenderReport(w.codec.PayloadType, w.isSVC, layer, srData) + rt.(REDTransformer).ForwardRTCPSenderReport(w.codec.PayloadType, layer, srData) } }) @@ -498,7 +504,7 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { } expectedBitrate := int64(0) - for _, vl := range ti.Layers { + for _, vl := range buffer.GetVideoLayersForMimeType(w.Mime(), ti) { if vl.SpatialLayer <= layer { expectedBitrate += int64(vl.Bitrate) } @@ -557,7 +563,7 @@ func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int3 // StreamTrackerManagerListener.OnMaxAvailableLayerChanged func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) { if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil { - onMaxLayerChange(maxAvailableLayer) + onMaxLayerChange(w.Mime(), maxAvailableLayer) } } @@ -619,7 +625,7 @@ func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer { func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer { // for svc codecs, use layer = 0 always. // spatial layers are in-built and handled by single buffer - if w.isSVC { + if w.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { layer = 0 } @@ -738,7 +744,7 @@ func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) { }) w.streamTrackerManager.RemoveTracker(layer) - if w.isSVC { + if w.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { w.streamTrackerManager.RemoveAllTrackers() } @@ -807,7 +813,7 @@ func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) { if spatialTrackers[spatialLayer] == nil { spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer) if spatialTrackers[spatialLayer] == nil { - if w.isSVC && pkt.DependencyDescriptor != nil { + if w.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && pkt.DependencyDescriptor != nil { w.streamTrackerManager.AddDependencyDescriptorTrackers() } spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer) @@ -842,13 +848,12 @@ func (w *WebRTCReceiver) closeTracks() { } func (w *WebRTCReceiver) DebugInfo() map[string]interface{} { - isSimulcast := !w.isSVC + var videoLayerMode livekit.VideoLayer_Mode if ti := w.trackInfo.Load(); ti != nil { - isSimulcast = isSimulcast && len(ti.Layers) > 1 + videoLayerMode = buffer.GetVideoLayerModeForMimeType(w.Mime(), ti) } info := map[string]interface{}{ - "SVC": w.isSVC, - "Simulcast": isSimulcast, + "VideoLayerMode": videoLayerMode.String(), } w.bufferMu.RLock() @@ -923,7 +928,7 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return nil } - if !w.isSVC { + if w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { return b.GetTemporalLayerFpsForSpatial(0) } diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index ffa9028cc..e18a57143 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -97,12 +97,11 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3 func (r *RedPrimaryReceiver) ForwardRTCPSenderReport( payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) { r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.HandleRTCPSenderReportData(payloadType, isSVC, layer, publisherSRData) + _ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData) }) } diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index d095f85dc..9151d259d 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -92,12 +92,11 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int func (r *RedReceiver) ForwardRTCPSenderReport( payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) { r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.HandleRTCPSenderReportData(payloadType, isSVC, layer, publisherSRData) + _ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData) }) } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 0258508ab..b10267392 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) @@ -111,10 +112,11 @@ var ( // --------------------------------------------------- type StreamTrackerManager struct { - logger logger.Logger - trackInfo atomic.Pointer[livekit.TrackInfo] - isSVC bool - clockRate uint32 + logger logger.Logger + trackInfo atomic.Pointer[livekit.TrackInfo] + mimeType mime.MimeType + videoLayerMode livekit.VideoLayer_Mode + clockRate uint32 trackerConfig StreamTrackerConfig @@ -137,13 +139,14 @@ type StreamTrackerManager struct { func NewStreamTrackerManager( logger logger.Logger, trackInfo *livekit.TrackInfo, - isSVC bool, + mimeType mime.MimeType, clockRate uint32, config StreamTrackerManagerConfig, ) *StreamTrackerManager { s := &StreamTrackerManager{ logger: logger, - isSVC: isSVC, + mimeType: mimeType, + videoLayerMode: buffer.GetVideoLayerModeForMimeType(mimeType, trackInfo), maxPublishedLayer: buffer.InvalidLayerSpatial, maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, @@ -492,7 +495,7 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { } // accumulate bitrates for SVC streams without dependency descriptor - if s.isSVC && s.ddTracker == nil { + if s.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && s.ddTracker == nil { for i := len(br) - 1; i >= 1; i-- { for j := len(br[i]) - 1; j >= 0; j-- { if br[i][j] != 0 { @@ -596,7 +599,7 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked() { s.maxExpectedLayer = buffer.InvalidLayerSpatial ti := s.trackInfo.Load() if ti != nil { - for _, layer := range ti.Layers { + for _, layer := range buffer.GetVideoLayersForMimeType(s.mimeType, ti) { if layer.SpatialLayer > s.maxExpectedLayer { s.maxExpectedLayer = layer.SpatialLayer }