diff --git a/go.mod b/go.mod index 4c5ed49d3..ed02dcdd3 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded - github.com/livekit/protocol v1.39.4-0.20250807185925-1d6123ccc90d + github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741 github.com/mackerelio/go-osstat v0.2.5 github.com/magefile/mage v1.15.0 @@ -131,7 +131,7 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/zap/exp v0.3.0 // indirect golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.42.0 // indirect + golang.org/x/net v0.43.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/tools v0.35.0 // indirect diff --git a/go.sum b/go.sum index 5ac785dcf..ec456cc08 100644 --- a/go.sum +++ b/go.sum @@ -167,10 +167,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded h1:ylZPdnlX1RW9Z15SD4mp87vT2D2shsk0hpLJwSPcq3g= github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded/go.mod h1:mSNtYzSf6iY9xM3UX42VEI+STHvMgHmrYzEHPcdhB8A= -github.com/livekit/protocol v1.39.4-0.20250807105828-ccbae8154e54 h1:Bh4IzI1UUqyxUnAJScR2rT/7V8mJqkzFSoEuMfm/T5k= -github.com/livekit/protocol v1.39.4-0.20250807105828-ccbae8154e54/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU= -github.com/livekit/protocol v1.39.4-0.20250807185925-1d6123ccc90d h1:3KiMLrVyB6LUYmLKj+KfAII9EV/wIOubHW8U/gk440U= -github.com/livekit/protocol v1.39.4-0.20250807185925-1d6123ccc90d/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU= +github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a h1:hqju+FP5ly5F1zg25DpP1jqgSdv9sIglDM03jgw1864= +github.com/livekit/protocol v1.39.4-0.20250808070328-82ec7d74738a/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU= github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741 h1:KKL1u94l6dF9u4cBwnnfozk27GH1txWy2SlvkfgmzoY= github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741/go.mod h1:AuDC5uOoEjQJEc69v4Li3t77Ocz0e0NdjQEuFfO+vfk= github.com/mackerelio/go-osstat v0.2.5 h1:+MqTbZUhoIt4m8qzkVoXUJg1EuifwlAJSk4Yl2GXh+o= @@ -260,8 +258,6 @@ github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= -github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= -github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= github.com/pion/turn/v4 v4.1.0 h1:+J56+aS8Bi6B4zij3ah6VvJpRuy8W8FtExR0OJPiTdM= github.com/pion/turn/v4 v4.1.0/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8= github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c= @@ -363,8 +359,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= @@ -375,8 +369,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -401,8 +393,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -446,8 +438,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -465,8 +455,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index e90a524b9..453516e45 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -208,10 +208,6 @@ func (t *MediaTrack) ToProto() *livekit.TrackInfo { return t.MediaTrackReceiver.TrackInfoClone() } -func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { - t.MediaTrackReceiver.UpdateCodecCid(codecs) -} - // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec // and if a receiver was added successfully func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRemote, mid string) (bool, bool) { diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index a63386af1..4ed7c08cc 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -711,13 +711,21 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mimeType mime.MimeType, rid string, ss t.updateTrackInfoOfReceivers() } -func (t *MediaTrackReceiver) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { +func (t *MediaTrackReceiver) UpdateCodecInfo(codecs []*livekit.SimulcastCodec) { t.lock.Lock() trackInfo := t.TrackInfoClone() for _, c := range codecs { for _, origin := range trackInfo.Codecs { if mime.GetMimeTypeCodec(origin.MimeType) == mime.NormalizeMimeTypeCodec(c.Codec) { origin.Cid = c.Cid + + if len(c.Layers) != 0 { + clonedLayers := make([]*livekit.VideoLayer, 0, len(c.Layers)) + for _, l := range c.Layers { + clonedLayers = append(clonedLayers, utils.CloneProto(l)) + } + origin.Layers = clonedLayers + } break } } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 8e6d16c31..58121423b 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1243,7 +1243,7 @@ func (p *ParticipantImpl) handleMigrateTracks() []*MediaTrack { // AddTrack is called when client intends to publish track. // records track details and lets client know it's ok to proceed func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { - p.params.Logger.Debugw("add track request", "trackID", req.Cid) + p.params.Logger.Debugw("add track request", "trackID", req.Cid, "request", logger.Proto(req)) if !p.CanPublishSource(req.Source) { p.pubLogger.Warnw("no permission to publish track", nil, "trackID", req.Sid, "kind", req.Type) return @@ -2627,7 +2627,7 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l return nil } - track.(*MediaTrack).UpdateCodecCid(req.SimulcastCodecs) + track.(*MediaTrack).UpdateCodecInfo(req.SimulcastCodecs) ti := track.ToProto() return ti } @@ -2664,17 +2664,29 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } p.setTrackID(req.Cid, ti) + cloneLayers := func(layers []*livekit.VideoLayer) []*livekit.VideoLayer { + clonedLayers := make([]*livekit.VideoLayer, 0, len(layers)) + for _, l := range layers { + clonedLayers = append(clonedLayers, utils.CloneProto(l)) + } + return clonedLayers + } + if len(req.SimulcastCodecs) == 0 { if req.Type == livekit.TrackType_VIDEO { // clients not supporting simulcast codecs, synthesise a codec ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ Cid: req.Cid, - Layers: req.Layers, + Layers: cloneLayers(req.Layers), }) } } else { seenCodecs := make(map[string]struct{}) for _, codec := range req.SimulcastCodecs { + if codec.Codec == "" { + continue + } + mimeType := codec.Codec if req.Type == livekit.TrackType_VIDEO { if !mime.IsMimeTypeStringVideo(mimeType) { @@ -2682,7 +2694,8 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mimeType}) { altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) - p.pubLogger.Infow("falling back to alternative codec", + p.pubLogger.Infow( + "falling back to alternative codec", "codec", mimeType, "altCodec", altCodec, "trackID", ti.Sid, @@ -2699,16 +2712,35 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } seenCodecs[mimeType] = struct{}{} - clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) - for _, l := range req.Layers { - clonedLayers = append(clonedLayers, utils.CloneProto(l)) - } ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ MimeType: mimeType, Cid: codec.Cid, - Layers: clonedLayers, }) } + + // set up layers with codec specific layers, + // fall back to common layers if codec specific layer is not available + for _, codec := range ti.Codecs { + found := false + for _, simulcastCodec := range req.SimulcastCodecs { + if mime.GetMimeTypeCodec(codec.MimeType) != mime.NormalizeMimeTypeCodec(simulcastCodec.Codec) { + continue + } + + if len(simulcastCodec.Layers) != 0 { + codec.Layers = cloneLayers(simulcastCodec.Layers) + } else { + codec.Layers = cloneLayers(req.Layers) + } + found = true + break + } + + if !found { + // could happen if an alternate codec is selected and that is not in the simulcast codecs list + codec.Layers = cloneLayers(req.Layers) + } + } } p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), utils.CloneProto(ti))