From 26c96ec283412a1006897beff533e7382bb71258 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 22 Dec 2023 17:09:49 +0530 Subject: [PATCH] Synthesise codec when adding pending track for no simulcast case also. (#2339) * Synthesise codec when adding pending track for no simulcast case also. Older clients not using simulcast codecs were failing e2e migration tests. Problem is that they did not have layer information and hence SSRC could not be set on migration. A codec was getting added later (when OnTrack was received). I missed adding layers in that code. Could have cloned layers there and added it. But, simplifying and adding at the start itself. Also, cleaning up code in `MediaTrackReceiver` for no codecs case as it should not happen any more. * clone per layer * fix priority determination --- pkg/rtc/mediatrack.go | 13 ++++++- pkg/rtc/mediatrackreceiver.go | 63 ++++-------------------------- pkg/rtc/participant.go | 73 ++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 89 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 73796cb73..7a3ca64fc 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -221,8 +221,17 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra break } } - if len(ti.Codecs) == 0 { - priority = 0 + if priority < 0 { + switch len(ti.Codecs) { + case 0: + // audio track + priority = 0 + case 1: + // older clients or non simulcast-codec, mime type only set later + if ti.Codecs[0].MimeType == "" { + priority = 0 + } + } } if priority < 0 { t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mime, "track", logger.Proto(ti)) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 3942264ef..b4128f690 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -182,11 +182,6 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority if priority == 0 { t.trackInfo.MimeType = receiver.Codec().MimeType t.trackInfo.Mid = mid - - // for clients don't have simulcast codecs (old version or single codec), add the primary codec - if len(t.trackInfo.Codecs) == 0 && t.trackInfo.Type == livekit.TrackType_VIDEO { - t.trackInfo.Codecs = append(t.trackInfo.Codecs, &livekit.SimulcastCodecInfo{}) - } } for i, ci := range t.trackInfo.Codecs { @@ -547,7 +542,7 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) } quality := buffer.SpatialLayerToVideoQuality(layer, t.trackInfo) // set video layer ssrc info - for _, ci := range t.trackInfo.Codecs { + for i, ci := range t.trackInfo.Codecs { if !strings.EqualFold(ci.MimeType, mime) { continue } @@ -567,26 +562,12 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) if !ssrcFound && matchingLayer != nil { matchingLayer.Ssrc = ssrc } - break - } - // for client don't use simulcast codecs (old client version or single codec) - if len(t.trackInfo.Codecs) == 0 { - // if origin layer has ssrc, don't override it - var matchingLayer *livekit.VideoLayer - ssrcFound := false - for _, l := range t.trackInfo.Layers { - if l.Quality == quality { - matchingLayer = l - if l.Ssrc != 0 { - ssrcFound = true - } - break - } - } - if !ssrcFound && matchingLayer != nil { - matchingLayer.Ssrc = ssrc + // for client don't use simulcast codecs (old client version or single codec) + if i == 0 { + t.trackInfo.Layers = ci.Layers } + break } t.lock.Unlock() @@ -636,25 +617,11 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { break } + // for client don't use simulcast codecs (old client version or single codec) if i == 0 { clonedInfo.Layers = ci.Layers } } - - // for client don't use simulcast codecs (old client version or single codec) - if len(clonedInfo.Codecs) == 0 { - for _, layer := range clonedInfo.Layers { - for _, originLayer := range t.trackInfo.Layers { - if layer.Quality == originLayer.Quality { - if originLayer.Ssrc != 0 { - layer.Ssrc = originLayer.Ssrc - } - break - } - } - } - } - t.trackInfo = clonedInfo t.lock.Unlock() @@ -679,27 +646,11 @@ func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { } } + // for client don't use simulcast codecs (old client version or single codec) if i == 0 { t.trackInfo.Layers = ci.Layers } } - - // for client don't use simulcast codecs (old client version or single codec) - if len(t.trackInfo.Codecs) == 0 { - originLayers := t.trackInfo.Layers - t.trackInfo.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - t.trackInfo.Layers = append(t.trackInfo.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - for _, l := range originLayers { - if l.Quality == t.trackInfo.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - t.trackInfo.Layers[layerIdx].Ssrc = l.Ssrc - } - break - } - } - } - } t.lock.Unlock() t.updateTrackInfoOfReceivers() diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b9c4d4942..154d7c794 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1645,41 +1645,52 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l ti.Stream = StreamFromTrackSource(ti.Source) } p.setStableTrackID(req.Cid, ti) - seenCodecs := make(map[string]struct{}) - for _, codec := range req.SimulcastCodecs { - mime := codec.Codec + + if len(req.SimulcastCodecs) == 0 { if req.Type == livekit.TrackType_VIDEO { - if !strings.HasPrefix(mime, "video/") { - mime = "video/" + mime + // clients not supporting simulcast codecs, synthesise a codec + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + Cid: req.Cid, + Layers: req.Layers, + }) + } + } else { + seenCodecs := make(map[string]struct{}) + for _, codec := range req.SimulcastCodecs { + mime := codec.Codec + if req.Type == livekit.TrackType_VIDEO { + if !strings.HasPrefix(mime, "video/") { + mime = "video/" + mime + } + if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { + altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) + p.pubLogger.Infow("falling back to alternative codec", + "codec", mime, + "altCodec", altCodec, + "trackID", ti.Sid, + ) + // select an alternative MIME type that's generally supported + mime = altCodec + } + } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { + mime = "audio/" + mime } - if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { - altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) - p.pubLogger.Infow("falling back to alternative codec", - "codec", mime, - "altCodec", altCodec, - "trackID", ti.Sid, - ) - // select an alternative MIME type that's generally supported - mime = altCodec + + if _, ok := seenCodecs[mime]; ok || mime == "" { + continue } - } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { - mime = "audio/" + mime - } + seenCodecs[mime] = struct{}{} - if _, ok := seenCodecs[mime]; ok || mime == "" { - continue + clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) + for _, l := range req.Layers { + clonedLayers = append(clonedLayers, proto.Clone(l).(*livekit.VideoLayer)) + } + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + MimeType: mime, + Cid: codec.Cid, + Layers: clonedLayers, + }) } - seenCodecs[mime] = struct{}{} - - clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) - for _, l := range req.Layers { - clonedLayers = append(clonedLayers, proto.Clone(l).(*livekit.VideoLayer)) - } - ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mime, - Cid: codec.Cid, - Layers: clonedLayers, - }) } p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), ti) @@ -1901,7 +1912,7 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo for _, codec := range ti.Codecs { for ssrc, info := range p.params.SimTracks { if info.Mid == codec.Mid { - mt.MediaTrackReceiver.SetLayerSsrc(codec.MimeType, info.Rid, ssrc) + mt.SetLayerSsrc(codec.MimeType, info.Rid, ssrc) } } }