From bf06596fcbfab60dffa85eaa8b8b048284201da6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 7 Oct 2025 16:23:28 +0530 Subject: [PATCH] Support Opus mixed with RED when encrypted. (#3986) Even when encrypted, can set up opus as the second codec to support the case of RED interspersed with Opus packets when the RED packet is too big to fit in one packet. The change here is to not go through all up stream codecs when trying to find a match in DownTrack.Bind when source is encrypted. When encrypted, the down track codec should match the primary upstream codec, i. e. the codec at index 0. --- pkg/rtc/subscribedtrack.go | 4 +++- pkg/rtc/wrappedreceiver.go | 2 +- pkg/sfu/downtrack.go | 43 ++++++++++++++++++++++++++------------ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 2c39bda3a..5309976e5 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -124,12 +124,14 @@ func NewSubscribedTrack(params SubscribedTrackParams) (*SubscribedTrack, error) streamID = PackSyncStreamID(params.MediaTrack.PublisherID(), params.MediaTrack.Stream()) } + isEncrypted := params.MediaTrack.IsEncrypted() var trailer []byte - if params.MediaTrack.IsEncrypted() { + if isEncrypted { trailer = params.Subscriber.GetTrailer() } downTrack, err := sfu.NewDownTrack(sfu.DownTrackParams{ Codecs: codecs, + IsEncrypted: isEncrypted, Source: params.MediaTrack.Source(), Receiver: params.WrappedReceiver, BufferFactory: params.Subscriber.GetBufferFactory(), diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index e29dfe799..4db580e68 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -60,7 +60,7 @@ func NewWrappedReceiver(params WrappedReceiverParams) *WrappedReceiver { } codecs := params.UpstreamCodecs - if len(codecs) == 1 && !params.IsEncrypted { + if len(codecs) == 1 { normalizedMimeType := mime.NormalizeMimeType(codecs[0].MimeType) if normalizedMimeType == mime.MimeTypeRED { // if upstream is opus/red, then add opus to match clients that don't support red diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 73a69c4c7..f5f7aec7f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -243,6 +243,7 @@ type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) type DownTrackParams struct { Codecs []webrtc.RTPCodecParameters + IsEncrypted bool Source livekit.TrackSource Receiver TrackReceiver BufferFactory *buffer.Factory @@ -355,8 +356,7 @@ type DownTrack struct { // NewDownTrack returns a DownTrack. func NewDownTrack(params DownTrackParams) (*DownTrack, error) { - codecs := params.Codecs - mimeType := mime.NormalizeMimeType(codecs[0].MimeType) + mimeType := mime.NormalizeMimeType(params.Codecs[0].MimeType) var kind webrtc.RTPCodecType switch { case mime.IsMimeTypeAudio(mimeType): @@ -367,19 +367,19 @@ func NewDownTrack(params DownTrackParams) (*DownTrack, error) { kind = webrtc.RTPCodecType(0) } + codec := params.Codecs[0].RTPCodecCapability d := &DownTrack{ params: params, id: params.Receiver.TrackID(), - upstreamCodecs: codecs, + upstreamCodecs: params.Codecs, kind: kind, - clockRate: codecs[0].ClockRate, + clockRate: codec.ClockRate, pacer: params.Pacer, maxLayerNotifierCh: make(chan string, 1), keyFrameRequesterCh: make(chan struct{}, 1), createdAt: time.Now().UnixNano(), receiver: params.Receiver, } - codec := codecs[0].RTPCodecCapability d.codec.Store(codec) d.bindState.Store(bindStateUnbound) d.params.Logger = params.Logger.WithValues( @@ -452,8 +452,9 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.bindLock.Unlock() return webrtc.RTPCodecParameters{}, ErrDownTrackAlreadyBound } - // the context's codec parameters will be set to the binded codec after Bind return so we keep - // a copy of the codec parameters here to use it later + + // the TrackLocalContext's codec parameters will be set to the bound codec after Bind returns, + // so keep a copy of the codec parameters here to use it later d.negotiatedCodecParameters = append([]webrtc.RTPCodecParameters{}, t.CodecParameters()...) var codec, matchedUpstreamCodec webrtc.RTPCodecParameters for _, c := range d.upstreamCodecs { @@ -462,6 +463,11 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, codec = matchCodec matchedUpstreamCodec = c break + } else { + // for encrypyted tracks, should match on primary codec, i. e. codec at index 0 + if d.params.IsEncrypted { + break + } } } @@ -469,7 +475,11 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, err := webrtc.ErrUnsupportedCodec onBinding := d.onBinding d.bindLock.Unlock() - d.params.Logger.Infow("bind error for unsupported codec", "codecs", d.upstreamCodecs, "remoteParameters", d.negotiatedCodecParameters) + d.params.Logger.Infow( + "bind error for unsupported codec", + "codecs", d.upstreamCodecs, + "remoteParameters", d.negotiatedCodecParameters, + ) if onBinding != nil { onBinding(err) } @@ -510,12 +520,19 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } } if d.upstreamPrimaryPT == 0 { - d.params.Logger.Errorw("failed to find upstream primary opus payload type for RED", nil, "matchedCodec", codec, "upstreamCodec", d.upstreamCodecs) + d.params.Logger.Errorw( + "failed to find upstream primary opus payload type for RED", nil, + "matchedCodec", codec, + "upstreamCodec", d.upstreamCodecs, + ) } var primaryPT, secondaryPT int if n, err := fmt.Sscanf(codec.SDPFmtpLine, "%d/%d", &primaryPT, &secondaryPT); err != nil || n != 2 { - d.params.Logger.Errorw("failed to parse primary and secondary payload type for RED", err, "matchedCodec", codec) + d.params.Logger.Errorw( + "failed to parse primary and secondary payload type for RED", err, + "matchedCodec", codec, + ) } d.primaryPT = uint8(primaryPT) } else if mime.IsMimeTypeStringAudio(matchedUpstreamCodec.MimeType) { @@ -2259,10 +2276,10 @@ func (d *DownTrack) addDummyExtensions(hdr *rtp.Header) { } } -func (d *DownTrack) getTranslatedPayloadType(src uint8) uint8 { - // send primary codec to subscriber if the publisher send primary codec to us when red is negotiated, +func (d *DownTrack) getTranslatedPayloadType(srcPT uint8) uint8 { + // send primary codec to subscriber if the publisher sent primary codec when red is negotiated, // this will happen when the payload is too large to encode into red payload (exceeds mtu). - if d.isRED && src == d.upstreamPrimaryPT && d.primaryPT != 0 { + if d.isRED && srcPT == d.upstreamPrimaryPT && d.primaryPT != 0 { return d.primaryPT } return uint8(d.payloadType.Load())