diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index 1a87a0f94..594d4e688 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -25,8 +25,8 @@ import ( ) const ( - frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" - repairedRTPStreamID = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id" + frameMarkingURI = "urn:ietf:params:rtp-hdrext:framemarking" + repairedRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id" ) type WebRTCConfig struct { @@ -92,9 +92,9 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { sdp.SDESMidURI, sdp.SDESRTPStreamIDURI, sdp.TransportCCURI, - frameMarking, + frameMarkingURI, dd.ExtensionURI, - repairedRTPStreamID, + repairedRTPStreamIDURI, //act.AbsCaptureTimeURI, }, }, diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index 579cb0a42..3b87e5e75 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -24,169 +24,188 @@ import ( "github.com/livekit/protocol/livekit" ) -var OpusCodecCapability = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeOpus.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "minptime=10;useinbandfec=1", -} -var RedCodecCapability = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRED.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "111/111", -} -var videoRTX = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRTX.String(), - ClockRate: 90000, -} +var ( + OpusCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeOpus.String(), + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", + }, + PayloadType: 111, + } + + RedCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeRED.String(), + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "111/111", + }, + PayloadType: 63, + } + + pcmuCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypePCMU.String(), + ClockRate: 8000, + }, + PayloadType: 0, + } + + pcmaCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypePCMA.String(), + ClockRate: 8000, + }, + PayloadType: 8, + } + + videoRTXCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeRTX.String(), + ClockRate: 90000, + }, + } + + vp8CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeVP8.String(), + ClockRate: 90000, + }, + PayloadType: 96, + } + + vp9ProfileId0CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeVP9.String(), + ClockRate: 90000, + SDPFmtpLine: "profile-id=0", + }, + PayloadType: 98, + } + + vp9ProfileId1CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeVP9.String(), + ClockRate: 90000, + SDPFmtpLine: "profile-id=1", + }, + PayloadType: 100, + } + + h264ProfileLevelId42e01fPacketizationMode0CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH264.String(), + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, + PayloadType: 125, + } + + h264ProfileLevelId42e01fPacketizationMode1CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH264.String(), + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", + }, + PayloadType: 108, + } + + h264HighProfileFmtp = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" + h264HighProfileCodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH264.String(), + ClockRate: 90000, + SDPFmtpLine: h264HighProfileFmtp, + }, + PayloadType: 123, + } + + av1CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeAV1.String(), + ClockRate: 90000, + }, + PayloadType: 35, + } + + h265CodecParameters = webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH265.String(), + ClockRate: 90000, + }, + PayloadType: 116, + } + + videoCodecsParameters = []webrtc.RTPCodecParameters{ + vp8CodecParameters, + vp9ProfileId0CodecParameters, + vp9ProfileId1CodecParameters, + h264ProfileLevelId42e01fPacketizationMode0CodecParameters, + h264ProfileLevelId42e01fPacketizationMode1CodecParameters, + h264HighProfileCodecParameters, + av1CodecParameters, + h265CodecParameters, + } +) func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { - opusCodec := OpusCodecCapability - opusCodec.RTCPFeedback = rtcpFeedback.Audio - var opusPayload webrtc.PayloadType - if IsCodecEnabled(codecs, opusCodec) { - opusPayload = 111 - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: opusCodec, - PayloadType: opusPayload, - }, webrtc.RTPCodecTypeAudio); err != nil { + // audio codecs + if IsCodecEnabled(codecs, OpusCodecParameters.RTPCodecCapability) { + cp := OpusCodecParameters + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Audio + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeAudio); err != nil { return err } - if IsCodecEnabled(codecs, RedCodecCapability) { - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: RedCodecCapability, - PayloadType: 63, - }, webrtc.RTPCodecTypeAudio); err != nil { + if IsCodecEnabled(codecs, RedCodecParameters.RTPCodecCapability) { + if err := me.RegisterCodec(RedCodecParameters, webrtc.RTPCodecTypeAudio); err != nil { return err } } } - for _, codec := range []webrtc.RTPCodecParameters{ - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypePCMU.String(), - ClockRate: 8000, - Channels: 1, - RTCPFeedback: rtcpFeedback.Audio, - }, - PayloadType: 0, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypePCMA.String(), - ClockRate: 8000, - Channels: 1, - RTCPFeedback: rtcpFeedback.Audio, - }, - PayloadType: 8, - }, - } { - if IsCodecEnabled(codecs, codec.RTPCodecCapability) { - if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil { - return err - } + for _, codec := range []webrtc.RTPCodecParameters{pcmuCodecParameters, pcmaCodecParameters} { + if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { + continue + } + + cp := codec + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Audio + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeAudio); err != nil { + return err } } - rtxEnabled := IsCodecEnabled(codecs, videoRTX) - - h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" - for _, codec := range []webrtc.RTPCodecParameters{ - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP8.String(), - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 96, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=0", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 98, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=1", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 100, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 125, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 108, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: h264HighProfileFmtp, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 123, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeAV1.String(), - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 35, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH265.String(), - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 116, - }, - } { + // video codecs + rtxEnabled := IsCodecEnabled(codecs, videoRTXCodecParameters.RTPCodecCapability) + for _, codec := range videoCodecsParameters { if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp { continue } if mime.IsMimeTypeStringRTX(codec.MimeType) { continue } - if IsCodecEnabled(codecs, codec.RTPCodecCapability) { - if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { - return err - } - if rtxEnabled { - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRTX.String(), - ClockRate: 90000, - SDPFmtpLine: fmt.Sprintf("apt=%d", codec.PayloadType), - }, - PayloadType: codec.PayloadType + 1, - }, webrtc.RTPCodecTypeVideo); err != nil { - return err - } - } + if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { + continue + } + + cp := codec + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Video + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + return err + } + + if !rtxEnabled { + continue + } + + cp = videoRTXCodecParameters + cp.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType) + cp.PayloadType = codec.PayloadType + 1 + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + return err } } return nil diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 79ac749a9..d534ccb2a 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1320,7 +1320,7 @@ func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { } if p.ProtocolVersion().SupportsSinglePeerConnection() { - if err := p.TransportManager.AddRemoteTrackAndNegotiate(ti); err != nil { + if err := p.TransportManager.AddRemoteTrackAndNegotiate(ti, p.enabledPublishCodecs, p.params.Config.Publisher.RTCPFeedback); err != nil { return } } diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index ed96e008b..6a8805815 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -141,7 +141,6 @@ func (p *ParticipantImpl) populateSdpCid(parsedOffer *sdp.SessionDescription) ([ p.pubLogger.Warnw("could not get unmatch audios", err) return nil, nil } - p.pubLogger.Infow("RAJA unmatch", "audios", unmatchAudios, "videos", unmatchVideos) // REMOVE processUnmatch(unmatchAudios, livekit.TrackType_AUDIO) processUnmatch(unmatchVideos, livekit.TrackType_VIDEO) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index eac019fe2..c9e5a58ed 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -972,7 +972,15 @@ func (t *PCTransport) AddTransceiverFromTrack(trackLocal webrtc.TrackLocal, para return } -func (t *PCTransport) AddRemoteTrackAndNegotiate(ti *livekit.TrackInfo) error { +func (t *PCTransport) AddTransceiverFromKind(kind webrtc.RTPCodecType, init webrtc.RTPTransceiverInit) (*webrtc.RTPTransceiver, error) { + return t.pc.AddTransceiverFromKind(kind, init) +} + +func (t *PCTransport) AddRemoteTrackAndNegotiate( + ti *livekit.TrackInfo, + publishEnabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, +) error { if ti == nil { return nil } @@ -981,7 +989,7 @@ func (t *PCTransport) AddRemoteTrackAndNegotiate(ti *livekit.TrackInfo) error { if ti.Type == livekit.TrackType_AUDIO { rtpCodecType = webrtc.RTPCodecTypeAudio } - _, err := t.pc.AddTransceiverFromKind( + transceiver, err := t.pc.AddTransceiverFromKind( rtpCodecType, webrtc.RTPTransceiverInit{ Direction: webrtc.RTPTransceiverDirectionRecvonly, @@ -996,8 +1004,26 @@ func (t *PCTransport) AddRemoteTrackAndNegotiate(ti *livekit.TrackInfo) error { return err } - t.Negotiate(true) + codecs := transceiver.Receiver().GetParameters().Codecs + t.params.Logger.Infow("RAJA codecs", "codecs", codecs) // REMOVE + enabledCodecs := make([]webrtc.RTPCodecParameters, 0, len(codecs)) + for _, c := range codecs { + for _, enabled := range publishEnabledCodecs { + if mime.NormalizeMimeType(enabled.Mime) == mime.NormalizeMimeType(c.RTPCodecCapability.MimeType) { + if rtpCodecType == webrtc.RTPCodecTypeVideo { + c.RTPCodecCapability.RTCPFeedback = rtcpFeedbackConfig.Video + } else { + c.RTPCodecCapability.RTCPFeedback = rtcpFeedbackConfig.Audio + } + // RAJA-TODO: only add RTX of enabled codecs + enabledCodecs = append(enabledCodecs, c) + break + } + } + } + transceiver.SetCodecPreferences(enabledCodecs) + t.Negotiate(true) return nil } @@ -2562,6 +2588,20 @@ func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription, } } + // SINGLE-PEER-CONNECTION-TODO: do this parsing and RTX deduction only for single peer connection mode + parsed, err := sd.Unmarshal() + if err != nil { + return err + } + + rtxRepairs := nonSimulcastRTXRepairsFromSDP(parsed, t.params.Logger) + if len(rtxRepairs) > 0 { + t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs) + for repair, base := range rtxRepairs { + t.params.Config.BufferFactory.SetRTXPair(repair, base) + } + } + if t.negotiationState == transport.NegotiationStateRetry { t.setNegotiationState(transport.NegotiationStateNone) diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index f6c383a01..4981189bb 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -286,8 +286,12 @@ func (t *TransportManager) AddTransceiverFromTrackLocal( } } -func (t *TransportManager) AddRemoteTrackAndNegotiate(ti *livekit.TrackInfo) error { - return t.subscriber.AddRemoteTrackAndNegotiate(ti) +func (t *TransportManager) AddRemoteTrackAndNegotiate( + ti *livekit.TrackInfo, + publishEnabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, +) error { + return t.subscriber.AddRemoteTrackAndNegotiate(ti, publishEnabledCodecs, rtcpFeedbackConfig) } func (t *TransportManager) RemoveTrackLocal(sender *webrtc.RTPSender) error { diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 54e01be3a..629a95d1c 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -63,16 +63,10 @@ func NewWrappedReceiver(params WrappedReceiverParams) *WrappedReceiver { normalizedMimeType := mime.NormalizeMimeType(codecs[0].MimeType) if normalizedMimeType == mime.MimeTypeRED { // if upstream is opus/red, then add opus to match clients that don't support red - codecs = append(codecs, webrtc.RTPCodecParameters{ - RTPCodecCapability: OpusCodecCapability, - PayloadType: 111, - }) + codecs = append(codecs, OpusCodecParameters) } else if !params.DisableRed && normalizedMimeType == mime.MimeTypeOpus { // if upstream is opus only and red enabled, add red to match clients that support red - codecs = append(codecs, webrtc.RTPCodecParameters{ - RTPCodecCapability: RedCodecCapability, - PayloadType: 63, - }) + codecs = append(codecs, RedCodecParameters) // prefer red codec codecs[0], codecs[1] = codecs[1], codecs[0] } diff --git a/test/client/client.go b/test/client/client.go index 4b24738b1..993eccd8c 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -785,9 +785,7 @@ func (c *RTCClient) AddFileTrack(path string, id string, label string) (writer * return nil, fmt.Errorf("%s has an unsupported extension", filepath.Base(path)) } - logger.Debugw("adding file track", - "mime", mime, - ) + logger.Debugw("adding file track", "mime", mime) track, err := webrtc.NewTrackLocalStaticSample( webrtc.RTPCodecCapability{MimeType: mime}, @@ -801,6 +799,37 @@ func (c *RTCClient) AddFileTrack(path string, id string, label string) (writer * return c.AddTrack(track, path) } +func (c *RTCClient) AddTransceiverOfKind(kind webrtc.RTPCodecType) error { + pc := c.publisher + if c.protocolVersion.SupportsSinglePeerConnection() { + pc = c.subscriber + } + if _, err := pc.AddTransceiverFromKind( + kind, + webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionRecvonly, + }, + ); err != nil { + return err + } + + // send AddTrackRequest with some mock parameters to trigger a negotiation + mimeType := "video/vp8" + trackType := livekit.TrackType_VIDEO + if kind == webrtc.RTPCodecTypeAudio { + mimeType = "audio/opus" + trackType = livekit.TrackType_AUDIO + } + if err := c.SendAddTrack(kind.String(), mimeType, kind.String(), trackType); err != nil { + return err + } + + if !c.protocolVersion.SupportsSinglePeerConnection() { + c.publisher.Negotiate(false) + } + return nil +} + // send AddTrack command to server to initiate server-side negotiation func (c *RTCClient) SendAddTrack(cid string, mimeType string, name string, trackType livekit.TrackType) error { return c.SendRequest(&livekit.SignalRequest{ diff --git a/test/singlenode_test.go b/test/singlenode_test.go index e74827427..8d3074373 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -660,10 +660,14 @@ func TestDeviceCodecOverride(t *testing.T) { defer c1.Stop() waitUntilConnected(t, c1) + /* RAJA-REMOVE // it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs tw, err := c1.AddStaticTrack("video/h264", "video", "webcam") require.NoError(t, err) defer stopWriters(tw) + */ + err := c1.AddTransceiverOfKind(webrtc.RTPCodecTypeVideo) + require.NoError(t, err) var sd *webrtc.SessionDescription // wait for server to receive track