diff --git a/pkg/config/config.go b/pkg/config/config.go index ced0c6708..17683dbf1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -355,6 +355,8 @@ var DefaultConfig = Config{ Room: RoomConfig{ AutoCreate: true, EnabledCodecs: []CodecSpec{ + {Mime: mime.MimeTypePCMU.String()}, + {Mime: mime.MimeTypePCMA.String()}, {Mime: mime.MimeTypeOpus.String()}, {Mime: mime.MimeTypeRED.String()}, {Mime: mime.MimeTypeVP8.String()}, diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index a6034468e..579cb0a42 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -64,6 +64,33 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac } } + for _, codec := range []webrtc.RTPCodecParameters{ + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypePCMU.String(), + ClockRate: 8000, + Channels: 1, + RTCPFeedback: rtcpFeedback.Audio, + }, + PayloadType: 0, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypePCMA.String(), + ClockRate: 8000, + Channels: 1, + RTCPFeedback: rtcpFeedback.Audio, + }, + PayloadType: 8, + }, + } { + if IsCodecEnabled(codecs, codec.RTPCodecCapability) { + if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil { + return err + } + } + } + rtxEnabled := IsCodecEnabled(codecs, videoRTX) h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" @@ -131,7 +158,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac }, { RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeH265, + MimeType: mime.MimeTypeH265.String(), ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video, }, diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index a342ee81d..015e07bb1 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -470,93 +470,134 @@ func TestDisablePublishCodec(t *testing.T) { require.Eventually(t, func() bool { return publishReceived.Load() }, 5*time.Second, 10*time.Millisecond) } -func TestPreferVideoCodecForPublisher(t *testing.T) { - participant := newParticipantForTestWithOpts("123", &participantOpts{ - publisher: true, - }) - participant.SetMigrateState(types.MigrateStateComplete) - - pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) - require.NoError(t, err) - defer pc.Close() - - for i := 0; i < 2; i++ { - // publish h264 track without client preferred codec - trackCid := fmt.Sprintf("preferh264video%d", i) - participant.AddTrack(&livekit.AddTrackRequest{ - Type: livekit.TrackType_VIDEO, - Name: "video", - Width: 1280, - Height: 720, - Source: livekit.TrackSource_CAMERA, - SimulcastCodecs: []*livekit.SimulcastCodec{ - { - Codec: "h264", - Cid: trackCid, - }, +func TestPreferMediaCodecForPublisher(t *testing.T) { + testCases := []struct { + name string + mediaKind string + trackBaseCid string + preferredCodec string + addTrack *livekit.AddTrackRequest + mimeTypeStringChecker func(string) bool + mimeTypeCodecStringChecker func(string) bool + transceiverMimeType mime.MimeType + }{ + { + name: "video", + mediaKind: "video", + trackBaseCid: "preferH264Video", + preferredCodec: "h264", + addTrack: &livekit.AddTrackRequest{ + Type: livekit.TrackType_VIDEO, + Name: "video", + Width: 1280, + Height: 720, + Source: livekit.TrackSource_CAMERA, }, - }) + mimeTypeStringChecker: mime.IsMimeTypeStringH264, + mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringH264, + transceiverMimeType: mime.MimeTypeVP8, + }, + { + name: "audio", + mediaKind: "audio", + trackBaseCid: "preferPCMAAudio", + preferredCodec: "pcma", + addTrack: &livekit.AddTrackRequest{ + Type: livekit.TrackType_AUDIO, + Name: "audio", + Source: livekit.TrackSource_MICROPHONE, + }, + mimeTypeStringChecker: mime.IsMimeTypeStringPCMA, + mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringPCMA, + transceiverMimeType: mime.MimeTypeOpus, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) - track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid) - require.NoError(t, err) - transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) - require.NoError(t, err) - codecs := transceiver.Receiver().GetParameters().Codecs + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() - if i > 0 { - // the negotiated codecs order could be updated by first negotiation, reorder to make h264 not preferred - for mime.IsMimeTypeStringH264(codecs[0].MimeType) { - codecs = append(codecs[1:], codecs[0]) - } - } - // h264 should not be preferred - require.False(t, mime.IsMimeTypeStringH264(codecs[0].MimeType), "codecs", codecs) - - sdp, err := pc.CreateOffer(nil) - require.NoError(t, err) - require.NoError(t, pc.SetLocalDescription(sdp)) - offerId := uint32(23) - - sink := &routingfakes.FakeMessageSink{} - participant.SetResponseSink(sink) - var answer webrtc.SessionDescription - var answerId uint32 - var answerReceived atomic.Bool - var answerIdReceived atomic.Uint32 - sink.WriteMessageCalls(func(msg proto.Message) error { - if res, ok := msg.(*livekit.SignalResponse); ok { - if res.GetAnswer() != nil { - answer, answerId = signalling.FromProtoSessionDescription(res.GetAnswer()) - pc.SetRemoteDescription(answer) - answerReceived.Store(true) - answerIdReceived.Store(answerId) + for i := 0; i < 2; i++ { + // publish preferred track without client using setCodecPreferences() + trackCid := fmt.Sprintf("%s-%d", tc.trackBaseCid, i) + req := utils.CloneProto(tc.addTrack) + req.SimulcastCodecs = []*livekit.SimulcastCodec{ + { + Codec: tc.preferredCodec, + Cid: trackCid, + }, } - } - return nil - }) - participant.HandleOffer(sdp, offerId) + participant.AddTrack(req) - require.Eventually(t, func() bool { return answerReceived.Load() && answerIdReceived.Load() == offerId }, 5*time.Second, 10*time.Millisecond) + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: tc.transceiverMimeType.String()}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + codecs := transceiver.Receiver().GetParameters().Codecs - var h264Preferred bool - parsed, err := answer.Unmarshal() - require.NoError(t, err) - var videoSectionIndex int - for _, m := range parsed.MediaDescriptions { - if m.MediaName.Media == "video" { - if videoSectionIndex == i { - codecs, err := lksdp.CodecsFromMediaDescription(m) - require.NoError(t, err) - if mime.IsMimeTypeCodecStringH264(codecs[0].Name) { - h264Preferred = true - break + if i > 0 { + // the negotiated codecs order could be updated by first negotiation, + // reorder to make tested preferred codec not preferred + for tc.mimeTypeStringChecker(codecs[0].MimeType) { + codecs = append(codecs[1:], codecs[0]) } } - videoSectionIndex++ - } - } + // preferred codec should not be preferred in `offer` + require.False(t, tc.mimeTypeStringChecker(codecs[0].MimeType), "codecs", codecs) - require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + require.NoError(t, pc.SetLocalDescription(sdp)) + offerId := uint32(23) + + sink := &routingfakes.FakeMessageSink{} + participant.SetResponseSink(sink) + var answer webrtc.SessionDescription + var answerId uint32 + var answerReceived atomic.Bool + var answerIdReceived atomic.Uint32 + sink.WriteMessageCalls(func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer, answerId = signalling.FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + answerIdReceived.Store(answerId) + } + } + return nil + }) + participant.HandleOffer(sdp, offerId) + + require.Eventually(t, func() bool { return answerReceived.Load() && answerIdReceived.Load() == offerId }, 5*time.Second, 10*time.Millisecond) + + var havePreferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var mediaSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == tc.mediaKind { + if mediaSectionIndex == i { + codecs, err := lksdp.CodecsFromMediaDescription(m) + require.NoError(t, err) + if tc.mimeTypeCodecStringChecker(codecs[0].Name) { + havePreferred = true + break + } + } + mediaSectionIndex++ + } + } + + require.Truef(t, havePreferred, "%s should be preferred for %s section %d, answer sdp: \n%s", tc.preferredCodec, tc.mediaKind, i, answer.SDP) + } + }) } } diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index c16ce7d89..cc95667df 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -152,8 +152,13 @@ func (p *ParticipantImpl) setCodecPreferencesForPublisher( unmatchAudios []*sdp.MediaDescription, unmatchVideos []*sdp.MediaDescription, ) *sdp.SessionDescription { - parsedOffer = p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unmatchAudios) - parsedOffer = p.setCodecPreferencesVideoForPublisher(parsedOffer, unmatchVideos) + parsedOffer, unprocessedUnmatchAudios := p.setCodecPreferencesForPublisherMedia( + parsedOffer, + unmatchAudios, + livekit.TrackType_AUDIO, + ) + parsedOffer = p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unprocessedUnmatchAudios) + parsedOffer, _ = p.setCodecPreferencesForPublisherMedia(parsedOffer, unmatchVideos, livekit.TrackType_VIDEO) return parsedOffer } @@ -231,14 +236,17 @@ func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher( return parsedOffer } -func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher( +func (p *ParticipantImpl) setCodecPreferencesForPublisherMedia( parsedOffer *sdp.SessionDescription, - unmatchVideos []*sdp.MediaDescription, -) *sdp.SessionDescription { - // unmatched video is pending for publish, set codec preference - for _, unmatchVideo := range unmatchVideos { - streamID, ok := lksdp.ExtractStreamID(unmatchVideo) + unmatches []*sdp.MediaDescription, + trackType livekit.TrackType, +) (*sdp.SessionDescription, []*sdp.MediaDescription) { + unprocessed := make([]*sdp.MediaDescription, 0, len(unmatches)) + // unmatched media is pending for publish, set codec preference + for _, unmatch := range unmatches { + streamID, ok := lksdp.ExtractStreamID(unmatch) if !ok { + unprocessed = append(unprocessed, unmatch) continue } @@ -248,13 +256,15 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher( if mt != nil { info = mt.ToProto() } else { - _, info, _, _, _ = p.getPendingTrack(streamID, livekit.TrackType_VIDEO, false) + _, info, _, _, _ = p.getPendingTrack(streamID, trackType, false) } if info == nil { p.pendingTracksLock.RUnlock() + unprocessed = append(unprocessed, unmatch) continue } + var mimeType string for _, c := range info.Codecs { if c.Cid == streamID || c.SdpCid == streamID { @@ -267,35 +277,66 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher( } p.pendingTracksLock.RUnlock() - if mimeType != "" { - codecs, err := lksdp.CodecsFromMediaDescription(unmatchVideo) - if err != nil { - p.pubLogger.Errorw( - "extract codecs from media section failed", err, - "media", unmatchVideo, - "parsedOffer", parsedOffer, - ) - continue - } + if mimeType == "" { + unprocessed = append(unprocessed, unmatch) + continue + } - var preferredCodecs, leftCodecs []string - for _, c := range codecs { - if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) { - preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) - } else { - leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) - } - } + codecs, err := lksdp.CodecsFromMediaDescription(unmatch) + if err != nil { + p.pubLogger.Errorw( + "extract codecs from media section failed", err, + "media", unmatch, + "parsedOffer", parsedOffer, + ) + unprocessed = append(unprocessed, unmatch) + continue + } - unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats[:0], preferredCodecs...) + var codecIdx int + var preferredCodecs, leftCodecs []string + for idx, c := range codecs { + if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) { + preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) + codecIdx = idx + } else { + leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) + } + } + + // could not find preferred mime in the offer + if len(preferredCodecs) == 0 { + unprocessed = append(unprocessed, unmatch) + continue + } + + unmatch.MediaName.Formats = append(unmatch.MediaName.Formats[:0], preferredCodecs...) + if trackType == livekit.TrackType_VIDEO { // if the client don't comply with codec order in SDP answer, only keep preferred codecs to force client to use it if p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer() { - unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats, leftCodecs...) + unmatch.MediaName.Formats = append(unmatch.MediaName.Formats, leftCodecs...) } + } else { + // ensure nack enabled for audio in publisher offer + var nackFound bool + for _, attr := range unmatch.Attributes { + if attr.Key == "rtcp-fb" && strings.Contains(attr.Value, fmt.Sprintf("%d nack", codecs[codecIdx].PayloadType)) { + nackFound = true + break + } + } + if !nackFound { + unmatch.Attributes = append(unmatch.Attributes, sdp.Attribute{ + Key: "rtcp-fb", + Value: fmt.Sprintf("%d nack", codecs[codecIdx].PayloadType), + }) + } + + unmatch.MediaName.Formats = append(unmatch.MediaName.Formats, leftCodecs...) } } - return parsedOffer + return parsedOffer, unprocessed } // configure publisher answer for audio track's dtx and stereo settings diff --git a/pkg/sfu/mime/mimetype.go b/pkg/sfu/mime/mimetype.go index 63ba28b9a..cada587ae 100644 --- a/pkg/sfu/mime/mimetype.go +++ b/pkg/sfu/mime/mimetype.go @@ -166,6 +166,14 @@ func IsMimeTypeCodecStringRED(codec string) bool { return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED } +func IsMimeTypeCodecStringPCMA(codec string) bool { + return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMA +} + +func IsMimeTypeCodecStringPCMU(codec string) bool { + return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMU +} + func IsMimeTypeCodecStringH264(codec string) bool { return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264 } @@ -336,6 +344,14 @@ func IsMimeTypeStringOpus(mime string) bool { return NormalizeMimeType(mime) == MimeTypeOpus } +func IsMimeTypeStringPCMA(mime string) bool { + return NormalizeMimeType(mime) == MimeTypePCMA +} + +func IsMimeTypeStringPCMU(mime string) bool { + return NormalizeMimeType(mime) == MimeTypePCMU +} + func IsMimeTypeStringRTX(mime string) bool { return NormalizeMimeType(mime) == MimeTypeRTX }