Support G.711 A-law and U-law (#3849)

* More codecs

* clean up

* clean up

* add to unprocessed for nil mime type

* enhance tests to check for audio codec preferences also

pkg/config/config.go

@@ -355,6 +355,8 @@ var DefaultConfig = Config{
 	Room: RoomConfig{
 		AutoCreate: true,
 		EnabledCodecs: []CodecSpec{
+			{Mime: mime.MimeTypePCMU.String()},
+			{Mime: mime.MimeTypePCMA.String()},
 			{Mime: mime.MimeTypeOpus.String()},
 			{Mime: mime.MimeTypeRED.String()},
 			{Mime: mime.MimeTypeVP8.String()},

pkg/rtc/mediaengine.go

@@ -64,6 +64,33 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 		}
 	}
 
+	for _, codec := range []webrtc.RTPCodecParameters{
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypePCMU.String(),
+				ClockRate:    8000,
+				Channels:     1,
+				RTCPFeedback: rtcpFeedback.Audio,
+			},
+			PayloadType: 0,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypePCMA.String(),
+				ClockRate:    8000,
+				Channels:     1,
+				RTCPFeedback: rtcpFeedback.Audio,
+			},
+			PayloadType: 8,
+		},
+	} {
+		if IsCodecEnabled(codecs, codec.RTPCodecCapability) {
+			if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil {
+				return err
+			}
+		}
+	}
+
 	rtxEnabled := IsCodecEnabled(codecs, videoRTX)
 
 	h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032"
@@ -131,7 +158,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 		},
 		{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:     webrtc.MimeTypeH265,
+				MimeType:     mime.MimeTypeH265.String(),
 				ClockRate:    90000,
 				RTCPFeedback: rtcpFeedback.Video,
 			},

pkg/rtc/participant_internal_test.go

@@ -470,93 +470,134 @@ func TestDisablePublishCodec(t *testing.T) {
 	require.Eventually(t, func() bool { return publishReceived.Load() }, 5*time.Second, 10*time.Millisecond)
 }
 
-func TestPreferVideoCodecForPublisher(t *testing.T) {
-	participant := newParticipantForTestWithOpts("123", &participantOpts{
-		publisher: true,
-	})
-	participant.SetMigrateState(types.MigrateStateComplete)
-
-	pc, err := webrtc.NewPeerConnection(webrtc.Configuration{})
-	require.NoError(t, err)
-	defer pc.Close()
-
-	for i := 0; i < 2; i++ {
-		// publish h264 track without client preferred codec
-		trackCid := fmt.Sprintf("preferh264video%d", i)
-		participant.AddTrack(&livekit.AddTrackRequest{
-			Type:   livekit.TrackType_VIDEO,
-			Name:   "video",
-			Width:  1280,
-			Height: 720,
-			Source: livekit.TrackSource_CAMERA,
-			SimulcastCodecs: []*livekit.SimulcastCodec{
-				{
-					Codec: "h264",
-					Cid:   trackCid,
-				},
+func TestPreferMediaCodecForPublisher(t *testing.T) {
+	testCases := []struct {
+		name                       string
+		mediaKind                  string
+		trackBaseCid               string
+		preferredCodec             string
+		addTrack                   *livekit.AddTrackRequest
+		mimeTypeStringChecker      func(string) bool
+		mimeTypeCodecStringChecker func(string) bool
+		transceiverMimeType        mime.MimeType
+	}{
+		{
+			name:           "video",
+			mediaKind:      "video",
+			trackBaseCid:   "preferH264Video",
+			preferredCodec: "h264",
+			addTrack: &livekit.AddTrackRequest{
+				Type:   livekit.TrackType_VIDEO,
+				Name:   "video",
+				Width:  1280,
+				Height: 720,
+				Source: livekit.TrackSource_CAMERA,
 			},
-		})
+			mimeTypeStringChecker:      mime.IsMimeTypeStringH264,
+			mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringH264,
+			transceiverMimeType:        mime.MimeTypeVP8,
+		},
+		{
+			name:           "audio",
+			mediaKind:      "audio",
+			trackBaseCid:   "preferPCMAAudio",
+			preferredCodec: "pcma",
+			addTrack: &livekit.AddTrackRequest{
+				Type:   livekit.TrackType_AUDIO,
+				Name:   "audio",
+				Source: livekit.TrackSource_MICROPHONE,
+			},
+			mimeTypeStringChecker:      mime.IsMimeTypeStringPCMA,
+			mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringPCMA,
+			transceiverMimeType:        mime.MimeTypeOpus,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			participant := newParticipantForTestWithOpts("123", &participantOpts{
+				publisher: true,
+			})
+			participant.SetMigrateState(types.MigrateStateComplete)
 
-		track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid)
-		require.NoError(t, err)
-		transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv})
-		require.NoError(t, err)
-		codecs := transceiver.Receiver().GetParameters().Codecs
+			pc, err := webrtc.NewPeerConnection(webrtc.Configuration{})
+			require.NoError(t, err)
+			defer pc.Close()
 
-		if i > 0 {
-			// the negotiated codecs order could be updated by first negotiation, reorder to make h264 not preferred
-			for mime.IsMimeTypeStringH264(codecs[0].MimeType) {
-				codecs = append(codecs[1:], codecs[0])
-			}
-		}
-		// h264 should not be preferred
-		require.False(t, mime.IsMimeTypeStringH264(codecs[0].MimeType), "codecs", codecs)
-
-		sdp, err := pc.CreateOffer(nil)
-		require.NoError(t, err)
-		require.NoError(t, pc.SetLocalDescription(sdp))
-		offerId := uint32(23)
-
-		sink := &routingfakes.FakeMessageSink{}
-		participant.SetResponseSink(sink)
-		var answer webrtc.SessionDescription
-		var answerId uint32
-		var answerReceived atomic.Bool
-		var answerIdReceived atomic.Uint32
-		sink.WriteMessageCalls(func(msg proto.Message) error {
-			if res, ok := msg.(*livekit.SignalResponse); ok {
-				if res.GetAnswer() != nil {
-					answer, answerId = signalling.FromProtoSessionDescription(res.GetAnswer())
-					pc.SetRemoteDescription(answer)
-					answerReceived.Store(true)
-					answerIdReceived.Store(answerId)
+			for i := 0; i < 2; i++ {
+				// publish preferred track without client using setCodecPreferences()
+				trackCid := fmt.Sprintf("%s-%d", tc.trackBaseCid, i)
+				req := utils.CloneProto(tc.addTrack)
+				req.SimulcastCodecs = []*livekit.SimulcastCodec{
+					{
+						Codec: tc.preferredCodec,
+						Cid:   trackCid,
+					},
 				}
-			}
-			return nil
-		})
-		participant.HandleOffer(sdp, offerId)
+				participant.AddTrack(req)
 
-		require.Eventually(t, func() bool { return answerReceived.Load() && answerIdReceived.Load() == offerId }, 5*time.Second, 10*time.Millisecond)
+				track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: tc.transceiverMimeType.String()}, trackCid, trackCid)
+				require.NoError(t, err)
+				transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv})
+				require.NoError(t, err)
+				codecs := transceiver.Receiver().GetParameters().Codecs
 
-		var h264Preferred bool
-		parsed, err := answer.Unmarshal()
-		require.NoError(t, err)
-		var videoSectionIndex int
-		for _, m := range parsed.MediaDescriptions {
-			if m.MediaName.Media == "video" {
-				if videoSectionIndex == i {
-					codecs, err := lksdp.CodecsFromMediaDescription(m)
-					require.NoError(t, err)
-					if mime.IsMimeTypeCodecStringH264(codecs[0].Name) {
-						h264Preferred = true
-						break
+				if i > 0 {
+					// the negotiated codecs order could be updated by first negotiation,
+					// reorder to make tested preferred codec not preferred
+					for tc.mimeTypeStringChecker(codecs[0].MimeType) {
+						codecs = append(codecs[1:], codecs[0])
 					}
 				}
-				videoSectionIndex++
-			}
-		}
+				// preferred codec should not be preferred in `offer`
+				require.False(t, tc.mimeTypeStringChecker(codecs[0].MimeType), "codecs", codecs)
 
-		require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP)
+				sdp, err := pc.CreateOffer(nil)
+				require.NoError(t, err)
+				require.NoError(t, pc.SetLocalDescription(sdp))
+				offerId := uint32(23)
+
+				sink := &routingfakes.FakeMessageSink{}
+				participant.SetResponseSink(sink)
+				var answer webrtc.SessionDescription
+				var answerId uint32
+				var answerReceived atomic.Bool
+				var answerIdReceived atomic.Uint32
+				sink.WriteMessageCalls(func(msg proto.Message) error {
+					if res, ok := msg.(*livekit.SignalResponse); ok {
+						if res.GetAnswer() != nil {
+							answer, answerId = signalling.FromProtoSessionDescription(res.GetAnswer())
+							pc.SetRemoteDescription(answer)
+							answerReceived.Store(true)
+							answerIdReceived.Store(answerId)
+						}
+					}
+					return nil
+				})
+				participant.HandleOffer(sdp, offerId)
+
+				require.Eventually(t, func() bool { return answerReceived.Load() && answerIdReceived.Load() == offerId }, 5*time.Second, 10*time.Millisecond)
+
+				var havePreferred bool
+				parsed, err := answer.Unmarshal()
+				require.NoError(t, err)
+				var mediaSectionIndex int
+				for _, m := range parsed.MediaDescriptions {
+					if m.MediaName.Media == tc.mediaKind {
+						if mediaSectionIndex == i {
+							codecs, err := lksdp.CodecsFromMediaDescription(m)
+							require.NoError(t, err)
+							if tc.mimeTypeCodecStringChecker(codecs[0].Name) {
+								havePreferred = true
+								break
+							}
+						}
+						mediaSectionIndex++
+					}
+				}
+
+				require.Truef(t, havePreferred, "%s should be preferred for %s section %d, answer sdp: \n%s", tc.preferredCodec, tc.mediaKind, i, answer.SDP)
+			}
+		})
 	}
 }
 

pkg/rtc/participant_sdp.go

@@ -152,8 +152,13 @@ func (p *ParticipantImpl) setCodecPreferencesForPublisher(
 	unmatchAudios []*sdp.MediaDescription,
 	unmatchVideos []*sdp.MediaDescription,
 ) *sdp.SessionDescription {
-	parsedOffer = p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unmatchAudios)
-	parsedOffer = p.setCodecPreferencesVideoForPublisher(parsedOffer, unmatchVideos)
+	parsedOffer, unprocessedUnmatchAudios := p.setCodecPreferencesForPublisherMedia(
+		parsedOffer,
+		unmatchAudios,
+		livekit.TrackType_AUDIO,
+	)
+	parsedOffer = p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unprocessedUnmatchAudios)
+	parsedOffer, _ = p.setCodecPreferencesForPublisherMedia(parsedOffer, unmatchVideos, livekit.TrackType_VIDEO)
 	return parsedOffer
 }
 
@@ -231,14 +236,17 @@ func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(
 	return parsedOffer
 }
 
-func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(
+func (p *ParticipantImpl) setCodecPreferencesForPublisherMedia(
 	parsedOffer *sdp.SessionDescription,
-	unmatchVideos []*sdp.MediaDescription,
-) *sdp.SessionDescription {
-	// unmatched video is pending for publish, set codec preference
-	for _, unmatchVideo := range unmatchVideos {
-		streamID, ok := lksdp.ExtractStreamID(unmatchVideo)
+	unmatches []*sdp.MediaDescription,
+	trackType livekit.TrackType,
+) (*sdp.SessionDescription, []*sdp.MediaDescription) {
+	unprocessed := make([]*sdp.MediaDescription, 0, len(unmatches))
+	// unmatched media is pending for publish, set codec preference
+	for _, unmatch := range unmatches {
+		streamID, ok := lksdp.ExtractStreamID(unmatch)
 		if !ok {
+			unprocessed = append(unprocessed, unmatch)
 			continue
 		}
 
@@ -248,13 +256,15 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(
 		if mt != nil {
 			info = mt.ToProto()
 		} else {
-			_, info, _, _, _ = p.getPendingTrack(streamID, livekit.TrackType_VIDEO, false)
+			_, info, _, _, _ = p.getPendingTrack(streamID, trackType, false)
 		}
 
 		if info == nil {
 			p.pendingTracksLock.RUnlock()
+			unprocessed = append(unprocessed, unmatch)
 			continue
 		}
+
 		var mimeType string
 		for _, c := range info.Codecs {
 			if c.Cid == streamID || c.SdpCid == streamID {
@@ -267,35 +277,66 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(
 		}
 		p.pendingTracksLock.RUnlock()
 
-		if mimeType != "" {
-			codecs, err := lksdp.CodecsFromMediaDescription(unmatchVideo)
-			if err != nil {
-				p.pubLogger.Errorw(
-					"extract codecs from media section failed", err,
-					"media", unmatchVideo,
-					"parsedOffer", parsedOffer,
-				)
-				continue
-			}
+		if mimeType == "" {
+			unprocessed = append(unprocessed, unmatch)
+			continue
+		}
 
-			var preferredCodecs, leftCodecs []string
-			for _, c := range codecs {
-				if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) {
-					preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
-				} else {
-					leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
-				}
-			}
+		codecs, err := lksdp.CodecsFromMediaDescription(unmatch)
+		if err != nil {
+			p.pubLogger.Errorw(
+				"extract codecs from media section failed", err,
+				"media", unmatch,
+				"parsedOffer", parsedOffer,
+			)
+			unprocessed = append(unprocessed, unmatch)
+			continue
+		}
 
-			unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats[:0], preferredCodecs...)
+		var codecIdx int
+		var preferredCodecs, leftCodecs []string
+		for idx, c := range codecs {
+			if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) {
+				preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+				codecIdx = idx
+			} else {
+				leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+			}
+		}
+
+		// could not find preferred mime in the offer
+		if len(preferredCodecs) == 0 {
+			unprocessed = append(unprocessed, unmatch)
+			continue
+		}
+
+		unmatch.MediaName.Formats = append(unmatch.MediaName.Formats[:0], preferredCodecs...)
+		if trackType == livekit.TrackType_VIDEO {
 			// if the client don't comply with codec order in SDP answer, only keep preferred codecs to force client to use it
 			if p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer() {
-				unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats, leftCodecs...)
+				unmatch.MediaName.Formats = append(unmatch.MediaName.Formats, leftCodecs...)
 			}
+		} else {
+			// ensure nack enabled for audio in publisher offer
+			var nackFound bool
+			for _, attr := range unmatch.Attributes {
+				if attr.Key == "rtcp-fb" && strings.Contains(attr.Value, fmt.Sprintf("%d nack", codecs[codecIdx].PayloadType)) {
+					nackFound = true
+					break
+				}
+			}
+			if !nackFound {
+				unmatch.Attributes = append(unmatch.Attributes, sdp.Attribute{
+					Key:   "rtcp-fb",
+					Value: fmt.Sprintf("%d nack", codecs[codecIdx].PayloadType),
+				})
+			}
+
+			unmatch.MediaName.Formats = append(unmatch.MediaName.Formats, leftCodecs...)
 		}
 	}
 
-	return parsedOffer
+	return parsedOffer, unprocessed
 }
 
 // configure publisher answer for audio track's dtx and stereo settings

pkg/sfu/mime/mimetype.go

@@ -166,6 +166,14 @@ func IsMimeTypeCodecStringRED(codec string) bool {
 	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED
 }
 
+func IsMimeTypeCodecStringPCMA(codec string) bool {
+	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMA
+}
+
+func IsMimeTypeCodecStringPCMU(codec string) bool {
+	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMU
+}
+
 func IsMimeTypeCodecStringH264(codec string) bool {
 	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264
 }
@@ -336,6 +344,14 @@ func IsMimeTypeStringOpus(mime string) bool {
 	return NormalizeMimeType(mime) == MimeTypeOpus
 }
 
+func IsMimeTypeStringPCMA(mime string) bool {
+	return NormalizeMimeType(mime) == MimeTypePCMA
+}
+
+func IsMimeTypeStringPCMU(mime string) bool {
+	return NormalizeMimeType(mime) == MimeTypePCMU
+}
+
 func IsMimeTypeStringRTX(mime string) bool {
 	return NormalizeMimeType(mime) == MimeTypeRTX
 }