Set track level codec settings for all pending tracks (#1148)

2026-03-30 17:45:40 +00:00 · 2022-11-07 17:45:23 +08:00
parent 0310aa9250
commit 5a8a6d11b2
2 changed files with 110 additions and 105 deletions
--- a/pkg/rtc/participant_sdp.go
+++ b/pkg/rtc/participant_sdp.go
@@ -20,58 +20,61 @@ func (p *ParticipantImpl) setCodecPreferencesForPublisher(offer webrtc.SessionDe
 }

 func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription {
-	parsed, lastAudio, err := p.TransportManager.GetLastUnmatchedMediaForOffer(offer, "audio")
-	if err != nil || lastAudio == nil {
+	parsed, unmatchAudios, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "audio")
+	if err != nil || len(unmatchAudios) == 0 {
 		return offer
 	}

-	streamID, ok := lksdp.ExtractStreamID(lastAudio)
-	if !ok {
-		return offer
-	}
+	for _, unmatchAudio := range unmatchAudios {
+		streamID, ok := lksdp.ExtractStreamID(unmatchAudio)
+		if !ok {
+			continue
+		}

-	p.pendingTracksLock.RLock()
-	_, info := p.getPendingTrack(streamID, livekit.TrackType_AUDIO)
-	// if RED is disabled for this track, don't prefer RED codec in offer
-	if info != nil && info.DisableRed {
+		p.pendingTracksLock.RLock()
+		_, info := p.getPendingTrack(streamID, livekit.TrackType_AUDIO)
+		// if RED is disabled for this track, don't prefer RED codec in offer
+		if info != nil && info.DisableRed {
+			p.pendingTracksLock.RUnlock()
+			continue
+		}
 		p.pendingTracksLock.RUnlock()
-		return offer
-	}
-	p.pendingTracksLock.RUnlock()

-	codecs, err := codecsFromMediaDescription(lastAudio)
-	if err != nil {
-		return offer
-	}
-
-	var opusPayload uint8
-	for _, codec := range codecs {
-		if strings.EqualFold(codec.Name, "opus") {
-			opusPayload = codec.PayloadType
-			break
+		codecs, err := codecsFromMediaDescription(unmatchAudio)
+		if err != nil {
+			p.params.Logger.Errorw("extract codecs from media section failed", err, "media", unmatchAudio)
+			continue
 		}
-	}
-	if opusPayload == 0 {
-		return offer
-	}

-	var preferredCodecs, leftCodecs []string
-	for _, codec := range codecs {
-		// codec contain opus/red
-		if strings.EqualFold(codec.Name, "red") && strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) {
-			preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
-		} else {
-			leftCodecs = append(leftCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
+		var opusPayload uint8
+		for _, codec := range codecs {
+			if strings.EqualFold(codec.Name, "opus") {
+				opusPayload = codec.PayloadType
+				break
+			}
+		}
+		if opusPayload == 0 {
+			continue
 		}
-	}

-	// no opus/red found
-	if len(preferredCodecs) == 0 {
-		return offer
-	}
+		var preferredCodecs, leftCodecs []string
+		for _, codec := range codecs {
+			// codec contain opus/red
+			if strings.EqualFold(codec.Name, "red") && strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) {
+				preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
+			} else {
+				leftCodecs = append(leftCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
+			}
+		}

-	lastAudio.MediaName.Formats = append(lastAudio.MediaName.Formats[:0], preferredCodecs...)
-	lastAudio.MediaName.Formats = append(lastAudio.MediaName.Formats, leftCodecs...)
+		// no opus/red found
+		if len(preferredCodecs) == 0 {
+			continue
+		}
+
+		unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats[:0], preferredCodecs...)
+		unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats, leftCodecs...)
+	}

 	bytes, err := parsed.Marshal()
 	if err != nil {
@@ -86,63 +89,66 @@ func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(offer webrtc.Se
 }

 func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription {
-	parsed, lastVideo, err := p.TransportManager.GetLastUnmatchedMediaForOffer(offer, "video")
-	if err != nil || lastVideo == nil {
+	parsed, unmatchVideos, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "video")
+	if err != nil || len(unmatchVideos) == 0 {
 		return offer
 	}
-	// last video is pending for publish, set codec preference
-	streamID, ok := lksdp.ExtractStreamID(lastVideo)
-	if !ok {
-		return offer
-	}
-
-	p.pendingTracksLock.RLock()
-	_, info := p.getPendingTrack(streamID, livekit.TrackType_VIDEO)
-	if info == nil {
-		p.pendingTracksLock.RUnlock()
-		return offer
-	}
-	var mime string
-	for _, c := range info.Codecs {
-		if c.Cid == streamID {
-			mime = c.MimeType
-			break
+	// unmatched video is pending for publish, set codec preference
+	for _, unmatchVideo := range unmatchVideos {
+		streamID, ok := lksdp.ExtractStreamID(unmatchVideo)
+		if !ok {
+			continue
 		}
-	}
-	if mime == "" && len(info.Codecs) > 0 {
-		mime = info.Codecs[0].MimeType
-	}
-	p.pendingTracksLock.RUnlock()

-	mime = strings.ToUpper(mime)
-	// remove dd extension if av1 not preferred
-	if !strings.Contains(mime, "AV1") {
-		for i, attr := range lastVideo.Attributes {
-			if strings.Contains(attr.Value, dd.ExtensionUrl) {
-				lastVideo.Attributes[i] = lastVideo.Attributes[len(lastVideo.Attributes)-1]
-				lastVideo.Attributes = lastVideo.Attributes[:len(lastVideo.Attributes)-1]
+		p.pendingTracksLock.RLock()
+		_, info := p.getPendingTrack(streamID, livekit.TrackType_VIDEO)
+		if info == nil {
+			p.pendingTracksLock.RUnlock()
+			continue
+		}
+		var mime string
+		for _, c := range info.Codecs {
+			if c.Cid == streamID {
+				mime = c.MimeType
 				break
 			}
 		}
-	}
-
-	if mime != "" {
-		codecs, err := codecsFromMediaDescription(lastVideo)
-		if err != nil {
-			return offer
+		if mime == "" && len(info.Codecs) > 0 {
+			mime = info.Codecs[0].MimeType
 		}
+		p.pendingTracksLock.RUnlock()

-		var preferredCodecs, leftCodecs []string
-		for _, c := range codecs {
-			if strings.HasSuffix(mime, strings.ToUpper(c.Name)) {
-				preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
-			} else {
-				leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+		mime = strings.ToUpper(mime)
+		// remove dd extension if av1 not preferred
+		if !strings.Contains(mime, "AV1") {
+			for i, attr := range unmatchVideo.Attributes {
+				if strings.Contains(attr.Value, dd.ExtensionUrl) {
+					unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1]
+					unmatchVideo.Attributes = unmatchVideo.Attributes[:len(unmatchVideo.Attributes)-1]
+					break
+				}
 			}
 		}

-		lastVideo.MediaName.Formats = append(lastVideo.MediaName.Formats[:0], preferredCodecs...)
-		lastVideo.MediaName.Formats = append(lastVideo.MediaName.Formats, leftCodecs...)
+		if mime != "" {
+			codecs, err := codecsFromMediaDescription(unmatchVideo)
+			if err != nil {
+				p.params.Logger.Errorw("extract codecs from media section failed", err, "media", unmatchVideo)
+				continue
+			}
+
+			var preferredCodecs, leftCodecs []string
+			for _, c := range codecs {
+				if strings.HasSuffix(mime, strings.ToUpper(c.Name)) {
+					preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+				} else {
+					leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+				}
+			}
+
+			unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats[:0], preferredCodecs...)
+			unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats, leftCodecs...)
+		}
 	}

 	bytes, err := parsed.Marshal()
--- a/pkg/rtc/transportmanager.go
+++ b/pkg/rtc/transportmanager.go
@@ -336,7 +336,7 @@ func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels [
 	return nil
 }

-func (t *TransportManager) GetLastUnmatchedMediaForOffer(offer webrtc.SessionDescription, mediaType string) (parsed *sdp.SessionDescription, unmatched *sdp.MediaDescription, err error) {
+func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescription, mediaType string) (parsed *sdp.SessionDescription, unmatched []*sdp.MediaDescription, err error) {
 	// prefer codec from offer for clients that don't support setCodecPreferences
 	parsed, err = offer.Unmarshal()
 	if err != nil {
@@ -344,18 +344,7 @@ func (t *TransportManager) GetLastUnmatchedMediaForOffer(offer webrtc.SessionDes
 		return
 	}

-	for i := len(parsed.MediaDescriptions) - 1; i >= 0; i-- {
-		media := parsed.MediaDescriptions[i]
-		if media.MediaName.Media == mediaType {
-			unmatched = media
-			break
-		}
-	}
-
-	if unmatched == nil {
-		return
-	}
-
+	var lastMatchedMid string
 	lastAnswer := t.lastPublisherAnswer.Load()
 	if lastAnswer != nil {
 		answer := lastAnswer.(webrtc.SessionDescription)
@@ -366,16 +355,26 @@ func (t *TransportManager) GetLastUnmatchedMediaForOffer(offer webrtc.SessionDes
 			return
 		}

-		for _, m := range parsedAnswer.MediaDescriptions {
-			mid, _ := m.Attribute(sdp.AttrKeyMID)
-			if lastMid, _ := unmatched.Attribute(sdp.AttrKeyMID); lastMid == mid {
-				// mid matched, return
-				unmatched = nil
-				return
+		for i := len(parsedAnswer.MediaDescriptions) - 1; i >= 0; i-- {
+			media := parsedAnswer.MediaDescriptions[i]
+			if media.MediaName.Media == mediaType {
+				lastMatchedMid, _ = media.Attribute(sdp.AttrKeyMID)
+				break
 			}
 		}
 	}

+	for i := len(parsed.MediaDescriptions) - 1; i >= 0; i-- {
+		media := parsed.MediaDescriptions[i]
+		if media.MediaName.Media == mediaType {
+			mid, _ := media.Attribute(sdp.AttrKeyMID)
+			if mid == lastMatchedMid {
+				break
+			}
+			unmatched = append(unmatched, media)
+		}
+	}
+
 	return
 }