livekit/pkg/rtc/participant_sdp.go

// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rtc

import (
	"fmt"
	"slices"
	"strconv"
	"strings"

	"github.com/pion/sdp/v3"
	"github.com/pion/webrtc/v4"
	"google.golang.org/protobuf/proto"

	"github.com/livekit/livekit-server/pkg/rtc/types"
	"github.com/livekit/protocol/codecs/mime"
	"github.com/livekit/protocol/livekit"
	"github.com/livekit/protocol/logger"
	lksdp "github.com/livekit/protocol/sdp"
)

func (p *ParticipantImpl) populateSdpCid(parsedOffer *sdp.SessionDescription) ([]*sdp.MediaDescription, []*sdp.MediaDescription) {
	processUnmatch := func(unmatches []*sdp.MediaDescription, trackType livekit.TrackType) {
		for _, unmatch := range unmatches {
			streamID, ok := lksdp.ExtractStreamID(unmatch)
			if !ok {
				continue
			}

			sdpCodecs, err := lksdp.CodecsFromMediaDescription(unmatch)
			if err != nil || len(sdpCodecs) == 0 {
				p.pubLogger.Errorw(
					"extract codecs from media section failed", err,
					"media", unmatch,
					"parsedOffer", parsedOffer,
				)
				continue
			}

			p.pendingTracksLock.Lock()
			if p.isReleasedTrackCidLocked(streamID) {
				p.pendingTracksLock.Unlock()
				continue
			}

			signalCid, info, _, migrated, _ := p.getPendingTrack(streamID, trackType, false)
			if migrated {
				p.pendingTracksLock.Unlock()
				continue
			}

			if info == nil {
				p.pendingTracksLock.Unlock()

				// could be already published track and the unmatch could be a back up codec publish
				numUnmatchedTracks := 0
				var unmatchedTrack types.MediaTrack
				var unmatchedSdpMimeType mime.MimeType

				found := false
				for _, sdpCodec := range sdpCodecs {
					sdpMimeType := mime.NormalizeMimeTypeCodec(sdpCodec.Name).ToMimeType()
					for _, publishedTrack := range p.GetPublishedTracks() {
						if sigCid, sdpCid := publishedTrack.(*MediaTrack).GetCidsForMimeType(sdpMimeType); sigCid != "" && sdpCid == "" {
							// a back up codec has a SDP cid match
							if sigCid == streamID {
								found = true
								break
							} else {
								numUnmatchedTracks++
								unmatchedTrack = publishedTrack
								unmatchedSdpMimeType = sdpMimeType
							}
						}
					}
					if found {
						break
					}
				}
				if !found && unmatchedTrack != nil {
					if numUnmatchedTracks != 1 {
						p.pubLogger.Warnw(
							"too many unmatched tracks", nil,
							"media", unmatch,
							"parsedOffer", parsedOffer,
						)
					}
					unmatchedTrack.(*MediaTrack).UpdateCodecSdpCid(unmatchedSdpMimeType, streamID)
					p.pubLogger.Debugw(
						"published track SDP cid updated",
						"trackID", unmatchedTrack.ID(),
						"track", logger.Proto(unmatchedTrack.ToProto()),
					)
				}
				continue
			}

			if len(info.Codecs) == 0 {
				p.pendingTracksLock.Unlock()
				p.pubLogger.Warnw(
					"track without codecs", nil,
					"trackID", info.Sid,
					"pendingTrack", p.pendingTracks[signalCid],
					"media", unmatch,
					"parsedOffer", parsedOffer,
				)
				continue
			}

			found := false
			updated := false
			for _, sdpCodec := range sdpCodecs {
				if mime.NormalizeMimeTypeCodec(sdpCodec.Name) == mime.GetMimeTypeCodec(info.Codecs[0].MimeType) {
					// set SdpCid only if different from SignalCid
					if streamID != info.Codecs[0].Cid {
						info.Codecs[0].SdpCid = streamID
						updated = true
					}
					found = true
					break
				}
				if found {
					break
				}
			}

			if !found {
				// not using SimulcastCodec, i. e. mime type not available till track publish
				if len(info.Codecs) == 1 {
					// set SdpCid only if different from SignalCid
					if streamID != info.Codecs[0].Cid {
						info.Codecs[0].SdpCid = streamID
						updated = true
					}
				}
			}

			if updated {
				p.pendingTracks[signalCid].trackInfos[0] = proto.CloneOf(info)
				p.pubLogger.Debugw(
					"pending track SDP cid updated",
					"signalCid", signalCid,
					"trackID", info.Sid,
					"pendingTrack", p.pendingTracks[signalCid],
				)
			}
			p.pendingTracksLock.Unlock()
		}
	}

	unmatchAudios, err := p.TransportManager.GetUnmatchMediaForOffer(parsedOffer, "audio")
	if err != nil {
		p.pubLogger.Warnw("could not get unmatch audios", err)
		return nil, nil
	}

	unmatchVideos, err := p.TransportManager.GetUnmatchMediaForOffer(parsedOffer, "video")
	if err != nil {
		p.pubLogger.Warnw("could not get unmatch videos", err)
		return nil, nil
	}

	processUnmatch(unmatchAudios, livekit.TrackType_AUDIO)
	processUnmatch(unmatchVideos, livekit.TrackType_VIDEO)
	return unmatchAudios, unmatchVideos
}

func (p *ParticipantImpl) setCodecPreferencesForPublisher(
	parsedOffer *sdp.SessionDescription,
	unmatchAudios []*sdp.MediaDescription,
	unmatchVideos []*sdp.MediaDescription,
) {
	unprocessedUnmatchAudios := p.setCodecPreferencesForPublisherMedia(
		parsedOffer,
		unmatchAudios,
		livekit.TrackType_AUDIO,
	)
	p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unprocessedUnmatchAudios)
	_ = p.setCodecPreferencesForPublisherMedia(
		parsedOffer,
		unmatchVideos,
		livekit.TrackType_VIDEO,
	)
}

func (p *ParticipantImpl) setCodecPreferencesForPublisherMedia(
	parsedOffer *sdp.SessionDescription,
	unmatches []*sdp.MediaDescription,
	trackType livekit.TrackType,
) []*sdp.MediaDescription {
	unprocessed := make([]*sdp.MediaDescription, 0, len(unmatches))
	for _, unmatch := range unmatches {
		var ti *livekit.TrackInfo
		var mimeType string

		mid := lksdp.GetMidValue(unmatch)
		if mid == "" {
			unprocessed = append(unprocessed, unmatch)
			continue
		}
		transceiver := p.TransportManager.GetPublisherRTPTransceiver(mid)
		if transceiver == nil {
			unprocessed = append(unprocessed, unmatch)
			continue
		}

		streamID, ok := lksdp.ExtractStreamID(unmatch)
		if !ok {
			unprocessed = append(unprocessed, unmatch)
			continue
		}

		p.pendingTracksLock.RLock()
		mt := p.getPublishedTrackBySdpCid(streamID)
		if mt != nil {
			ti = mt.ToProto()
		} else {
			_, ti, _, _, _ = p.getPendingTrack(streamID, trackType, false)
		}
		p.pendingTracksLock.RUnlock()

		if ti == nil {
			unprocessed = append(unprocessed, unmatch)
			continue
		}

		for _, c := range ti.Codecs {
			if c.Cid == streamID || c.SdpCid == streamID {
				mimeType = c.MimeType
				break
			}
		}
		if mimeType == "" && len(ti.Codecs) > 0 {
			mimeType = ti.Codecs[0].MimeType
		}

		if mimeType == "" {
			unprocessed = append(unprocessed, unmatch)
			continue
		}

		configureReceiverCodecs(
			transceiver,
			mimeType,
			p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer(),
		)
	}

	return unprocessed
}

func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(
	parsedOffer *sdp.SessionDescription,
	unmatchAudios []*sdp.MediaDescription,
) {
	for _, unmatchAudio := range unmatchAudios {
		mid := lksdp.GetMidValue(unmatchAudio)
		if mid == "" {
			continue
		}
		transceiver := p.TransportManager.GetPublisherRTPTransceiver(mid)
		if transceiver == nil {
			continue
		}

		streamID, ok := lksdp.ExtractStreamID(unmatchAudio)
		if !ok {
			continue
		}

		p.pendingTracksLock.RLock()
		_, ti, _, _, _ := p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false)
		p.pendingTracksLock.RUnlock()
		if ti == nil {
			continue
		}

		codecs, err := lksdp.CodecsFromMediaDescription(unmatchAudio)
		if err != nil {
			p.pubLogger.Errorw(
				"extract codecs from media section failed", err,
				"media", unmatchAudio,
				"parsedOffer", parsedOffer,
			)
			continue
		}

		var opusPayload uint8
		for _, codec := range codecs {
			if mime.IsMimeTypeCodecStringOpus(codec.Name) {
				opusPayload = codec.PayloadType
				break
			}
		}
		if opusPayload == 0 {
			continue
		}

		preferRED := IsRedEnabled(ti)
		// if RED is enabled for this track, prefer RED codec in offer
		for _, codec := range codecs {
			// codec contain opus/red
			if preferRED &&
				mime.IsMimeTypeCodecStringRED(codec.Name) &&
				strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) {
				configureReceiverCodecs(transceiver, "audio/red", true)
				break
			}
		}
	}
}

// configure publisher answer for audio track's dtx and stereo settings
func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescription) webrtc.SessionDescription {
	parsedOffer := p.TransportManager.LastPublisherOfferParsed()
	if parsedOffer == nil {
		return answer
	}

	parsedAnswer, err := answer.Unmarshal()
	if err != nil {
		return answer
	}

	// payload type lookup scans the whole answer, resolve it once
	opusPT, opusPTErr := parsedAnswer.GetPayloadTypeForCodec(sdp.Codec{Name: mime.MimeTypeCodecOpus.String()})
	changed := false
	for _, m := range parsedAnswer.MediaDescriptions {
		switch m.MediaName.Media {
		case "audio":
			_, ok := m.Attribute(sdp.AttrKeyInactive)
			if ok {
				continue
			}
			mid, ok := m.Attribute(sdp.AttrKeyMID)
			if !ok {
				continue
			}
			// find track info from offer's stream id
			var ti *livekit.TrackInfo
			for _, om := range parsedOffer.MediaDescriptions {
				_, ok := om.Attribute(sdp.AttrKeyInactive)
				if ok {
					continue
				}
				omid, ok := om.Attribute(sdp.AttrKeyMID)
				if ok && omid == mid {
					streamID, ok := lksdp.ExtractStreamID(om)
					if !ok {
						continue
					}
					track, _ := p.getPublishedTrackBySdpCid(streamID).(*MediaTrack)
					if track == nil {
						p.pendingTracksLock.RLock()
						_, ti, _, _, _ = p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false)
						p.pendingTracksLock.RUnlock()
					} else {
						ti = track.ToProto()
					}
					break
				}
			}

			if ti == nil {
				// no need to configure
				continue
			}

			if opusPTErr != nil {
				p.pubLogger.Infow("failed to get opus payload type", "error", opusPTErr, "trackID", ti.Sid)
				continue
			}

			for i, attr := range m.Attributes {
				if strings.HasPrefix(attr.String(), fmt.Sprintf("fmtp:%d", opusPT)) {
					if !slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) {
						attr.Value += ";usedtx=1"
					} else {
						attr.Value = strings.ReplaceAll(attr.Value, ";usedtx=1", "")
					}
					if slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) {
						attr.Value += ";stereo=1;maxaveragebitrate=510000"
					} else {
						attr.Value = strings.ReplaceAll(attr.Value, ";stereo=1", "")
					}
					m.Attributes[i] = attr
					changed = true
				}
			}

		default:
			continue
		}
	}

	if !changed {
		return answer
	}

	bytes, err := parsedAnswer.Marshal()
	if err != nil {
		p.pubLogger.Infow("failed to marshal answer", "error", err)
		return answer
	}
	answer.SDP = string(bytes)
	return answer
}