Files
livekit/pkg/rtc/mediatrack.go
T
Artur Shellunts b0454b00a2 Count padding bytes in telemetry (#264)
Count padding bytes in telemetry outgoing total bytes
2021-12-20 12:04:50 +01:00

769 lines
21 KiB
Go

package rtc
import (
"context"
"errors"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v3"
"github.com/pion/webrtc/v3/pkg/rtcerr"
"github.com/livekit/livekit-server/pkg/config"
"github.com/livekit/livekit-server/pkg/rtc/types"
"github.com/livekit/livekit-server/pkg/sfu"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/twcc"
"github.com/livekit/livekit-server/pkg/telemetry"
)
var (
FeedbackTypes = []webrtc.RTCPFeedback{
{Type: webrtc.TypeRTCPFBGoogREMB},
{Type: webrtc.TypeRTCPFBNACK},
{Type: webrtc.TypeRTCPFBNACK, Parameter: "pli"}}
)
const (
lostUpdateDelta = time.Second
layerSelectionTolerance = 0.8
)
// MediaTrack represents a WebRTC track that needs to be forwarded
// Implements the PublishedTrack interface
type MediaTrack struct {
params MediaTrackParams
ssrc webrtc.SSRC
streamID string
codec webrtc.RTPCodecParameters
muted utils.AtomicFlag
numUpTracks uint32
simulcasted utils.AtomicFlag
buffer *buffer.Buffer
// channel to send RTCP packets to the source
lock sync.RWMutex
// map of target participantId -> types.SubscribedTrack
subscribedTracks sync.Map
twcc *twcc.Responder
audioLevel *AudioLevel
receiver sfu.Receiver
lastPLI time.Time
layerDimensions sync.Map // quality => *livekit.VideoLayer
// track audio fraction lost
fracLostLock sync.Mutex
maxDownFracLost uint8
maxDownFracLostTs time.Time
currentUpFracLost uint32
maxUpFracLost uint8
maxUpFracLostTs time.Time
// quality level enable/disable
maxQualityLock sync.Mutex
maxSubscriberQuality map[string]livekit.VideoQuality
maxSubscribedQuality livekit.VideoQuality
allSubscribersMuted bool
onSubscribedMaxQualityChange func(trackSid string, subscribedQualities []*livekit.SubscribedQuality) error
onClose []func()
}
type MediaTrackParams struct {
TrackInfo *livekit.TrackInfo
SignalCid string
SdpCid string
ParticipantID string
ParticipantIdentity string
RTCPChan chan []rtcp.Packet
BufferFactory *buffer.Factory
ReceiverConfig ReceiverConfig
AudioConfig config.AudioConfig
Telemetry telemetry.TelemetryService
Logger logger.Logger
}
func NewMediaTrack(track *webrtc.TrackRemote, params MediaTrackParams) *MediaTrack {
t := &MediaTrack{
params: params,
ssrc: track.SSRC(),
streamID: track.StreamID(),
codec: track.Codec(),
maxSubscriberQuality: make(map[string]livekit.VideoQuality),
}
if params.TrackInfo.Muted {
t.SetMuted(true)
}
if params.TrackInfo != nil && t.Kind() == livekit.TrackType_VIDEO {
t.UpdateVideoLayers(params.TrackInfo.Layers)
// LK-TODO: maybe use this or simulcast flag in TrackInfo to set simulcasted here
}
return t
}
func (t *MediaTrack) ID() string {
return t.params.TrackInfo.Sid
}
func (t *MediaTrack) SignalCid() string {
return t.params.SignalCid
}
func (t *MediaTrack) SdpCid() string {
return t.params.SdpCid
}
func (t *MediaTrack) Kind() livekit.TrackType {
return t.params.TrackInfo.Type
}
func (t *MediaTrack) Source() livekit.TrackSource {
return t.params.TrackInfo.Source
}
func (t *MediaTrack) IsSimulcast() bool {
return t.simulcasted.Get()
}
func (t *MediaTrack) Name() string {
return t.params.TrackInfo.Name
}
func (t *MediaTrack) IsMuted() bool {
return t.muted.Get()
}
func (t *MediaTrack) SetMuted(muted bool) {
t.muted.TrySet(muted)
t.lock.RLock()
if t.receiver != nil {
t.receiver.SetUpTrackPaused(muted)
}
t.lock.RUnlock()
// mute all subscribed tracks
t.subscribedTracks.Range(func(_, value interface{}) bool {
if st, ok := value.(types.SubscribedTrack); ok {
st.SetPublisherMuted(muted)
}
return true
})
// update quality based on subscription if unmuting
if !muted {
t.updateQualityChange()
}
}
func (t *MediaTrack) AddOnClose(f func()) {
if f == nil {
return
}
t.onClose = append(t.onClose, f)
}
func (t *MediaTrack) IsSubscriber(subId string) bool {
_, ok := t.subscribedTracks.Load(subId)
return ok
}
func (t *MediaTrack) PublishLossPercentage() uint32 {
return FixedPointToPercent(uint8(atomic.LoadUint32(&t.currentUpFracLost)))
}
// AddSubscriber subscribes sub to current mediaTrack
func (t *MediaTrack) AddSubscriber(sub types.Participant) error {
t.lock.Lock()
defer t.lock.Unlock()
// don't subscribe to the same track multiple times
if _, ok := t.subscribedTracks.Load(sub.ID()); ok {
return nil
}
if t.receiver == nil {
// cannot add, no receiver
return errors.New("cannot subscribe without a receiver in place")
}
codec := t.receiver.Codec()
// using DownTrack from ion-sfu
streamId := t.params.ParticipantID
if sub.ProtocolVersion().SupportsPackedStreamId() {
// when possible, pack both IDs in streamID to allow new streams to be generated
// react-native-webrtc still uses stream based APIs and require this
streamId = PackStreamID(t.params.ParticipantID, t.ID())
}
receiver := NewWrappedReceiver(t.receiver, t.ID(), streamId)
downTrack, err := sfu.NewDownTrack(webrtc.RTPCodecCapability{
MimeType: codec.MimeType,
ClockRate: codec.ClockRate,
Channels: codec.Channels,
SDPFmtpLine: codec.SDPFmtpLine,
RTCPFeedback: FeedbackTypes,
}, receiver, t.params.BufferFactory, sub.ID(), t.params.ReceiverConfig.PacketBufferSize)
if err != nil {
return err
}
subTrack := NewSubscribedTrack(t, sub.ID(), t.params.ParticipantIdentity, downTrack)
var transceiver *webrtc.RTPTransceiver
var sender *webrtc.RTPSender
if sub.ProtocolVersion().SupportsTransceiverReuse() {
//
// AddTrack will create a new transceiver or re-use an unused one
// if the attributes match. This prevents SDP from bloating
// because of dormant transceivers building up.
//
sender, err = sub.SubscriberPC().AddTrack(downTrack)
if err != nil {
return err
}
// as there is no way to get transceiver from sender, search
for _, tr := range sub.SubscriberPC().GetTransceivers() {
if tr.Sender() == sender {
transceiver = tr
break
}
}
if transceiver == nil {
// cannot add, no transceiver
return errors.New("cannot subscribe without a transceiver in place")
}
} else {
transceiver, err = sub.SubscriberPC().AddTransceiverFromTrack(downTrack, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
if err != nil {
return err
}
sender = transceiver.Sender()
if sender == nil {
// cannot add, no sender
return errors.New("cannot subscribe without a sender in place")
}
}
sendParameters := sender.GetParameters()
downTrack.SetRTPHeaderExtensions(sendParameters.HeaderExtensions)
downTrack.SetTransceiver(transceiver)
// when outtrack is bound, start loop to send reports
downTrack.OnBind(func() {
go subTrack.Bound()
go t.sendDownTrackBindingReports(sub)
})
downTrack.OnPacketSent(func(_ *sfu.DownTrack, size int) {
t.params.Telemetry.OnDownstreamPacket(sub.ID(), size)
})
downTrack.OnPaddingSent(func(_ *sfu.DownTrack, size int) {
t.params.Telemetry.OnDownstreamPacket(sub.ID(), size)
})
downTrack.OnRTCP(func(pkts []rtcp.Packet) {
t.params.Telemetry.HandleRTCP(livekit.StreamType_DOWNSTREAM, sub.ID(), pkts)
})
downTrack.OnCloseHandler(func() {
go func() {
t.subscribedTracks.Delete(sub.ID())
t.params.Telemetry.TrackUnsubscribed(context.Background(), sub.ID(), t.ToProto())
// ignore if the subscribing sub is not connected
if sub.SubscriberPC().ConnectionState() == webrtc.PeerConnectionStateClosed {
return
}
// if the source has been terminated, we'll need to terminate all of the subscribedtracks
// however, if the dest sub has disconnected, then we can skip
if sender == nil {
return
}
t.params.Logger.Debugw("removing peerconnection track",
"track", t.ID(),
"subscriber", sub.Identity(),
"subscriberID", sub.ID(),
"kind", t.Kind(),
)
if err := sub.SubscriberPC().RemoveTrack(sender); err != nil {
if err == webrtc.ErrConnectionClosed {
// sub closing, can skip removing subscribedtracks
return
}
if _, ok := err.(*rtcerr.InvalidStateError); !ok {
// most of these are safe to ignore, since the track state might have already
// been set to Inactive
t.params.Logger.Debugw("could not remove remoteTrack from forwarder",
"error", err,
"subscriber", sub.Identity(),
"subscriberID", sub.ID(),
)
}
}
t.NotifySubscriberMute(sub.ID())
sub.RemoveSubscribedTrack(subTrack)
sub.Negotiate()
}()
})
if t.Kind() == livekit.TrackType_AUDIO {
downTrack.AddReceiverReportListener(t.handleMaxLossFeedback)
}
t.subscribedTracks.Store(sub.ID(), subTrack)
subTrack.SetPublisherMuted(t.IsMuted())
t.receiver.AddDownTrack(downTrack)
// since sub will lock, run it in a goroutine to avoid deadlocks
go func() {
t.NotifySubscriberMaxQuality(sub.ID(), livekit.VideoQuality_HIGH) // start with HIGH, let subscription change it later
sub.AddSubscribedTrack(subTrack)
sub.Negotiate()
}()
t.params.Telemetry.TrackSubscribed(context.Background(), sub.ID(), t.ToProto())
return nil
}
func (t *MediaTrack) NumUpTracks() (uint32, uint32) {
numRegistered := atomic.LoadUint32(&t.numUpTracks)
var numPublishing uint32
if t.simulcasted.Get() {
t.lock.RLock()
numPublishing = uint32(t.receiver.NumAvailableSpatialLayers())
t.lock.RUnlock()
} else {
numPublishing = 1
}
return numPublishing, numRegistered
}
// AddReceiver adds a new RTP receiver to the track
func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, twcc *twcc.Responder) {
t.lock.Lock()
defer t.lock.Unlock()
buff, rtcpReader := t.params.BufferFactory.GetBufferPair(uint32(track.SSRC()))
if buff == nil || rtcpReader == nil {
logger.Errorw("could not retrieve buffer pair", nil,
"track", t.ID())
return
}
buff.OnFeedback(t.handlePublisherFeedback)
if t.Kind() == livekit.TrackType_AUDIO {
t.audioLevel = NewAudioLevel(t.params.AudioConfig.ActiveLevel, t.params.AudioConfig.MinPercentile)
buff.OnAudioLevel(func(level uint8, duration uint32) {
t.audioLevel.Observe(level, duration)
})
} else if t.Kind() == livekit.TrackType_VIDEO {
if twcc != nil {
buff.OnTransportWideCC(func(sn uint16, timeNS int64, marker bool) {
twcc.Push(sn, timeNS, marker)
})
}
}
rtcpReader.OnPacket(func(bytes []byte) {
pkts, err := rtcp.Unmarshal(bytes)
if err != nil {
t.params.Logger.Errorw("could not unmarshal RTCP", err)
return
}
for _, pkt := range pkts {
switch pkt := pkt.(type) {
case *rtcp.SourceDescription:
// do nothing for now
case *rtcp.SenderReport:
buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime)
}
}
})
if t.receiver == nil {
t.receiver = sfu.NewWebRTCReceiver(receiver, track, t.params.ParticipantID,
sfu.WithPliThrottle(0),
sfu.WithLoadBalanceThreshold(20),
sfu.WithStreamTrackers())
t.receiver.SetRTCPCh(t.params.RTCPChan)
t.receiver.OnCloseHandler(func() {
t.lock.Lock()
t.receiver = nil
onclose := t.onClose
t.lock.Unlock()
t.RemoveAllSubscribers()
t.params.Telemetry.TrackUnpublished(context.Background(), t.params.ParticipantID, t.ToProto(), uint32(track.SSRC()))
for _, f := range onclose {
f()
}
})
t.params.Telemetry.TrackPublished(context.Background(), t.params.ParticipantID, t.ToProto())
if t.Kind() == livekit.TrackType_AUDIO {
t.buffer = buff
}
}
t.receiver.AddUpTrack(track, buff)
t.params.Telemetry.AddUpTrack(t.params.ParticipantID, buff)
atomic.AddUint32(&t.numUpTracks, 1)
// LK-TODO: can remove this completely when VideoLayers protocol becomes the default as it has info from client or if we decide to use TrackInfo.Simulcast
if atomic.LoadUint32(&t.numUpTracks) > 1 || track.RID() != "" {
// cannot only rely on numUpTracks since we fire metadata events immediately after the first layer
t.simulcasted.TrySet(true)
}
buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, buffer.Options{
MaxBitRate: t.params.ReceiverConfig.maxBitrate,
})
}
// RemoveSubscriber removes participant from subscription
// stop all forwarders to the client
func (t *MediaTrack) RemoveSubscriber(participantId string) {
subTrack := t.getSubscribedTrack(participantId)
if subTrack != nil {
go subTrack.DownTrack().Close()
}
}
func (t *MediaTrack) RemoveAllSubscribers() {
t.params.Logger.Debugw("removing all subscribers", "track", t.ID())
t.lock.Lock()
defer t.lock.Unlock()
t.subscribedTracks.Range(func(_, val interface{}) bool {
if subTrack, ok := val.(types.SubscribedTrack); ok {
go subTrack.DownTrack().Close()
}
return true
})
t.subscribedTracks = sync.Map{}
}
func (t *MediaTrack) ToProto() *livekit.TrackInfo {
info := t.params.TrackInfo
info.Muted = t.IsMuted()
info.Simulcast = t.simulcasted.Get()
layers := make([]*livekit.VideoLayer, 0)
t.layerDimensions.Range(func(_, val interface{}) bool {
if layer, ok := val.(*livekit.VideoLayer); ok {
layers = append(layers, layer)
}
return true
})
info.Layers = layers
return info
}
func (t *MediaTrack) UpdateVideoLayers(layers []*livekit.VideoLayer) {
for _, layer := range layers {
t.layerDimensions.Store(layer.Quality, layer)
}
t.subscribedTracks.Range(func(_, val interface{}) bool {
if st, ok := val.(types.SubscribedTrack); ok {
st.UpdateVideoLayer()
}
return true
})
// TODO: this might need to trigger a participant update for clients to pick up dimension change
}
// GetQualityForDimension finds the closest quality to use for desired dimensions
// affords a 20% tolerance on dimension
func (t *MediaTrack) GetQualityForDimension(width, height uint32) livekit.VideoQuality {
quality := livekit.VideoQuality_HIGH
if t.Kind() == livekit.TrackType_AUDIO || t.params.TrackInfo.Height == 0 {
return quality
}
origSize := t.params.TrackInfo.Height
requestedSize := height
if t.params.TrackInfo.Width < t.params.TrackInfo.Height {
// for portrait videos
origSize = t.params.TrackInfo.Width
requestedSize = width
}
// default sizes representing qualities low - high
layerSizes := []uint32{180, 360, origSize}
var providedSizes []uint32
t.layerDimensions.Range(func(_, val interface{}) bool {
if layer, ok := val.(*livekit.VideoLayer); ok {
providedSizes = append(providedSizes, layer.Height)
}
return true
})
if len(providedSizes) > 0 {
layerSizes = providedSizes
// comparing height always
requestedSize = height
sort.Slice(layerSizes, func(i, j int) bool {
return layerSizes[i] < layerSizes[j]
})
}
// finds the lowest layer that could satisfy client demands
requestedSize = uint32(float32(requestedSize) * layerSelectionTolerance)
for i, s := range layerSizes {
quality = livekit.VideoQuality(i)
if s >= requestedSize {
break
}
}
return quality
}
func (t *MediaTrack) getSubscribedTrack(id string) types.SubscribedTrack {
if val, ok := t.subscribedTracks.Load(id); ok {
if st, ok := val.(types.SubscribedTrack); ok {
return st
}
}
return nil
}
// TODO: send for all downtracks from the source participant
// https://tools.ietf.org/html/rfc7941
func (t *MediaTrack) sendDownTrackBindingReports(sub types.Participant) {
var sd []rtcp.SourceDescriptionChunk
subTrack := t.getSubscribedTrack(sub.ID())
if subTrack == nil {
return
}
chunks := subTrack.DownTrack().CreateSourceDescriptionChunks()
if chunks == nil {
return
}
sd = append(sd, chunks...)
pkts := []rtcp.Packet{
&rtcp.SourceDescription{Chunks: sd},
}
go func() {
defer RecoverSilent()
batch := pkts
i := 0
for {
if err := sub.SubscriberPC().WriteRTCP(batch); err != nil {
t.params.Logger.Errorw("could not write RTCP", err)
return
}
if i > 5 {
return
}
i++
time.Sleep(20 * time.Millisecond)
}
}()
}
func (t *MediaTrack) handlePublisherFeedback(packets []rtcp.Packet) {
var maxLost uint8
var hasReport bool
for _, p := range packets {
switch pkt := p.(type) {
// sfu.Buffer generates ReceiverReports for the publisher
case *rtcp.ReceiverReport:
for _, rr := range pkt.Reports {
if rr.FractionLost > maxLost {
maxLost = rr.FractionLost
}
hasReport = true
}
}
}
if hasReport {
t.fracLostLock.Lock()
if maxLost > t.maxUpFracLost {
t.maxUpFracLost = maxLost
}
now := time.Now()
if now.Sub(t.maxUpFracLostTs) > lostUpdateDelta {
atomic.StoreUint32(&t.currentUpFracLost, uint32(t.maxUpFracLost))
t.maxUpFracLost = 0
t.maxUpFracLostTs = now
}
t.fracLostLock.Unlock()
}
// also look for sender reports
// feedback for the source RTCP
t.params.RTCPChan <- packets
}
// handles max loss for audio packets
func (t *MediaTrack) handleMaxLossFeedback(_ *sfu.DownTrack, report *rtcp.ReceiverReport) {
var (
shouldUpdate bool
maxLost uint8
)
t.fracLostLock.Lock()
for _, rr := range report.Reports {
if t.maxDownFracLost < rr.FractionLost {
t.maxDownFracLost = rr.FractionLost
}
}
now := time.Now()
if now.Sub(t.maxDownFracLostTs) > lostUpdateDelta {
shouldUpdate = true
maxLost = t.maxDownFracLost
t.maxDownFracLost = 0
t.maxDownFracLostTs = now
}
t.fracLostLock.Unlock()
if shouldUpdate && t.buffer != nil {
// ok to access buffer since receivers are added before subscribers
t.buffer.SetLastFractionLostReport(maxLost)
}
}
func (t *MediaTrack) DebugInfo() map[string]interface{} {
info := map[string]interface{}{
"ID": t.ID(),
"SSRC": t.ssrc,
"Kind": t.Kind().String(),
"PubMuted": t.muted.Get(),
}
subscribedTrackInfo := make([]map[string]interface{}, 0)
t.subscribedTracks.Range(func(_, val interface{}) bool {
if track, ok := val.(*SubscribedTrack); ok {
dt := track.dt.DebugInfo()
dt["PubMuted"] = track.pubMuted.Get()
dt["SubMuted"] = track.subMuted.Get()
subscribedTrackInfo = append(subscribedTrackInfo, dt)
}
return true
})
info["DownTracks"] = subscribedTrackInfo
if t.receiver != nil {
receiverInfo := t.receiver.DebugInfo()
for k, v := range receiverInfo {
info[k] = v
}
}
return info
}
func (t *MediaTrack) Receiver() sfu.TrackReceiver {
return t.receiver
}
func (t *MediaTrack) OnSubscribedMaxQualityChange(f func(trackSid string, subscribedQualities []*livekit.SubscribedQuality) error) {
t.onSubscribedMaxQualityChange = f
}
func (t *MediaTrack) NotifySubscriberMute(subscriberID string) {
if !t.IsSimulcast() {
return
}
t.maxQualityLock.Lock()
_, ok := t.maxSubscriberQuality[subscriberID]
if !ok {
t.maxQualityLock.Unlock()
return
}
delete(t.maxSubscriberQuality, subscriberID)
t.maxQualityLock.Unlock()
t.updateQualityChange()
}
func (t *MediaTrack) NotifySubscriberMaxQuality(subscriberID string, quality livekit.VideoQuality) {
if !t.IsSimulcast() {
return
}
t.maxQualityLock.Lock()
maxQuality, ok := t.maxSubscriberQuality[subscriberID]
if ok && maxQuality == quality {
t.maxQualityLock.Unlock()
return
}
t.maxSubscriberQuality[subscriberID] = quality
t.maxQualityLock.Unlock()
t.updateQualityChange()
}
func (t *MediaTrack) updateQualityChange() {
if t.IsMuted() {
return
}
var subscribedQualities []*livekit.SubscribedQuality
t.maxQualityLock.Lock()
allSubscribersMuted := false
maxSubscribedQuality := livekit.VideoQuality_LOW
if len(t.maxSubscriberQuality) == 0 {
allSubscribersMuted = true
} else {
for _, subQuality := range t.maxSubscriberQuality {
if subQuality > maxSubscribedQuality {
maxSubscribedQuality = subQuality
}
}
}
if allSubscribersMuted {
if !t.allSubscribersMuted {
t.allSubscribersMuted = true
subscribedQualities = []*livekit.SubscribedQuality{
&livekit.SubscribedQuality{Quality: livekit.VideoQuality_LOW, Enabled: false},
&livekit.SubscribedQuality{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
&livekit.SubscribedQuality{Quality: livekit.VideoQuality_HIGH, Enabled: false},
}
}
} else {
t.allSubscribersMuted = false
if maxSubscribedQuality != t.maxSubscribedQuality {
t.maxSubscribedQuality = maxSubscribedQuality
subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{Quality: livekit.VideoQuality_LOW, Enabled: true})
if t.maxSubscribedQuality == livekit.VideoQuality_LOW {
subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{Quality: livekit.VideoQuality_MEDIUM, Enabled: false})
} else {
subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{Quality: livekit.VideoQuality_MEDIUM, Enabled: true})
}
if t.maxSubscribedQuality != livekit.VideoQuality_HIGH {
subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{Quality: livekit.VideoQuality_HIGH, Enabled: false})
} else {
subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{Quality: livekit.VideoQuality_HIGH, Enabled: true})
}
}
}
t.maxQualityLock.Unlock()
if len(subscribedQualities) != 0 && t.onSubscribedMaxQualityChange != nil {
t.onSubscribedMaxQualityChange(t.ID(), subscribedQualities)
}
}