From 4404b6796bace15dd1e452c2032051a63809d8c2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 20 Feb 2024 10:32:35 +0530 Subject: [PATCH 01/73] Some optimisations in the forwarding path. (#2035) * WIP commit * WIP * Fix tests * clean up * Release pool in pacer * fix tests * fix tests * remove debug * fix test --- pkg/sfu/buffer/helpers.go | 13 +- pkg/sfu/codecmunger/codecmunger.go | 4 +- pkg/sfu/codecmunger/null.go | 8 +- pkg/sfu/codecmunger/vp8.go | 24 ++-- pkg/sfu/codecmunger/vp8_test.go | 54 +++++--- pkg/sfu/downtrack.go | 79 ++++------- pkg/sfu/forwarder.go | 76 +++++----- pkg/sfu/forwarder_test.go | 214 +++++++++++++++++------------ pkg/sfu/rtpmunger.go | 14 +- pkg/sfu/rtpmunger_test.go | 24 ++-- pkg/sfu/sequencer.go | 31 +++-- pkg/sfu/sequencer_test.go | 98 +++++++------ 12 files changed, 356 insertions(+), 283 deletions(-) diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index 5be302959..c6838ff4e 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -163,13 +163,16 @@ func (v *VP8) Unmarshal(payload []byte) error { func (v *VP8) Marshal() ([]byte, error) { buf := make([]byte, v.HeaderSize) - err := v.MarshalTo(buf) - return buf, err + n, err := v.MarshalTo(buf) + if err != nil { + return nil, err + } + return buf[:n], err } -func (v *VP8) MarshalTo(buf []byte) error { +func (v *VP8) MarshalTo(buf []byte) (int, error) { if len(buf) < v.HeaderSize { - return errShortPacket + return 0, errShortPacket } idx := 0 @@ -223,7 +226,7 @@ func (v *VP8) MarshalTo(buf []byte) error { idx++ } - return nil + return idx, nil } // ------------------------------------- diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go index 850cecb8d..94543ea38 100644 --- a/pkg/sfu/codecmunger/codecmunger.go +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -33,7 +33,7 @@ type CodecMunger interface { SetLast(extPkt *buffer.ExtPacket) UpdateOffsets(extPkt *buffer.ExtPacket) - UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) + UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) - UpdateAndGetPadding(newPicture bool) ([]byte, error) + UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) } diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index 32e3d9ee9..c845b4ce1 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -45,10 +45,10 @@ func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) { } -func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) { - return nil, nil +func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) { + return 0, 0, nil } -func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) { - return nil, nil +func (n *Null) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { + return 0, nil } diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index bb270387d..c26ea0e3c 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -158,10 +158,10 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } -func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) ([]byte, error) { +func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32, outputHeader []byte) (int, int, error) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { - return nil, ErrNotVP8 + return 0, 0, ErrNotVP8 } extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M) @@ -170,7 +170,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap if snOutOfOrder { pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId) if !ok { - return nil, ErrOutOfOrderVP8PictureIdCacheMiss + return 0, 0, ErrOutOfOrderVP8PictureIdCacheMiss } // the out-of-order picture id cannot be deleted from the cache @@ -195,7 +195,11 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return vp8Packet.Marshal() + n, err := vp8Packet.MarshalTo(outputHeader) + if err != nil { + return 0, 0, err + } + return vp8.HeaderSize, n, nil } prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() @@ -263,7 +267,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap v.pictureIdOffset += 1 } - return nil, ErrFilteredVP8TemporalLayer + return 0, 0, ErrFilteredVP8TemporalLayer } } } @@ -298,10 +302,14 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return vp8Packet.Marshal() + n, err := vp8Packet.MarshalTo(outputHeader) + if err != nil { + return 0, 0, err + } + return vp8.HeaderSize, n, nil } -func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { +func (v *VP8) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { offset := 0 if newPicture { offset = 1 @@ -359,7 +367,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { IsKeyFrame: true, HeaderSize: headerSize, } - return vp8Packet.Marshal() + return vp8Packet.MarshalTo(outputHeader) } // for testing only diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go index c72965189..155c7249c 100644 --- a/pkg/sfu/codecmunger/vp8_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -166,6 +166,7 @@ func TestUpdateOffsets(t *testing.T) { func TestOutOfOrderPictureId(t *testing.T) { v := newVP8() + buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -189,16 +190,17 @@ func TestOutOfOrderPictureId(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - v.UpdateAndGet(extPkt, false, false, 2) + v.UpdateAndGet(extPkt, false, false, 2, buf) // out-of-order sequence number not in the missing picture id cache vp8.PictureID = 13466 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - codecBytes, err := v.UpdateAndGet(extPkt, true, false, 2) + nIn, nOut, err := v.UpdateAndGet(extPkt, true, false, 2, buf) require.Error(t, err) require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) - require.Nil(t, codecBytes) + require.Equal(t, 0, nIn) + require.Equal(t, 0, nOut) // create a hole in picture id vp8.PictureID = 13469 @@ -221,9 +223,10 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) require.NoError(t, err) - require.Equal(t, marshalledVP8, codecBytes) + require.Equal(t, 6, nIn) + require.Equal(t, marshalledVP8, buf[:nOut]) // all three, the last, the current and the in-between should have been added to missing picture id cache value, ok := v.PictureIdOffset(13467) @@ -259,13 +262,15 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - codecBytes, err = v.UpdateAndGet(extPkt, true, false, 2) + nIn, nOut, err = v.UpdateAndGet(extPkt, true, false, 2, buf) require.NoError(t, err) - require.Equal(t, marshalledVP8, codecBytes) + require.Equal(t, 6, nIn) + require.Equal(t, marshalledVP8, buf[:nOut]) } func TestTemporalLayerFiltering(t *testing.T) { v := newVP8() + buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -291,10 +296,11 @@ func TestTemporalLayerFiltering(t *testing.T) { v.SetLast(extPkt) // translate - tp, err := v.UpdateAndGet(extPkt, false, false, 0) + nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 0, buf) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) + require.Equal(t, 0, nIn) + require.Equal(t, 0, nOut) dropped, _ := v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -304,10 +310,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23334 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tp, err = v.UpdateAndGet(extPkt, false, false, 0) + nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) + require.Equal(t, 0, nIn) + require.Equal(t, 0, nOut) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -317,10 +324,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23337 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tp, err = v.UpdateAndGet(extPkt, false, false, 0) + nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) + require.Equal(t, 0, nIn) + require.Equal(t, 0, nOut) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -328,6 +336,7 @@ func TestTemporalLayerFiltering(t *testing.T) { func TestGapInSequenceNumberSamePicture(t *testing.T) { v := newVP8() + buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 65533, @@ -370,9 +379,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - codecBytes, err := v.UpdateAndGet(extPkt, false, false, 2) + nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 2, buf) require.NoError(t, err) - require.Equal(t, marshalledVP8, codecBytes) + require.Equal(t, 6, nIn) + require.Equal(t, marshalledVP8, buf[:nOut]) // telling there is a gap in sequence number will add pictures to missing picture cache expectedVP8 = &buffer.VP8{ @@ -392,9 +402,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) require.NoError(t, err) - require.Equal(t, marshalledVP8, codecBytes) + require.Equal(t, 6, nIn) + require.Equal(t, marshalledVP8, buf[:nOut]) value, ok := v.PictureIdOffset(13467) require.True(t, ok) @@ -403,6 +414,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { func TestUpdateAndGetPadding(t *testing.T) { v := newVP8() + buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -430,7 +442,7 @@ func TestUpdateAndGetPadding(t *testing.T) { v.SetLast(extPkt) // getting padding with repeat of last picture - blankBytes, err := v.UpdateAndGetPadding(false) + n, err := v.UpdateAndGetPadding(false, buf) require.NoError(t, err) expectedVP8 := buffer.VP8{ FirstByte: 16, @@ -449,10 +461,10 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, blankBytes) + require.Equal(t, marshalledVP8, buf[:n]) // getting padding with new picture - blankBytes, err = v.UpdateAndGetPadding(true) + n, err = v.UpdateAndGetPadding(true, buf) require.NoError(t, err) expectedVP8 = buffer.VP8{ FirstByte: 16, @@ -471,7 +483,7 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, blankBytes) + require.Equal(t, marshalledVP8, buf[:n]) } func TestVP8PictureIdWrapHandler(t *testing.T) { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 6d42e755b..93ef0c1a2 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -687,25 +687,20 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } - var payload []byte poolEntity := PacketFactory.Get().(*[]byte) - if len(tp.codecBytes) != 0 { - incomingVP8, ok := extPkt.Payload.(buffer.VP8) - if ok { - payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, poolEntity) - } - } - if payload == nil { - payload = (*poolEntity)[:len(extPkt.Packet.Payload)] - copy(payload, extPkt.Packet.Payload) + payload := *poolEntity + shouldForward, incomingHeaderSize, outgoingHeaderSize, err := d.forwarder.TranslateCodecHeader(extPkt, &tp.rtp, payload) + if !shouldForward { + PacketFactory.Put(poolEntity) + return err } + copy(payload[outgoingHeaderSize:], extPkt.Packet.Payload[incomingHeaderSize:]) + payload = payload[:outgoingHeaderSize+len(extPkt.Packet.Payload)-incomingHeaderSize] - hdr, err := d.getTranslatedRTPHeader(extPkt, tp) + hdr, err := d.getTranslatedRTPHeader(extPkt, &tp) if err != nil { d.params.Logger.Errorw("could not get translated RTP header", err) - if poolEntity != nil { - PacketFactory.Put(poolEntity) - } + PacketFactory.Put(poolEntity) return err } @@ -726,7 +721,8 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { tp.rtp.extTimestamp, hdr.Marker, int8(layer), - tp.codecBytes, + payload[:outgoingHeaderSize], + incomingHeaderSize, tp.ddBytes, ) } @@ -740,7 +736,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { extSequenceNumber: tp.rtp.extSequenceNumber, extTimestamp: tp.rtp.extTimestamp, isKeyFrame: extPkt.KeyFrame, - tp: tp, + tp: &tp, }, ) d.pacer.Enqueue(pacer.Packet{ @@ -1434,20 +1430,19 @@ func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { } func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { - blankVP8, err := d.forwarder.GetPadding(frameEndNeeded) - if err != nil { - return nil, err - } - // 8x8 key frame // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) payload := make([]byte, 1000) - copy(payload[:len(blankVP8)], blankVP8) - copy(payload[len(blankVP8):], VP8KeyFrame8x8) - trailerLen := d.maybeAddTrailer(payload[len(blankVP8)+len(VP8KeyFrame8x8):]) - return payload[:len(blankVP8)+len(VP8KeyFrame8x8)+trailerLen], nil + n, err := d.forwarder.GetPadding(frameEndNeeded, payload) + if err != nil { + return nil, err + } + + copy(payload[n:], VP8KeyFrame8x8) + trailerLen := d.maybeAddTrailer(payload[n+len(VP8KeyFrame8x8):]) + return payload[:n+len(VP8KeyFrame8x8)+trailerLen], nil } func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { @@ -1663,22 +1658,11 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { pkt.Header.SSRC = d.ssrc pkt.Header.PayloadType = d.payloadType - var payload []byte poolEntity := PacketFactory.Get().(*[]byte) - if d.mime == "video/vp8" && len(pkt.Payload) > 0 && len(epm.codecBytes) != 0 { - var incomingVP8 buffer.VP8 - if err = incomingVP8.Unmarshal(pkt.Payload); err != nil { - d.params.Logger.Errorw("could not unmarshal VP8 packet", err) - PacketFactory.Put(poolEntity) - continue - } - - payload = d.translateVP8PacketTo(&pkt, &incomingVP8, epm.codecBytes, poolEntity) - } - if payload == nil { - payload = (*poolEntity)[:len(pkt.Payload)] - copy(payload, pkt.Payload) - } + payload := *poolEntity + copy(payload, epm.codecBytes[:epm.numCodecBytesOut]) + copy(payload[epm.numCodecBytesOut:], pkt.Payload[epm.numCodecBytesIn:]) + payload = payload[:int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] d.sendingPacket( &pkt.Header, @@ -1724,11 +1708,10 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { } func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (*rtp.Header, error) { - tpRTP := tp.rtp hdr := extPkt.Packet.Header hdr.PayloadType = d.payloadType - hdr.Timestamp = uint32(tpRTP.extTimestamp) - hdr.SequenceNumber = uint16(tpRTP.extSequenceNumber) + hdr.Timestamp = uint32(tp.rtp.extTimestamp) + hdr.SequenceNumber = uint16(tp.rtp.extSequenceNumber) hdr.SSRC = d.ssrc if tp.marker { hdr.Marker = tp.marker @@ -1737,16 +1720,6 @@ func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *Transla return &hdr, nil } -func (d *DownTrack) translateVP8PacketTo(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 []byte, outbuf *[]byte) []byte { - buf := (*outbuf)[:len(pkt.Payload)+len(translatedVP8)-incomingVP8.HeaderSize] - srcPayload := pkt.Payload[incomingVP8.HeaderSize:] - dstPayload := buf[len(translatedVP8):] - copy(dstPayload, srcPayload) - - copy(buf[:len(translatedVP8)], translatedVP8) - return buf -} - func (d *DownTrack) DebugInfo() map[string]interface{} { stats := map[string]interface{}{ "LastPli": d.rtpStats.LastPli(), diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 86fe39967..6d7c1d91a 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -147,8 +147,7 @@ type TranslationParams struct { shouldDrop bool isResuming bool isSwitching bool - rtp *TranslationParamsRTP - codecBytes []byte + rtp TranslationParamsRTP ddBytes []byte marker bool } @@ -1433,12 +1432,12 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye return } -func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { +func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { f.lock.Lock() defer f.lock.Unlock() if f.muted || f.pubMuted { - return &TranslationParams{ + return TranslationParams{ shouldDrop: true, }, nil } @@ -1450,7 +1449,9 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) return f.getTranslationParamsVideo(extPkt, layer) } - return nil, ErrUnknownKind + return TranslationParams{ + shouldDrop: true, + }, ErrUnknownKind } func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { @@ -1647,11 +1648,11 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } // should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (*TranslationParams, error) { +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error { if f.lastSSRC != extPkt.Packet.SSRC { if err := f.processSourceSwitch(extPkt, layer); err != nil { tp.shouldDrop = true - return tp, nil + return nil } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC @@ -1661,29 +1662,34 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - return tp, nil + return nil } - return tp, err + return err } tp.rtp = tpRTP + return nil +} + +// should be called with lock held +func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { + tp := TranslationParams{} + if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { + tp.shouldDrop = true + return tp, err + } return tp, nil } // should be called with lock held -func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { - return f.getTranslationParamsCommon(extPkt, layer, &TranslationParams{}) -} - -// should be called with lock held -func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { +func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { maybeRollback := func(isSwitching bool) { if isSwitching { f.vls.Rollback() } } - tp := &TranslationParams{} + tp := TranslationParams{} if !f.vls.GetTarget().IsValid() { // stream is paused by streamallocator tp.shouldDrop = true @@ -1746,38 +1752,46 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in return tp, nil } - _, err := f.getTranslationParamsCommon(extPkt, layer, tp) - if tp.shouldDrop || len(extPkt.Packet.Payload) == 0 { + err := f.getTranslationParamsCommon(extPkt, layer, &tp) + if tp.shouldDrop { maybeRollback(result.IsSwitching) return tp, err } + return tp, nil +} + +func (f *Forwarder) TranslateCodecHeader(extPkt *buffer.ExtPacket, tpr *TranslationParamsRTP, outputBuffer []byte) (bool, int, int, error) { + maybeRollback := func(isSwitching bool) { + if isSwitching { + f.vls.Rollback() + } + } + // codec specific forwarding check and any needed packet munging tl, isSwitching := f.vls.SelectTemporal(extPkt) - codecBytes, err := f.codecMunger.UpdateAndGet( + inputSize, outputSize, err := f.codecMunger.UpdateAndGet( extPkt, - tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, - tp.rtp.snOrdering == SequenceNumberOrderingGap, + tpr.snOrdering == SequenceNumberOrderingOutOfOrder, + tpr.snOrdering == SequenceNumberOrderingGap, tl, + outputBuffer, ) if err != nil { - tp.rtp = nil - tp.shouldDrop = true if err == codecmunger.ErrFilteredVP8TemporalLayer || err == codecmunger.ErrOutOfOrderVP8PictureIdCacheMiss { if err == codecmunger.ErrFilteredVP8TemporalLayer { // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - maybeRollback(result.IsSwitching || isSwitching) - return tp, nil + maybeRollback(isSwitching) + return false, 0, 0, nil } - maybeRollback(result.IsSwitching || isSwitching) - return tp, err + maybeRollback(isSwitching) + return false, 0, 0, err } - tp.codecBytes = codecBytes - return tp, nil + return true, inputSize, outputSize, nil } func (f *Forwarder) maybeStart() { @@ -1854,11 +1868,11 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S return snts, frameEndNeeded, err } -func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { +func (f *Forwarder) GetPadding(frameEndNeeded bool, outputBuffer []byte) (int, error) { f.lock.Lock() defer f.lock.Unlock() - return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) + return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded, outputBuffer) } func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 5e58e3e41..aaecb03b8 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1200,7 +1200,7 @@ func TestForwarderGetTranslationParamsMuted(t *testing.T) { } actualTP, err := f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) } func TestForwarderGetTranslationParamsAudio(t *testing.T) { @@ -1216,7 +1216,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { // should lock onto the first packet expectedTP := TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23333, extTimestamp: 0xabcdef, @@ -1224,7 +1224,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err := f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) require.True(t, f.started) require.Equal(t, f.lastSSRC, params.SSRC) @@ -1234,7 +1234,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // add a missing sequence number to the cache err = f.rtpMunger.snRangeMap.ExcludeRange(23334, 23335) @@ -1261,7 +1261,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, extSequenceNumber: 23334, extTimestamp: 0xabcdef, @@ -1269,7 +1269,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // padding only packet in order should be dropped params = &testutils.TestExtPacketParams{ @@ -1284,7 +1284,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // in order packet should be forwarded params = &testutils.TestExtPacketParams{ @@ -1296,7 +1296,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23336, extTimestamp: 0xabcdef, @@ -1304,7 +1304,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // padding only packet after a gap should not be dropped params = &testutils.TestExtPacketParams{ @@ -1315,7 +1315,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingGap, extSequenceNumber: 23338, extTimestamp: 0xabcdef, @@ -1323,7 +1323,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // out-of-order should be forwarded using cache params = &testutils.TestExtPacketParams{ @@ -1335,7 +1335,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, extSequenceNumber: 23335, extTimestamp: 0xabcdef, @@ -1343,7 +1343,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // switching source should lock onto the new source, but sequence number should be contiguous params = &testutils.TestExtPacketParams{ @@ -1355,7 +1355,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23339, extTimestamp: 0xabcdf0, @@ -1363,11 +1363,12 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) require.Equal(t, f.lastSSRC, params.SSRC) } func TestForwarderGetTranslationParamsVideo(t *testing.T) { + buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -1400,7 +1401,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err := f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // although target layer matches, not a key frame, so should drop f.vls.SetTarget(buffer.VideoLayer{ @@ -1412,7 +1413,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // should lock onto packet (key frame) vp8 = &buffer.VP8{ @@ -1431,6 +1432,22 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedTP = TranslationParams{ + isSwitching: true, + isResuming: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23333, + extTimestamp: 0xabcdef, + }, + marker: true, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + require.True(t, f.started) + require.Equal(t, f.lastSSRC, params.SSRC) + expectedVP8 := &buffer.VP8{ FirstByte: 25, I: true, @@ -1448,22 +1465,12 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - expectedTP = TranslationParams{ - isSwitching: true, - isResuming: true, - rtp: &TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23333, - extTimestamp: 0xabcdef, - }, - codecBytes: marshalledVP8, - marker: true, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) + shouldForward, incomingHeaderSize, outgoingHeaderSize, err := f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) - require.True(t, f.started) - require.Equal(t, f.lastSSRC, params.SSRC) + require.True(t, shouldForward) + require.Equal(t, 6, incomingHeaderSize) + require.Equal(t, 6, outgoingHeaderSize) + require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) // send a duplicate, should be dropped expectedTP = TranslationParams{ @@ -1472,7 +1479,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // out-of-order packet not in cache should be dropped params = &testutils.TestExtPacketParams{ @@ -1487,7 +1494,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // padding only packet in order should be dropped params = &testutils.TestExtPacketParams{ @@ -1501,7 +1508,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // in order packet should be forwarded params = &testutils.TestExtPacketParams{ @@ -1511,6 +1518,17 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23334, + extTimestamp: 0xabcdef, + }, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1528,17 +1546,12 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23334, - extTimestamp: 0xabcdef, - }, - codecBytes: marshalledVP8, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) + shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.True(t, shouldForward) + require.Equal(t, 6, incomingHeaderSize) + require.Equal(t, 6, outgoingHeaderSize) + require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) // temporal layer matching target, should be forwarded params = &testutils.TestExtPacketParams{ @@ -1564,6 +1577,17 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23335, + extTimestamp: 0xabcdef, + }, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1581,17 +1605,12 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23335, - extTimestamp: 0xabcdef, - }, - codecBytes: marshalledVP8, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) + shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.True(t, shouldForward) + require.Equal(t, 6, incomingHeaderSize) + require.Equal(t, 6, outgoingHeaderSize) + require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) // temporal layer higher than target, should be dropped params = &testutils.TestExtPacketParams{ @@ -1617,11 +1636,19 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ - shouldDrop: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23336, + extTimestamp: 0xabcdef, + }, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) + + shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + require.NoError(t, err) + require.False(t, shouldForward) // RTP sequence number and VP8 picture id should be contiguous after dropping higher temporal layer picture params = &testutils.TestExtPacketParams{ @@ -1646,6 +1673,17 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23336, + extTimestamp: 0xabcdef, + }, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1663,17 +1701,12 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23336, - extTimestamp: 0xabcdef, - }, - codecBytes: marshalledVP8, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) + shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.True(t, shouldForward) + require.Equal(t, 6, incomingHeaderSize) + require.Equal(t, 6, outgoingHeaderSize) + require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) // padding only packet after a gap should be forwarded params = &testutils.TestExtPacketParams{ @@ -1684,7 +1717,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingGap, extSequenceNumber: 23338, extTimestamp: 0xabcdef, @@ -1692,7 +1725,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // out-of-order should be forwarded using cache, even if it is padding only params = &testutils.TestExtPacketParams{ @@ -1703,7 +1736,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - rtp: &TranslationParamsRTP{ + rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, extSequenceNumber: 23337, extTimestamp: 0xabcdef, @@ -1711,7 +1744,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) + require.Equal(t, expectedTP, actualTP) // switching SSRC (happens for new layer or new track source) // should lock onto the new source, but sequence number should be contiguous @@ -1743,6 +1776,19 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedTP = TranslationParams{ + isSwitching: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23339, + extTimestamp: 0xabcdf0, + }, + } + actualTP, err = f.GetTranslationParams(extPkt, 1) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + require.Equal(t, f.lastSSRC, params.SSRC) + expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1760,19 +1806,12 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - expectedTP = TranslationParams{ - isSwitching: true, - rtp: &TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23339, - extTimestamp: 0xabcdf0, - }, - codecBytes: marshalledVP8, - } - actualTP, err = f.GetTranslationParams(extPkt, 1) + shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) require.NoError(t, err) - require.Equal(t, expectedTP, *actualTP) - require.Equal(t, f.lastSSRC, params.SSRC) + require.True(t, shouldForward) + require.Equal(t, 5, incomingHeaderSize) + require.Equal(t, 6, outgoingHeaderSize) + require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) } func TestForwarderGetSnTsForPadding(t *testing.T) { @@ -1920,6 +1959,7 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) { } func TestForwarderGetPaddingVP8(t *testing.T) { + buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -1970,11 +2010,11 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - blankVP8, err := f.GetPadding(true) + n, err := f.GetPadding(true, buf) require.NoError(t, err) marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, blankVP8) + require.Equal(t, marshalledVP8, buf[:n]) // getting padding with no frame end needed, should get next picture id expectedVP8 = buffer.VP8{ @@ -1992,9 +2032,9 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - blankVP8, err = f.GetPadding(false) + n, err = f.GetPadding(false, buf) require.NoError(t, err) marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, blankVP8) + require.Equal(t, marshalledVP8, buf[:n]) } diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 467ec2b3f..9ff94ea1a 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -180,7 +180,7 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { r.lastMarker = r.secondLastMarker } -func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (*TranslationParamsRTP, error) { +func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (TranslationParamsRTP, error) { diff := int64(extPkt.ExtSequenceNumber - r.extHighestIncomingSN) if (diff == 1 && len(extPkt.Packet.Payload) != 0) || diff > 1 { // in-order - either contiguous packet with payload OR packet following a gap, may or may not have payload @@ -210,7 +210,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (*Tr r.isInRtxGateRegion = false } - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: ordering, extSequenceNumber: extMungedSN, extTimestamp: extMungedTS, @@ -221,7 +221,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (*Tr // out-of-order, look up sequence number offset cache snOffset, err := r.snRangeMap.GetValue(extPkt.ExtSequenceNumber) if err != nil { - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, }, ErrOutOfOrderSequenceNumberCacheMiss } @@ -237,12 +237,12 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (*Tr "snOffsetIncoming", snOffset, "snOffsetHighest", r.snOffset, ) - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, }, ErrOutOfOrderSequenceNumberCacheMiss } - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, extSequenceNumber: extSequenceNumber, extTimestamp: extPkt.ExtTimestamp - r.tsOffset, @@ -259,13 +259,13 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (*Tr r.updateSnOffset() - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, }, ErrPaddingOnlyPacket } // can get duplicate packet due to FEC - return &TranslationParamsRTP{ + return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingDuplicate, }, ErrDuplicatePacket } diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index 128d37b54..08e51a128 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -213,7 +213,7 @@ func TestOutOfOrderSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) params = &testutils.TestExtPacketParams{ SequenceNumber: 23332, @@ -229,7 +229,7 @@ func TestOutOfOrderSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.Error(t, err, ErrOutOfOrderSequenceNumberCacheMiss) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) } func TestDuplicateSequenceNumber(t *testing.T) { @@ -253,7 +253,7 @@ func TestDuplicateSequenceNumber(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.ErrorIs(t, err, ErrDuplicatePacket) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) } func TestPaddingOnlyPacket(t *testing.T) { @@ -275,7 +275,7 @@ func TestPaddingOnlyPacket(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.Error(t, err) require.ErrorIs(t, err, ErrPaddingOnlyPacket) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(23333), r.extHighestIncomingSN) require.Equal(t, uint64(23333), r.extLastSN) snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -297,7 +297,7 @@ func TestPaddingOnlyPacket(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(23335), r.extHighestIncomingSN) require.Equal(t, uint64(23334), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -338,7 +338,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+1), r.extHighestIncomingSN) require.Equal(t, uint64(65536+1), r.extLastSN) snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -367,7 +367,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.ErrorIs(t, err, ErrPaddingOnlyPacket) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+2), r.extHighestIncomingSN) require.Equal(t, uint64(65536+1), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -391,7 +391,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+4), r.extHighestIncomingSN) require.Equal(t, uint64(65536+3), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -418,7 +418,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.ErrorIs(t, err, ErrPaddingOnlyPacket) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+5), r.extHighestIncomingSN) require.Equal(t, uint64(65536+3), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -442,7 +442,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+7), r.extHighestIncomingSN) require.Equal(t, uint64(65536+5), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -475,7 +475,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+7), r.extHighestIncomingSN) require.Equal(t, uint64(65536+5), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) @@ -498,7 +498,7 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+7), r.extHighestIncomingSN) require.Equal(t, uint64(65536+5), r.extLastSN) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 4e26a8735..4eee083a0 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -67,7 +67,9 @@ type packetMeta struct { // Spatial layer of packet layer int8 // Information that differs depending on the codec - codecBytes []byte + codecBytes [8]byte + numCodecBytesIn uint8 + numCodecBytesOut uint8 // Dependency Descriptor of packet ddBytes []byte } @@ -127,6 +129,7 @@ func (s *sequencer) push( marker bool, layer int8, codecBytes []byte, + numCodecBytesIn int, ddBytes []byte, ) { s.Lock() @@ -190,15 +193,23 @@ func (s *sequencer) push( slot := extModifiedSNAdjusted % uint64(s.size) s.meta[slot] = packetMeta{ - sourceSeqNo: uint16(extIncomingSN), - targetSeqNo: uint16(extModifiedSN), - timestamp: uint32(extModifiedTS), - marker: marker, - layer: layer, - codecBytes: append([]byte{}, codecBytes...), - ddBytes: append([]byte{}, ddBytes...), - lastNack: s.getRefTime(packetTime), // delay retransmissions after the original transmission + sourceSeqNo: uint16(extIncomingSN), + targetSeqNo: uint16(extModifiedSN), + timestamp: uint32(extModifiedTS), + marker: marker, + layer: layer, + numCodecBytesIn: uint8(numCodecBytesIn), + ddBytes: append([]byte{}, ddBytes...), + lastNack: s.getRefTime(packetTime), // delay retransmissions after the original transmission } + pm := &s.meta[slot] + pm.numCodecBytesOut = uint8(len(codecBytes)) + if pm.numCodecBytesOut > uint8(len(pm.codecBytes)) { + s.logger.Errorw("codec bytes too large", nil, "need", pm.numCodecBytesOut, "bufSize", len(pm.codecBytes)) + s.invalidateSlot(int(slot)) + return + } + copy(pm.codecBytes[:pm.numCodecBytesOut], codecBytes) if extModifiedSN > s.extHighestSN { s.extHighestSN = extModifiedSN @@ -322,7 +333,7 @@ func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta { extSequenceNumber: extSN, extTimestamp: extTS, } - epm.codecBytes = append([]byte{}, meta.codecBytes...) + copy(epm.codecBytes[:epm.numCodecBytesOut], meta.codecBytes[:epm.numCodecBytesOut]) epm.ddBytes = append([]byte{}, meta.ddBytes...) extPacketMetas = append(extPacketMetas, epm) } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index c71bd039f..595acf02c 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -29,11 +29,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint64(1); i < 518; i++ { - seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, nil) + seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, 0, nil) } // send the last two out-of-order - seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, nil) - seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, nil) + seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil) + seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} res := seq.getExtPacketMetas(req) @@ -63,14 +63,14 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.extTimestamp, uint64(123)) } - seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, nil) + seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil) m := seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) m = seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, nil) + seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil) m = seq.getExtPacketMetas([]uint16{505 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) @@ -87,14 +87,16 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { isPadding bool } type fields struct { - inputs []input - offset uint64 - markerOdd bool - markerEven bool - codecBytesOdd []byte - codecBytesEven []byte - ddBytesOdd []byte - ddBytesEven []byte + inputs []input + offset uint64 + markerOdd bool + markerEven bool + codecBytesOdd []byte + numCodecBytesInOdd int + codecBytesEven []byte + numCodecBytesInEven int + ddBytesOdd []byte + ddBytesEven []byte } tests := []struct { @@ -117,13 +119,15 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { {65532, true}, {65534, false}, }, - offset: 5, - markerOdd: true, - markerEven: false, - codecBytesOdd: []byte{1, 2, 3, 4}, - codecBytesEven: []byte{5, 6, 7}, - ddBytesOdd: []byte{8, 9, 10}, - ddBytesEven: []byte{11, 12}, + offset: 5, + markerOdd: true, + markerEven: false, + codecBytesOdd: []byte{1, 2, 3, 4}, + numCodecBytesInOdd: 3, + codecBytesEven: []byte{5, 6, 7}, + numCodecBytesInEven: 4, + ddBytesOdd: []byte{8, 9, 10}, + ddBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */}, @@ -144,9 +148,9 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset) } else { if i.seqNo%2 == 0 { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) + n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven) } else { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) + n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd) } } } @@ -158,11 +162,13 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { got = append(got, sn.sourceSeqNo) if sn.sourceSeqNo%2 == 0 { require.Equal(t, tt.fields.markerEven, sn.marker) - require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes) + require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) - require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes) + require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) } } @@ -182,14 +188,16 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { isPadding bool } type fields struct { - inputs []input - offset uint64 - markerOdd bool - markerEven bool - codecBytesOdd []byte - codecBytesEven []byte - ddBytesOdd []byte - ddBytesEven []byte + inputs []input + offset uint64 + markerOdd bool + markerEven bool + codecBytesOdd []byte + numCodecBytesInOdd int + codecBytesEven []byte + numCodecBytesInEven int + ddBytesOdd []byte + ddBytesEven []byte } tests := []struct { @@ -213,13 +221,15 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { {12, false}, {13, false}, }, - offset: 5, - markerOdd: true, - markerEven: false, - codecBytesOdd: []byte{1, 2, 3, 4}, - codecBytesEven: []byte{5, 6, 7}, - ddBytesOdd: []byte{8, 9, 10}, - ddBytesEven: []byte{11, 12}, + offset: 5, + markerOdd: true, + markerEven: false, + codecBytesOdd: []byte{1, 2, 3, 4}, + numCodecBytesInOdd: 3, + codecBytesEven: []byte{5, 6, 7}, + numCodecBytesInEven: 4, + ddBytesOdd: []byte{8, 9, 10}, + ddBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5, 12 + 5}, @@ -238,9 +248,9 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset) } else { if i.seqNo%2 == 0 { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) + n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven) } else { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) + n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd) } } } @@ -252,11 +262,13 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { got = append(got, sn.sourceSeqNo) if sn.sourceSeqNo%2 == 0 { require.Equal(t, tt.fields.markerEven, sn.marker) - require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes) + require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) - require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes) + require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) } } From 628b7c702455bc9d18416a5ba438997231c0c463 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 20 Feb 2024 17:06:34 +0800 Subject: [PATCH 02/73] Fix nil bufferfactory in test (#2495) --- pkg/rtc/participant_internal_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index a1b758ff8..5f5f8eac6 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -25,6 +25,7 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -730,6 +731,8 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p if err != nil { panic(err) } + ff := buffer.NewFactoryOfBufferFactory(500) + rtcConf.SetBufferFactory(ff.CreateBufferFactory()) grants := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } From 0088eef3c1f8a7dc1bc393cb81fa86c0dece2a7f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 20 Feb 2024 15:30:21 +0530 Subject: [PATCH 03/73] Reduce heap for dependency descriptor in forwarding path. (#2496) * Reduce heap for dependency descriptor in forwarding path. Marshaled dependency descriptor is held in sequencer adding heap objcts. Store DD bytes in sequencer to avoid heap usage. Also, accomodating over sized objects via storing in slice and using it in case the bytes do not fit in the internal array. NOTE: Marshal DD still does a make([]byte...), but I think it should be on the stack given the short use of it. Have to verify. * fix test and also add cases for oversized codec/dd bytes --- pkg/sfu/downtrack.go | 21 ++++++-- pkg/sfu/sequencer.go | 27 ++++++---- pkg/sfu/sequencer_test.go | 102 ++++++++++++++++++++++++++++++++------ 3 files changed, 121 insertions(+), 29 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 93ef0c1a2..a4df99776 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1660,9 +1660,22 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { poolEntity := PacketFactory.Get().(*[]byte) payload := *poolEntity - copy(payload, epm.codecBytes[:epm.numCodecBytesOut]) - copy(payload[epm.numCodecBytesOut:], pkt.Payload[epm.numCodecBytesIn:]) - payload = payload[:int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] + if len(epm.codecBytesSlice) != 0 { + n := copy(payload, epm.codecBytesSlice) + m := copy(payload[n:], pkt.Payload[epm.numCodecBytesIn:]) + payload = payload[:n+m] + } else { + copy(payload, epm.codecBytes[:epm.numCodecBytesOut]) + copy(payload[epm.numCodecBytesOut:], pkt.Payload[epm.numCodecBytesIn:]) + payload = payload[:int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] + } + + var ddBytes []byte + if len(epm.ddBytesSlice) != 0 { + ddBytes = epm.ddBytesSlice + } else { + ddBytes = epm.ddBytes[:epm.ddBytesSize] + } d.sendingPacket( &pkt.Header, @@ -1677,7 +1690,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { ) d.pacer.Enqueue(pacer.Packet{ Header: &pkt.Header, - Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: epm.ddBytes}}, + Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: ddBytes}}, Payload: payload, AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 4eee083a0..0fafcfc81 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -70,8 +70,11 @@ type packetMeta struct { codecBytes [8]byte numCodecBytesIn uint8 numCodecBytesOut uint8 + codecBytesSlice []byte // Dependency Descriptor of packet - ddBytes []byte + ddBytes [8]byte + ddBytesSize uint8 + ddBytesSlice []byte } type extPacketMeta struct { @@ -199,17 +202,23 @@ func (s *sequencer) push( marker: marker, layer: layer, numCodecBytesIn: uint8(numCodecBytesIn), - ddBytes: append([]byte{}, ddBytes...), lastNack: s.getRefTime(packetTime), // delay retransmissions after the original transmission } pm := &s.meta[slot] + pm.numCodecBytesOut = uint8(len(codecBytes)) - if pm.numCodecBytesOut > uint8(len(pm.codecBytes)) { - s.logger.Errorw("codec bytes too large", nil, "need", pm.numCodecBytesOut, "bufSize", len(pm.codecBytes)) - s.invalidateSlot(int(slot)) - return + if len(codecBytes) > len(pm.codecBytes) { + pm.codecBytesSlice = append([]byte{}, codecBytes...) + } else { + copy(pm.codecBytes[:pm.numCodecBytesOut], codecBytes) + } + + pm.ddBytesSize = uint8(len(ddBytes)) + if len(ddBytes) > len(pm.ddBytes) { + pm.ddBytesSlice = append([]byte{}, ddBytes...) + } else { + copy(pm.ddBytes[:pm.ddBytesSize], ddBytes) } - copy(pm.codecBytes[:pm.numCodecBytesOut], codecBytes) if extModifiedSN > s.extHighestSN { s.extHighestSN = extModifiedSN @@ -333,8 +342,8 @@ func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta { extSequenceNumber: extSN, extTimestamp: extTS, } - copy(epm.codecBytes[:epm.numCodecBytesOut], meta.codecBytes[:epm.numCodecBytesOut]) - epm.ddBytes = append([]byte{}, meta.ddBytes...) + epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...) + epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...) extPacketMetas = append(extPacketMetas, epm) } } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 595acf02c..39dcec906 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -95,8 +95,10 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { numCodecBytesInOdd int codecBytesEven []byte numCodecBytesInEven int + codecBytesOversized []byte ddBytesOdd []byte ddBytesEven []byte + ddBytesOversized []byte } tests := []struct { @@ -126,8 +128,10 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { numCodecBytesInOdd: 3, codecBytesEven: []byte{5, 6, 7}, numCodecBytesInEven: 4, + codecBytesOversized: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, ddBytesOdd: []byte{8, 9, 10}, ddBytesEven: []byte{11, 12}, + ddBytesOversized: []byte{11, 12, 13, 14, 15, 16, 17, 18, 19}, }, args: args{ seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */}, @@ -147,10 +151,44 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { if i.isPadding { n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset) } else { - if i.seqNo%2 == 0 { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven) + if i.seqNo%5 == 0 { + n.push( + time.Now(), + i.seqNo, + i.seqNo+tt.fields.offset, + 123, + tt.fields.markerOdd, + 3, + tt.fields.codecBytesOversized, + len(tt.fields.codecBytesOversized), + tt.fields.ddBytesOversized, + ) } else { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd) + if i.seqNo%2 == 0 { + n.push( + time.Now(), + i.seqNo, + i.seqNo+tt.fields.offset, + 123, + tt.fields.markerEven, + 3, + tt.fields.codecBytesEven, + tt.fields.numCodecBytesInEven, + tt.fields.ddBytesEven, + ) + } else { + n.push( + time.Now(), + i.seqNo, + i.seqNo+tt.fields.offset, + 123, + tt.fields.markerOdd, + 3, + tt.fields.codecBytesOdd, + tt.fields.numCodecBytesInOdd, + tt.fields.ddBytesOdd, + ) + } } } } @@ -160,16 +198,26 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { var got []uint16 for _, sn := range g { got = append(got, sn.sourceSeqNo) - if sn.sourceSeqNo%2 == 0 { - require.Equal(t, tt.fields.markerEven, sn.marker) - require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut]) - require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) - require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) - } else { + if sn.sourceSeqNo%5 == 0 { require.Equal(t, tt.fields.markerOdd, sn.marker) - require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) - require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) - require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) + require.Equal(t, tt.fields.codecBytesOversized, sn.codecBytesSlice) + require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.numCodecBytesIn) + require.Equal(t, tt.fields.ddBytesOversized, sn.ddBytesSlice) + require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.ddBytesSize) + } else { + if sn.sourceSeqNo%2 == 0 { + require.Equal(t, tt.fields.markerEven, sn.marker) + require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) + require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) + require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) + } else { + require.Equal(t, tt.fields.markerOdd, sn.marker) + require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) + require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) + require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) + require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) + } } } if !reflect.DeepEqual(got, tt.want) { @@ -248,9 +296,29 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset) } else { if i.seqNo%2 == 0 { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven) + n.push( + time.Now(), + i.seqNo, + i.seqNo+tt.fields.offset, + 123, + tt.fields.markerEven, + 3, + tt.fields.codecBytesEven, + tt.fields.numCodecBytesInEven, + tt.fields.ddBytesEven, + ) } else { - n.push(time.Now(), i.seqNo, i.seqNo+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd) + n.push( + time.Now(), + i.seqNo, + i.seqNo+tt.fields.offset, + 123, + tt.fields.markerOdd, + 3, + tt.fields.codecBytesOdd, + tt.fields.numCodecBytesInOdd, + tt.fields.ddBytesOdd, + ) } } } @@ -264,12 +332,14 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { require.Equal(t, tt.fields.markerEven, sn.marker) require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) - require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) + require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) + require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) - require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) + require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) + require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) } } if !reflect.DeepEqual(got, tt.want) { From bd7950f56c04dc43238e8f96bae3df0e05b375ff Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 21 Feb 2024 20:21:59 +0530 Subject: [PATCH 04/73] Cleaning up logs added for debugging (#2497) --- pkg/rtc/mediatrack.go | 2 -- pkg/rtc/participant.go | 2 +- pkg/sfu/downtrack.go | 12 ------------ pkg/sfu/forwarder.go | 14 -------------- 4 files changed, 1 insertion(+), 29 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 16f6c24f3..57141fb62 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -257,11 +257,9 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra sfu.WithStreamTrackers(), ) newWR.OnCloseHandler(func() { - t.params.Logger.Infow("webrtc receiver closed") t.MediaTrackReceiver.SetClosing() t.MediaTrackReceiver.ClearReceiver(mime, false) if t.MediaTrackReceiver.TryClose() { - t.params.Logger.Infow("mediaTrack closed") if t.dynacastManager != nil { t.dynacastManager.Close() } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 74e27dc2f..a3b939955 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2104,7 +2104,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv p.dirty.Store(true) - p.pubLogger.Infow("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti)) + p.pubLogger.Debugw("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti)) if onTrackUnpublished := p.getOnTrackUnpublished(); onTrackUnpublished != nil { onTrackUnpublished(p, mt) } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index a4df99776..a4d0b1be5 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1627,18 +1627,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { if err == io.EOF { break } - // TODO-VP9-DEBUG-REMOVE-START - d.params.Logger.Debugw( - "NACK miss", - "isn", epm.sourceSeqNo, - "osn", epm.targetSeqNo, - "ots", epm.timestamp, - "eosn", epm.extSequenceNumber, - "eots", epm.extTimestamp, - "sid", epm.layer, - "error", err, - ) - // TODO-VP9-DEBUG-REMOVE-END nackMisses++ continue } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 6d7c1d91a..fceebbdd6 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1704,20 +1704,6 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in if tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, result.RTPMarker); err == nil { if tpRTP.snOrdering == SequenceNumberOrderingContiguous { f.rtpMunger.PacketDropped(extPkt) - } else { - // TODO-VP9-DEBUG-REMOVE-START - f.logger.Debugw( - "dropping packet skipped as not contiguous", - "isn", extPkt.ExtSequenceNumber, - "its", extPkt.ExtTimestamp, - "osn", tpRTP.extSequenceNumber, - "ots", tpRTP.extTimestamp, - "payloadLen", len(extPkt.Packet.Payload), - "sid", extPkt.Spatial, - "tid", extPkt.Temporal, - "snOrdering", tpRTP.snOrdering, - ) - // TODO-VP9-DEBUG-REMOVE-END } } } From 6895eff496027fc08d7913389d0ea86fb27ddd48 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 21 Feb 2024 22:58:56 +0530 Subject: [PATCH 05/73] Buffer size config for video and audio. (#2498) * Buffer size config for video and audio. There was only one buffer size in config. In upstream, config value was used for video. Audio used a hard coded value of 200 packets. But, in the down stream sequencer, the config value was used for both video and audio. So, if video was set up for high bit rate (deep buffers), audio sequencer ended up using a lot of memory too in sequencer. Split config to be able to control that and also not hard code audio. Another optimisation here would be to not instantiate sequencer unkess NACK is negotiated. * deprecate packet_buffer_size --- config-sample.yaml | 7 ++++--- pkg/config/config.go | 14 ++++++++++---- pkg/rtc/config.go | 12 ++++++++++-- pkg/rtc/mediatracksubscriptions.go | 5 ++++- pkg/rtc/participant_internal_test.go | 2 +- pkg/rtc/room.go | 4 ++-- pkg/sfu/buffer/factory.go | 6 +++--- 7 files changed, 34 insertions(+), 16 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index ae34d759f..da88700be 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -94,8 +94,10 @@ rtc: # allow_pause: true # # allows automatic connection fallback to TCP and TURN/TLS (if configured) when UDP has been unstable, default true # allow_tcp_fallback: true - # # number of packets to buffer in the SFU, defaults to 500 - # packet_buffer_size: 500 + # # number of packets to buffer in the SFU for video, defaults to 500 + # packet_buffer_size_video: 500 + # # number of packets to buffer in the SFU for audio, defaults to 200 + # packet_buffer_size_audio: 200 # # minimum amount of time between pli/fir rtcp packets being sent to an individual # # producer. Increasing these times can lead to longer black screens when new participants join, # # while reducing them can lead to higher stream bitrate. @@ -309,4 +311,3 @@ keys: # # value less or equal than 0 means no limit. # subscription_limit_video: 0 # subscription_limit_audio: 0 - diff --git a/pkg/config/config.go b/pkg/config/config.go index f6f0891a3..c7d86c0bd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,7 +76,7 @@ type Config struct { Region string `yaml:"region,omitempty"` SignalRelay SignalRelayConfig `yaml:"signal_relay,omitempty"` PSRPC rpc.PSRPCConfig `yaml:"psrpc,omitempty"` - // LogLevel is deprecated + // Deprecated: LogLevel is deprecated LogLevel string `yaml:"log_level,omitempty"` Logging LoggingConfig `yaml:"logging,omitempty"` Limit LimitConfig `yaml:"limit,omitempty"` @@ -91,8 +91,12 @@ type RTCConfig struct { StrictACKs bool `yaml:"strict_acks,omitempty"` - // Number of packets to buffer for NACK + // Deprecated: use PacketBufferSizeVideo and PacketBufferSizeAudio PacketBufferSize int `yaml:"packet_buffer_size,omitempty"` + // Number of packets to buffer for NACK - video + PacketBufferSizeVideo int `yaml:"packet_buffer_size_video,omitempty"` + // Number of packets to buffer for NACK - audio + PacketBufferSizeAudio int `yaml:"packet_buffer_size_audio,omitempty"` // Throttle periods for pli/fir rtcp packets PLIThrottle PLIThrottleConfig `yaml:"pli_throttle,omitempty"` @@ -327,8 +331,10 @@ var DefaultConfig = Config{ ICEPortRangeEnd: 0, STUNServers: []string{}, }, - PacketBufferSize: 500, - StrictACKs: true, + PacketBufferSize: 500, + PacketBufferSizeVideo: 500, + PacketBufferSizeAudio: 200, + StrictACKs: true, PLIThrottle: PLIThrottleConfig{ LowQuality: 500 * time.Millisecond, MidQuality: time.Second, diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index 239b08608..f8a8054be 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -39,7 +39,8 @@ type WebRTCConfig struct { } type ReceiverConfig struct { - PacketBufferSize int + PacketBufferSizeVideo int + PacketBufferSizeAudio int } type RTPHeaderExtensionConfig struct { @@ -72,6 +73,12 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { if rtcConf.PacketBufferSize == 0 { rtcConf.PacketBufferSize = 500 } + if rtcConf.PacketBufferSizeVideo == 0 { + rtcConf.PacketBufferSizeVideo = rtcConf.PacketBufferSize + } + if rtcConf.PacketBufferSizeAudio == 0 { + rtcConf.PacketBufferSizeAudio = rtcConf.PacketBufferSize + } // publisher configuration publisherConfig := DirectionConfig{ @@ -129,7 +136,8 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { return &WebRTCConfig{ WebRTCConfig: *webRTCConfig, Receiver: ReceiverConfig{ - PacketBufferSize: rtcConf.PacketBufferSize, + PacketBufferSizeVideo: rtcConf.PacketBufferSizeVideo, + PacketBufferSizeAudio: rtcConf.PacketBufferSizeAudio, }, Publisher: publisherConfig, Subscriber: subscriberConfig, diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index d960f8de2..3f92e922d 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -103,11 +103,14 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * t.subscribedTracksMu.Unlock() var rtcpFeedback []webrtc.RTCPFeedback + var maxTrack int switch t.params.MediaTrack.Kind() { case livekit.TrackType_AUDIO: rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Audio + maxTrack = t.params.ReceiverConfig.PacketBufferSizeAudio case livekit.TrackType_VIDEO: rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Video + maxTrack = t.params.ReceiverConfig.PacketBufferSizeVideo } codecs := wr.Codecs() for _, c := range codecs { @@ -130,7 +133,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * BufferFactory: sub.GetBufferFactory(), SubID: subscriberID, StreamID: streamID, - MaxTrack: t.params.ReceiverConfig.PacketBufferSize, + MaxTrack: maxTrack, PlayoutDelayLimit: sub.GetPlayoutDelayConfig(), Pacer: sub.GetPacer(), Trailer: trailer, diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 5f5f8eac6..a7bd9b4e3 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -731,7 +731,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p if err != nil { panic(err) } - ff := buffer.NewFactoryOfBufferFactory(500) + ff := buffer.NewFactoryOfBufferFactory(500, 200) rtcConf.SetBufferFactory(ff.CreateBufferFactory()) grants := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 3b94a0cf9..082f91f71 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -87,7 +87,7 @@ type Room struct { joinedAt atomic.Int64 // time that the last participant left the room leftAt atomic.Int64 - holds atomic.Int32 + holds atomic.Int32 lock sync.RWMutex @@ -164,7 +164,7 @@ func NewRoom( participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), hasPublished: make(map[livekit.ParticipantIdentity]bool), - bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), + bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSizeVideo, config.Receiver.PacketBufferSizeAudio), batchedUpdates: make(map[livekit.ParticipantIdentity]*participantUpdate), closed: make(chan struct{}), trailer: []byte(utils.RandomSecret()), diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go index d59038d28..2520c6cce 100644 --- a/pkg/sfu/buffer/factory.go +++ b/pkg/sfu/buffer/factory.go @@ -28,17 +28,17 @@ type FactoryOfBufferFactory struct { audioPool *sync.Pool } -func NewFactoryOfBufferFactory(trackingPackets int) *FactoryOfBufferFactory { +func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory { return &FactoryOfBufferFactory{ videoPool: &sync.Pool{ New: func() interface{} { - b := make([]byte, trackingPackets*bucket.MaxPktSize) + b := make([]byte, trackingPacketsVideo*bucket.MaxPktSize) return &b }, }, audioPool: &sync.Pool{ New: func() interface{} { - b := make([]byte, bucket.MaxPktSize*200) + b := make([]byte, trackingPacketsAudio*bucket.MaxPktSize) return &b }, }, From f3e02d513296fc31d53cd7bd754736d35cc76ebb Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 22 Feb 2024 02:04:13 +0530 Subject: [PATCH 06/73] Prevent multiple debounce of quality downgrade. (#2499) --- pkg/rtc/dynacastmanager.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index c9395e60a..896e36449 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -40,7 +40,8 @@ type DynacastManager struct { maxSubscribedQuality map[string]livekit.VideoQuality committedMaxSubscribedQuality map[string]livekit.VideoQuality - maxSubscribedQualityDebounce func(func()) + maxSubscribedQualityDebounce func(func()) + maxSubscribedQualityDebouncePending bool qualityNotifyOpQueue *utils.OpsQueue @@ -223,13 +224,21 @@ func (d *DynacastManager) update(force bool) { } if downgradesOnly { - d.params.Logger.Debugw("debouncing quality downgrade", - "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, - "maxSubscribedQuality", d.maxSubscribedQuality, - ) - d.maxSubscribedQualityDebounce(func() { - d.update(true) - }) + if !d.maxSubscribedQualityDebouncePending { + d.params.Logger.Debugw("debouncing quality downgrade", + "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, + "maxSubscribedQuality", d.maxSubscribedQuality, + ) + d.maxSubscribedQualityDebounce(func() { + d.update(true) + }) + d.maxSubscribedQualityDebouncePending = true + } else { + d.params.Logger.Debugw("quality downgrade waiting for debounce", + "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, + "maxSubscribedQuality", d.maxSubscribedQuality, + ) + } d.lock.Unlock() return } @@ -237,6 +246,7 @@ func (d *DynacastManager) update(force bool) { // clear debounce on send d.maxSubscribedQualityDebounce(func() {}) + d.maxSubscribedQualityDebouncePending = false d.params.Logger.Debugw("committing quality change", "force", force, From 787af0e84cf39fd913b24fde2625c936864851ab Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 22 Feb 2024 11:55:37 +0800 Subject: [PATCH 07/73] update pion/ice for tcpmux memory improvement (#2500) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a26ffd656..29a2c5b94 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.13 + github.com/pion/ice/v2 v2.3.14 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.13 github.com/pion/rtp v1.8.3 diff --git a/go.sum b/go.sum index 3b4e5741e..f38802320 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,9 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.13 h1:xOxP+4V9nSDlUaGFRf/LvAuGHDXRcjIdsbbXPK/w7c8= github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.14 h1:A7UaEmalw12Fko8YO0qguUbWyE69BnN4mDEqT7cLWQI= +github.com/pion/ice/v2 v2.3.14/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= From 132d1ad31a7458f670cb40fd77ffacea3bfb46f6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 23 Feb 2024 08:20:03 +0530 Subject: [PATCH 08/73] Null out on slice compaction. (#2503) Go 1.22 does this automatically with `slices` - https://go.dev/blog/generic-slice-functions --- pkg/rtc/mediatrackreceiver.go | 1 + pkg/rtc/participant.go | 1 + pkg/rtc/participant_sdp.go | 1 + 3 files changed, 3 insertions(+) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index d0818ceb1..b0bb43ed5 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -259,6 +259,7 @@ func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { for idx, receiver := range receivers { if strings.EqualFold(receiver.Codec().MimeType, mime) { receivers[idx] = receivers[len(receivers)-1] + receivers[len(receivers)-1] = nil receivers = receivers[:len(receivers)-1] break } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a3b939955..5e8ffd061 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -733,6 +733,7 @@ func (p *ParticipantImpl) removePendingMigratedTrack(mt *MediaTrack) { for i, t := range p.pendingMigratedTracks { if t == mt { p.pendingMigratedTracks[i] = p.pendingMigratedTracks[len(p.pendingMigratedTracks)-1] + p.pendingMigratedTracks[len(p.pendingMigratedTracks)-1] = nil p.pendingMigratedTracks = p.pendingMigratedTracks[:len(p.pendingMigratedTracks)-1] break } diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index ed47d2da9..2516291d3 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -157,6 +157,7 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess for i, attr := range unmatchVideo.Attributes { if strings.Contains(attr.Value, dd.ExtensionURI) { unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] + unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] = sdp.Attribute{} unmatchVideo.Attributes = unmatchVideo.Attributes[:len(unmatchVideo.Attributes)-1] break } From 90ab3fdf68641d69deef37381d670bd8dd74f934 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 23 Feb 2024 13:10:58 +0800 Subject: [PATCH 09/73] Reduce FrameIntegrityChecker's allocation (#2504) --- pkg/sfu/buffer/frameintegrity.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/frameintegrity.go b/pkg/sfu/buffer/frameintegrity.go index bc6fe30a7..536f793cf 100644 --- a/pkg/sfu/buffer/frameintegrity.go +++ b/pkg/sfu/buffer/frameintegrity.go @@ -23,7 +23,7 @@ type FrameEntity struct { endSeq *uint64 integrity bool - packetsConsective func(uint64, uint64) bool + pktHistory *PacketHistory } func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) { @@ -40,7 +40,7 @@ func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) } if fe.startSeq != nil && fe.endSeq != nil { - if fe.packetsConsective(*fe.startSeq, *fe.endSeq) { + if fe.pktHistory.PacketsConsecutive(*fe.startSeq, *fe.endSeq) { fe.integrity = true } } @@ -179,7 +179,7 @@ func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecke } for i := range fc.frames { - fc.frames[i].packetsConsective = fc.pktHistory.PacketsConsecutive + fc.frames[i].pktHistory = fc.pktHistory fc.frames[i].Reset() } return fc From 013c933636c2f0a806019fd461afd1625d24e785 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Fri, 23 Feb 2024 18:56:11 +0200 Subject: [PATCH 10/73] Properly persist participant kind. (#2505) --- pkg/rtc/participant.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5e8ffd061..1851128cf 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -33,6 +33,12 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/mediatransportutil/pkg/twcc" + "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc/supervisor" @@ -46,11 +52,6 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" sutils "github.com/livekit/livekit-server/pkg/utils" - "github.com/livekit/mediatransportutil/pkg/twcc" - "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" ) const ( @@ -512,6 +513,7 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. Metadata: p.grants.Metadata, Region: p.params.Region, IsPublisher: p.IsPublisher(), + Kind: p.grants.GetParticipantKind(), } p.lock.RUnlock() From e5b8e250645efd6235d6c230555c6a283e7ca185 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sat, 24 Feb 2024 00:38:49 -0800 Subject: [PATCH 11/73] use shared psrpc utils (#2506) * use shared psrpc utils * fix * deps --- go.mod | 12 +-- go.sum | 26 +++---- pkg/routing/signal.go | 2 +- pkg/rtc/uptrackmanager.go | 14 ++-- pkg/service/roommanager.go | 5 +- pkg/service/signal.go | 2 +- pkg/service/wire.go | 3 +- pkg/service/wire_gen.go | 5 +- pkg/telemetry/prometheus/node.go | 3 +- pkg/telemetry/prometheus/psrpc.go | 125 ------------------------------ 10 files changed, 34 insertions(+), 163 deletions(-) delete mode 100644 pkg/telemetry/prometheus/psrpc.go diff --git a/go.mod b/go.mod index 29a2c5b94..746678cf1 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 - github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582 + github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175 github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -38,7 +38,7 @@ require ( github.com/pion/webrtc/v3 v3.2.28 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 - github.com/redis/go-redis/v9 v9.4.0 + github.com/redis/go-redis/v9 v9.5.1 github.com/rs/cors v1.10.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 @@ -47,7 +47,7 @@ require ( github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 @@ -95,14 +95,14 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect - google.golang.org/grpc v1.61.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect + google.golang.org/grpc v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index f38802320..5feec2771 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 h1:Zw88krOHni51OzDUlrduYb3m7VcsaKw06TnnDhsQpjg= github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582 h1:hSOSQs2pKF6TD9CEwu7+LatqfvF/LiyIbeCoUPCGRho= -github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582/go.mod h1:/kviHT6yTNqHdZ9QsvRuxAHf7LaBROa7qe5naT1oVrU= +github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175 h1:xX29sbgPxTmwUexxmDvNr+NgREHJEIvMOwuW870DatI= +github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175/go.mod h1:kT55fgJZgd1CAIO2w8tcc78ugw5xVjrSliy8pHGUeCw= github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58 h1:yH55rBGLRO+ict2mu6bKZ5iPwTIrIwU1i0ydgThi4+k= github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -238,8 +238,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= -github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= @@ -280,11 +280,11 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -297,8 +297,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -424,10 +424,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 h1:FSL3lRCkhaPFxqi0s9o+V4UI2WTzAVOvkgbd4kVV4Wg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4= -google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= -google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= +google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index e7181038d..c4f05b781 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -55,7 +55,7 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. c, err := rpc.NewTypedSignalClient( nodeID, bus, - middleware.WithClientMetrics(prometheus.PSRPCMetricsObserver{}), + middleware.WithClientMetrics(rpc.PSRPCMetricsObserver{}), psrpc.WithClientChannelSize(config.StreamBufferSize), ) if err != nil { diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 2e81324ab..65d0893d7 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -168,16 +168,12 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( // owner for the data, we'd prefer to use their TimedVersion // ignore older version if !timedVersion.After(&u.subscriptionPermissionVersion) { - perms := "" - if u.subscriptionPermission != nil { - perms = u.subscriptionPermission.String() - } u.params.Logger.Debugw( "skipping older subscription permission version", - "existingValue", perms, - "existingVersion", u.subscriptionPermissionVersion.String(), + "existingValue", logger.Proto(u.subscriptionPermission), + "existingVersion", &u.subscriptionPermissionVersion, "requestingValue", logger.Proto(subscriptionPermission), - "requestingVersion", timedVersion.String(), + "requestingVersion", &timedVersion, ) u.lock.Unlock() return nil @@ -193,7 +189,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( if subscriptionPermission == nil { u.params.Logger.Debugw( "updating subscription permission, setting to nil", - "version", u.subscriptionPermissionVersion.String(), + "version", &u.subscriptionPermissionVersion, ) // possible to get a nil when migrating u.lock.Unlock() @@ -203,7 +199,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( u.params.Logger.Debugw( "updating subscription permission", "permissions", logger.Proto(u.subscriptionPermission), - "version", u.subscriptionPermissionVersion.String(), + "version", &u.subscriptionPermissionVersion, ) if err := u.parseSubscriptionPermissionsLocked(subscriptionPermission, func(pID livekit.ParticipantID) types.LocalParticipant { u.lock.Unlock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index be65c314f..d64cceabb 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -32,6 +32,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/must" "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/clientconfiguration" @@ -455,7 +456,7 @@ func (r *RoomManager) StartSession( } participantTopic := rpc.FormatParticipantTopic(roomName, participant.Identity()) - participantServer := utils.Must(rpc.NewTypedParticipantServer(r, r.bus)) + participantServer := must.Get(rpc.NewTypedParticipantServer(r, r.bus)) killParticipantServer := r.participantServers.Replace(participantTopic, participantServer) if err := participantServer.RegisterAllParticipantTopics(participantTopic); err != nil { killParticipantServer() @@ -547,7 +548,7 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room newRoom := rtc.NewRoom(ri, internal, *r.rtcConfig, &r.config.Audio, r.serverInfo, r.telemetry, r.agentClient, r.egressLauncher) roomTopic := rpc.FormatRoomTopic(roomName) - roomServer := utils.Must(rpc.NewTypedRoomServer(r, r.bus)) + roomServer := must.Get(rpc.NewTypedRoomServer(r, r.bus)) killRoomServer := r.roomServers.Replace(roomTopic, roomServer) if err := roomServer.RegisterAllRoomTopics(roomTopic); err != nil { killRoomServer() diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 6c4ff820e..69c04baab 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -63,7 +63,7 @@ func NewSignalServer( nodeID, &signalService{region, sessionHandler, config}, bus, - middleware.WithServerMetrics(prometheus.PSRPCMetricsObserver{}), + middleware.WithServerMetrics(rpc.PSRPCMetricsObserver{}), psrpc.WithServerChannelSize(config.StreamBufferSize), ) if err != nil { diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 0a2501a1d..654f26345 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -32,7 +32,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -233,7 +232,7 @@ func getPSRPCConfig(config *config.Config) rpc.PSRPCConfig { } func getPSRPCClientParams(config rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.ClientParams { - return rpc.NewClientParams(config, bus, logger.GetLogger(), prometheus.PSRPCMetricsObserver{}) + return rpc.NewClientParams(config, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) } func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index c8307a3bf..82d232e55 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -13,7 +13,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -284,7 +283,7 @@ func getPSRPCConfig(config2 *config.Config) rpc.PSRPCConfig { } func getPSRPCClientParams(config2 rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.ClientParams { - return rpc.NewClientParams(config2, bus, logger.GetLogger(), prometheus.PSRPCMetricsObserver{}) + return rpc.NewClientParams(config2, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) } func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index dad09977e..ba6dd4507 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -23,6 +23,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/rpc" ) const ( @@ -108,7 +109,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { initPacketStats(nodeID, nodeType, env) initRoomStats(nodeID, nodeType, env) - initPSRPCStats(nodeID, nodeType, env) + rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}) initQualityStats(nodeID, nodeType, env) } diff --git a/pkg/telemetry/prometheus/psrpc.go b/pkg/telemetry/prometheus/psrpc.go deleted file mode 100644 index f07c90034..000000000 --- a/pkg/telemetry/prometheus/psrpc.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prometheus - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/psrpc" - "github.com/livekit/psrpc/pkg/middleware" -) - -var ( - psrpcRequestTime *prometheus.HistogramVec - psrpcStreamSendTime *prometheus.HistogramVec - psrpcStreamReceiveTotal *prometheus.CounterVec - psrpcStreamCurrent *prometheus.GaugeVec - psrpcErrorTotal *prometheus.CounterVec -) - -func initPSRPCStats(nodeID string, nodeType livekit.NodeType, env string) { - labels := []string{"role", "kind", "service", "method"} - streamLabels := []string{"role", "service", "method"} - - psrpcRequestTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: livekitNamespace, - Subsystem: "psrpc", - Name: "request_time_ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, - }, labels) - psrpcStreamSendTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: livekitNamespace, - Subsystem: "psrpc", - Name: "stream_send_time_ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, - }, streamLabels) - psrpcStreamReceiveTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: livekitNamespace, - Subsystem: "psrpc", - Name: "stream_receive_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - }, streamLabels) - psrpcStreamCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: livekitNamespace, - Subsystem: "psrpc", - Name: "stream_count", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - }, streamLabels) - psrpcErrorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: livekitNamespace, - Subsystem: "psrpc", - Name: "error_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - }, labels) - - prometheus.MustRegister(psrpcRequestTime) - prometheus.MustRegister(psrpcStreamSendTime) - prometheus.MustRegister(psrpcStreamReceiveTotal) - prometheus.MustRegister(psrpcStreamCurrent) - prometheus.MustRegister(psrpcErrorTotal) -} - -var _ middleware.MetricsObserver = PSRPCMetricsObserver{} - -type PSRPCMetricsObserver struct{} - -func (o PSRPCMetricsObserver) OnUnaryRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { - if err != nil { - psrpcErrorTotal.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Inc() - } else if role == middleware.ClientRole { - psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) - } else { - psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) - } -} - -func (o PSRPCMetricsObserver) OnMultiRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, responseCount int, errorCount int) { - if responseCount == 0 { - psrpcErrorTotal.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Inc() - } else if role == middleware.ClientRole { - psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) - } else { - psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) - } -} - -func (o PSRPCMetricsObserver) OnStreamSend(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { - if err != nil { - psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() - } else { - psrpcStreamSendTime.WithLabelValues(role.String(), info.Service, info.Method).Observe(float64(duration.Milliseconds())) - } -} - -func (o PSRPCMetricsObserver) OnStreamRecv(role middleware.MetricRole, info psrpc.RPCInfo, err error) { - if err != nil { - psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() - } else { - psrpcStreamReceiveTotal.WithLabelValues(role.String(), info.Service, info.Method).Inc() - } -} - -func (o PSRPCMetricsObserver) OnStreamOpen(role middleware.MetricRole, info psrpc.RPCInfo) { - psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Inc() -} - -func (o PSRPCMetricsObserver) OnStreamClose(role middleware.MetricRole, info psrpc.RPCInfo) { - psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Dec() -} From eed457d8bec72a88842fca4865ec2e7002587c88 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 25 Feb 2024 01:31:04 +0530 Subject: [PATCH 12/73] Do not need pending migration tracks. (#2507) Simplify a bit. Pending migratiion tracks need not be maintained as when a migrated track is added, it is added to up track manager and treated as a published track. When up track manager closes, published tracks will close. So, no need to maintain a separate list. --- pkg/rtc/participant.go | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 1851128cf..efdce9d63 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -172,8 +172,6 @@ type ParticipantImpl struct { pendingTracksLock utils.RWMutex pendingTracks map[string]*pendingTrackInfo pendingPublishingTracks map[livekit.TrackID]*pendingTrackInfo - // migrated in tracks that have not fired need close at participant close - pendingMigratedTracks []*MediaTrack // supported codecs enabledPublishCodecs []*livekit.Codec @@ -714,7 +712,6 @@ func (p *ParticipantImpl) handleMigrateTracks() { p.pubLogger.Warnw("could not find migrated track", nil, "cid", cid) } } - p.pendingMigratedTracks = append(p.pendingMigratedTracks, addedTracks...) if len(addedTracks) != 0 { p.dirty.Store(true) @@ -730,19 +727,6 @@ func (p *ParticipantImpl) handleMigrateTracks() { }() } -func (p *ParticipantImpl) removePendingMigratedTrack(mt *MediaTrack) { - p.pendingTracksLock.Lock() - for i, t := range p.pendingMigratedTracks { - if t == mt { - p.pendingMigratedTracks[i] = p.pendingMigratedTracks[len(p.pendingMigratedTracks)-1] - p.pendingMigratedTracks[len(p.pendingMigratedTracks)-1] = nil - p.pendingMigratedTracks = p.pendingMigratedTracks[:len(p.pendingMigratedTracks)-1] - break - } - } - p.pendingTracksLock.Unlock() -} - // AddTrack is called when client intends to publish track. // records track details and lets client know it's ok to proceed func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { @@ -813,14 +797,9 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.pendingTracksLock.Lock() p.pendingTracks = make(map[string]*pendingTrackInfo) - pendingMigratedTracksToClose := p.pendingMigratedTracks - p.pendingMigratedTracks = p.pendingMigratedTracks[:0] + p.pendingPublishingTracks = make(map[livekit.TrackID]*pendingTrackInfo) p.pendingTracksLock.Unlock() - for _, t := range pendingMigratedTracksToClose { - t.Close(isExpectedToResume) - } - p.UpTrackManager.Close(isExpectedToResume) p.updateState(livekit.ParticipantInfo_DISCONNECTED) @@ -1967,9 +1946,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei p.pendingTracksLock.Unlock() - if mt.AddReceiver(rtpReceiver, track, mid) { - p.removePendingMigratedTrack(mt) - } + mt.AddReceiver(rtpReceiver, track, mid) if newTrack { go func() { From 6696c1e63d5d8c0a3f1708cdc67ac27485865015 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 25 Feb 2024 13:46:55 +0530 Subject: [PATCH 13/73] Close published track always. (#2508) Simplify and close published track always. Avoid the round about path of unbind to close. --- pkg/rtc/participant.go | 2 +- pkg/rtc/types/interfaces.go | 2 +- .../types/typesfakes/fake_local_participant.go | 18 ++++++++---------- pkg/rtc/types/typesfakes/fake_participant.go | 18 ++++++++---------- pkg/rtc/uptrackmanager.go | 12 ++++-------- pkg/service/wire_gen.go | 2 +- pkg/sfu/forwarder.go | 2 +- 7 files changed, 24 insertions(+), 32 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index efdce9d63..1c8fe892c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1384,7 +1384,7 @@ func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) err } func (p *ParticipantImpl) removePublishedTrack(track types.MediaTrack) { - p.RemovePublishedTrack(track, false, false) + p.RemovePublishedTrack(track, false) if p.ProtocolVersion().SupportsUnpublish() { p.sendTrackUnpublished(track.ID()) } else { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index a08443d7a..da5cc5e2a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -255,7 +255,7 @@ type Participant interface { IsPublisher() bool GetPublishedTrack(trackID livekit.TrackID) MediaTrack GetPublishedTracks() []MediaTrack - RemovePublishedTrack(track MediaTrack, willBeResumed bool, shouldClose bool) + RemovePublishedTrack(track MediaTrack, willBeResumed bool) GetAudioLevel() (smoothedLevel float64, active bool) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index c20a1a1d7..131e2f87c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -627,12 +627,11 @@ type FakeLocalParticipant struct { protocolVersionReturnsOnCall map[int]struct { result1 types.ProtocolVersion } - RemovePublishedTrackStub func(types.MediaTrack, bool, bool) + RemovePublishedTrackStub func(types.MediaTrack, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool - arg3 bool } RemoveTrackFromSubscriberStub func(*webrtc.RTPSender) error removeTrackFromSubscriberMutex sync.RWMutex @@ -4267,18 +4266,17 @@ func (fake *FakeLocalParticipant) ProtocolVersionReturnsOnCall(i int, result1 ty }{result1} } -func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { +func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - arg3 bool - }{arg1, arg2, arg3}) + }{arg1, arg2}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2, arg3) + fake.RemovePublishedTrackStub(arg1, arg2) } } @@ -4288,17 +4286,17 @@ func (fake *FakeLocalParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { +func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { +func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeLocalParticipant) RemoveTrackFromSubscriber(arg1 *webrtc.RTPSender) error { diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 98fff9c80..14f5d4fcc 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -158,12 +158,11 @@ type FakeParticipant struct { isRecorderReturnsOnCall map[int]struct { result1 bool } - RemovePublishedTrackStub func(types.MediaTrack, bool, bool) + RemovePublishedTrackStub func(types.MediaTrack, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool - arg3 bool } SetMetadataStub func(string) setMetadataMutex sync.RWMutex @@ -1008,18 +1007,17 @@ func (fake *FakeParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { +func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - arg3 bool - }{arg1, arg2, arg3}) + }{arg1, arg2}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2, arg3) + fake.RemovePublishedTrackStub(arg1, arg2) } } @@ -1029,17 +1027,17 @@ func (fake *FakeParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { +func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { +func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeParticipant) SetMetadata(arg1 string) { diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 65d0893d7..9ec2f6089 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -46,8 +46,8 @@ type UpTrackManager struct { closed bool // publishedTracks that participant is publishing - publishedTracks map[livekit.TrackID]types.MediaTrack - subscriptionPermission *livekit.SubscriptionPermission + publishedTracks map[livekit.TrackID]types.MediaTrack + subscriptionPermission *livekit.SubscriptionPermission // subscriber permission for published tracks subscriberPermissions map[livekit.ParticipantIdentity]*livekit.TrackPermission // subscriberIdentity => *livekit.TrackPermission @@ -268,12 +268,8 @@ func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) { }) } -func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, willBeResumed bool, shouldClose bool) { - if shouldClose { - track.Close(willBeResumed) - } else { - track.ClearAllReceivers(willBeResumed) - } +func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, willBeResumed bool) { + track.Close(willBeResumed) u.lock.Lock() delete(u.publishedTracks, track.ID()) u.lock.Unlock() diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 82d232e55..ea62d53fc 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index fceebbdd6..79d0c2ee5 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -247,7 +247,7 @@ func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) bool { } f.vls.SetMaxSeenSpatial(maxPublishedLayer) - f.logger.Debugw("setting max published layer", "maxPublishedLayer", maxPublishedLayer) + f.logger.Debugw("setting max published layer", "layer", maxPublishedLayer) return true } From 2066bf115409ede08376c97d516f790ff5043def Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 25 Feb 2024 15:24:42 -0800 Subject: [PATCH 14/73] Update go deps (#2480) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 746678cf1..2695ebd61 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 github.com/urfave/cli/v2 v2.27.1 - github.com/urfave/negroni/v3 v3.0.0 + github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 diff --git a/go.sum b/go.sum index 5feec2771..ef9cde1da 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 h1:F04Na0QJP9GJrw github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= -github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= +github.com/urfave/negroni/v3 v3.1.0 h1:lzmuxGSpnJCT/ujgIAjkU3+LW3NX8alCglO/L6KjIGQ= +github.com/urfave/negroni/v3 v3.1.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From de08de1a14307db93fdb525bef1b793a88aac2cc Mon Sep 17 00:00:00 2001 From: Russ d'Sa Date: Sun, 25 Feb 2024 15:28:45 -0800 Subject: [PATCH 15/73] Update README.md (#2510) --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9e9ff0648..5b5d86238 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,11 @@ https://docs.livekit.io - Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) - [AI voice assistant using ChatGPT](https://livekit.io/kitt) ([source](https://github.com/livekit-examples/kitt)) +## Ecosystem +- [Agents](https://github.com/livekit/agents) - build real-time multimodal AI applications with programmable backend participants +- [Egress](https://github.com/livekit/egress): record or multi-stream rooms and export individual tracks +- [Ingress](https://github.com/livekit/ingress) - ingest streams from external sources like RTMP, WHIP, HLS, or OBS Studio + ## SDKs & Tools ### Client SDKs @@ -181,11 +186,9 @@ enabling you to build automations that behave like end-users. | Python (community) | [tradablebits/livekit-server-sdk-python](https://github.com/tradablebits/livekit-server-sdk-python) | | | PHP (community) | [agence104/livekit-server-sdk-php](https://github.com/agence104/livekit-server-sdk-php) | | -### Ecosystem & Tools +### Tools - [CLI](https://github.com/livekit/livekit-cli) - command line interface & load tester -- [Egress](https://github.com/livekit/egress) - export and record your rooms -- [Ingress](https://github.com/livekit/ingress) - ingest streams from RTMP / OBS Studio - [Docker image](https://hub.docker.com/r/livekit/livekit-server) - [Helm charts](https://github.com/livekit/livekit-helm) From 8b6fc9ae534e50f5f8efa887bace97064aeacdcb Mon Sep 17 00:00:00 2001 From: Russ d'Sa Date: Sun, 25 Feb 2024 15:44:38 -0800 Subject: [PATCH 16/73] Update README.md (#2511) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b5d86238..c8207dc18 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,9 @@ https://docs.livekit.io - [AI voice assistant using ChatGPT](https://livekit.io/kitt) ([source](https://github.com/livekit-examples/kitt)) ## Ecosystem -- [Agents](https://github.com/livekit/agents) - build real-time multimodal AI applications with programmable backend participants +- [Agents](https://github.com/livekit/agents): build real-time multimodal AI applications with programmable backend participants - [Egress](https://github.com/livekit/egress): record or multi-stream rooms and export individual tracks -- [Ingress](https://github.com/livekit/ingress) - ingest streams from external sources like RTMP, WHIP, HLS, or OBS Studio +- [Ingress](https://github.com/livekit/ingress): ingest streams from external sources like RTMP, WHIP, HLS, or OBS Studio ## SDKs & Tools From ef4bc0b46db5be4536219d18468ff04ba2a0e4a7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 26 Feb 2024 15:15:04 +0530 Subject: [PATCH 17/73] Support no dynacast pause. (#2512) --- pkg/rtc/dynacastmanager.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 896e36449..0fed210ce 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -59,9 +59,11 @@ func NewDynacastManager(params DynacastManagerParams) *DynacastManager { dynacastQuality: make(map[string]*DynacastQuality), maxSubscribedQuality: make(map[string]livekit.VideoQuality), committedMaxSubscribedQuality: make(map[string]livekit.VideoQuality), - maxSubscribedQualityDebounce: debounce.New(params.DynacastPauseDelay), qualityNotifyOpQueue: utils.NewOpsQueue("quality-notify", 64, true), } + if params.DynacastPauseDelay > 0 { + d.maxSubscribedQualityDebounce = debounce.New(params.DynacastPauseDelay) + } d.qualityNotifyOpQueue.Start() return d } @@ -223,7 +225,7 @@ func (d *DynacastManager) update(force bool) { return } - if downgradesOnly { + if downgradesOnly && d.maxSubscribedQualityDebounce != nil { if !d.maxSubscribedQualityDebouncePending { d.params.Logger.Debugw("debouncing quality downgrade", "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, @@ -245,8 +247,10 @@ func (d *DynacastManager) update(force bool) { } // clear debounce on send - d.maxSubscribedQualityDebounce(func() {}) - d.maxSubscribedQualityDebouncePending = false + if d.maxSubscribedQualityDebounce != nil { + d.maxSubscribedQualityDebounce(func() {}) + d.maxSubscribedQualityDebouncePending = false + } d.params.Logger.Debugw("committing quality change", "force", force, From 22474e0f5d1312638db79ff93a2a1c0e3775e345 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 26 Feb 2024 20:55:41 +0530 Subject: [PATCH 18/73] Remove unused interface method (#2513) * Remove unused interface method * cleanup --- pkg/rtc/types/interfaces.go | 1 - .../typesfakes/fake_local_media_track.go | 39 ------------------- pkg/rtc/types/typesfakes/fake_media_track.go | 39 ------------------- pkg/service/wire_gen.go | 2 +- 4 files changed, 1 insertion(+), 80 deletions(-) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index da5cc5e2a..8ad80ef24 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -482,7 +482,6 @@ type MediaTrack interface { GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime string) int32 Receivers() []sfu.TrackReceiver - ClearAllReceivers(willBeResumed bool) IsEncrypted() bool } diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index be86e74c4..0fbdcb6db 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -28,11 +28,6 @@ type FakeLocalMediaTrack struct { result1 types.SubscribedTrack result2 error } - ClearAllReceiversStub func(bool) - clearAllReceiversMutex sync.RWMutex - clearAllReceiversArgsForCall []struct { - arg1 bool - } CloseStub func(bool) closeMutex sync.RWMutex closeArgsForCall []struct { @@ -442,38 +437,6 @@ func (fake *FakeLocalMediaTrack) AddSubscriberReturnsOnCall(i int, result1 types }{result1, result2} } -func (fake *FakeLocalMediaTrack) ClearAllReceivers(arg1 bool) { - fake.clearAllReceiversMutex.Lock() - fake.clearAllReceiversArgsForCall = append(fake.clearAllReceiversArgsForCall, struct { - arg1 bool - }{arg1}) - stub := fake.ClearAllReceiversStub - fake.recordInvocation("ClearAllReceivers", []interface{}{arg1}) - fake.clearAllReceiversMutex.Unlock() - if stub != nil { - fake.ClearAllReceiversStub(arg1) - } -} - -func (fake *FakeLocalMediaTrack) ClearAllReceiversCallCount() int { - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - return len(fake.clearAllReceiversArgsForCall) -} - -func (fake *FakeLocalMediaTrack) ClearAllReceiversCalls(stub func(bool)) { - fake.clearAllReceiversMutex.Lock() - defer fake.clearAllReceiversMutex.Unlock() - fake.ClearAllReceiversStub = stub -} - -func (fake *FakeLocalMediaTrack) ClearAllReceiversArgsForCall(i int) bool { - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - argsForCall := fake.clearAllReceiversArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalMediaTrack) Close(arg1 bool) { fake.closeMutex.Lock() fake.closeArgsForCall = append(fake.closeArgsForCall, struct { @@ -2153,8 +2116,6 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.addOnCloseMutex.RUnlock() fake.addSubscriberMutex.RLock() defer fake.addSubscriberMutex.RUnlock() - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.getAllSubscribersMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index d4bdfc17e..7f42687a2 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -28,11 +28,6 @@ type FakeMediaTrack struct { result1 types.SubscribedTrack result2 error } - ClearAllReceiversStub func(bool) - clearAllReceiversMutex sync.RWMutex - clearAllReceiversArgsForCall []struct { - arg1 bool - } CloseStub func(bool) closeMutex sync.RWMutex closeArgsForCall []struct { @@ -378,38 +373,6 @@ func (fake *FakeMediaTrack) AddSubscriberReturnsOnCall(i int, result1 types.Subs }{result1, result2} } -func (fake *FakeMediaTrack) ClearAllReceivers(arg1 bool) { - fake.clearAllReceiversMutex.Lock() - fake.clearAllReceiversArgsForCall = append(fake.clearAllReceiversArgsForCall, struct { - arg1 bool - }{arg1}) - stub := fake.ClearAllReceiversStub - fake.recordInvocation("ClearAllReceivers", []interface{}{arg1}) - fake.clearAllReceiversMutex.Unlock() - if stub != nil { - fake.ClearAllReceiversStub(arg1) - } -} - -func (fake *FakeMediaTrack) ClearAllReceiversCallCount() int { - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - return len(fake.clearAllReceiversArgsForCall) -} - -func (fake *FakeMediaTrack) ClearAllReceiversCalls(stub func(bool)) { - fake.clearAllReceiversMutex.Lock() - defer fake.clearAllReceiversMutex.Unlock() - fake.ClearAllReceiversStub = stub -} - -func (fake *FakeMediaTrack) ClearAllReceiversArgsForCall(i int) bool { - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - argsForCall := fake.clearAllReceiversArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeMediaTrack) Close(arg1 bool) { fake.closeMutex.Lock() fake.closeArgsForCall = append(fake.closeArgsForCall, struct { @@ -1739,8 +1702,6 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.addOnCloseMutex.RUnlock() fake.addSubscriberMutex.RLock() defer fake.addSubscriberMutex.RUnlock() - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.getAllSubscribersMutex.RLock() diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index ea62d53fc..82d232e55 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 8442b2b37cbbf995d44225e38b52b4801d53049b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 27 Feb 2024 12:11:24 +0530 Subject: [PATCH 19/73] Maintain subscription count. (#2515) * Maintain subscription count. Does not affect function as it is not decremented only if limits are configured. But, good to maintain proper count anyway. * wire --- pkg/rtc/participant.go | 2 +- pkg/rtc/subscriptionmanager.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 1c8fe892c..f037a46cb 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -464,7 +464,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio if canSubscribe { // reconcile everything - p.SubscriptionManager.queueReconcile("") + p.SubscriptionManager.ReconcileAll() } else { // revoke all subscriptions for _, st := range p.SubscriptionManager.GetSubscribedTracks() { diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index eb879d1a6..8232c7527 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -255,6 +255,10 @@ func (m *SubscriptionManager) WaitUntilSubscribed(timeout time.Duration) error { return context.DeadlineExceeded } +func (m *SubscriptionManager) ReconcileAll() { + m.queueReconcile(trackIDForReconcileSubscriptions) +} + func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) (*trackSubscription, bool) { m.lock.RLock() defer m.lock.RUnlock() @@ -628,9 +632,11 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w var relieveFromLimits bool switch subTrack.MediaTrack().Kind() { case livekit.TrackType_VIDEO: - relieveFromLimits = m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Dec() == m.params.SubscriptionLimitVideo-1 + videoCount := m.subscribedVideoCount.Dec() + relieveFromLimits = m.params.SubscriptionLimitVideo > 0 && videoCount == m.params.SubscriptionLimitVideo-1 case livekit.TrackType_AUDIO: - relieveFromLimits = m.params.SubscriptionLimitAudio > 0 && m.subscribedAudioCount.Dec() == m.params.SubscriptionLimitAudio-1 + audioCount := m.subscribedAudioCount.Dec() + relieveFromLimits = m.params.SubscriptionLimitAudio > 0 && audioCount == m.params.SubscriptionLimitAudio-1 } // remove from subscribedTo From fb5595880ae120c91662275406e67cfbdad06751 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 27 Feb 2024 12:23:40 +0530 Subject: [PATCH 20/73] Missed lock in codec translate (#2516) --- pkg/sfu/forwarder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 79d0c2ee5..7abc6ebb4 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1748,6 +1748,9 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in } func (f *Forwarder) TranslateCodecHeader(extPkt *buffer.ExtPacket, tpr *TranslationParamsRTP, outputBuffer []byte) (bool, int, int, error) { + f.lock.Lock() + defer f.lock.Unlock() + maybeRollback := func(isSwitching bool) { if isSwitching { f.vls.Rollback() From 7649e4ffab39143ba7ab41919e955f2cf4064a89 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 27 Feb 2024 15:45:32 +0530 Subject: [PATCH 21/73] Post data and signal stats once in 5 minutes (#2518) --- pkg/config/config.go | 5 +++-- pkg/telemetry/signalanddatastats.go | 25 ++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index c7d86c0bd..0530fb045 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -47,8 +47,9 @@ const ( StreamTrackerTypePacket StreamTrackerType = "packet" StreamTrackerTypeFrame StreamTrackerType = "frame" - StatsUpdateInterval = time.Second * 10 - TelemetryStatsUpdateInterval = time.Second * 30 + StatsUpdateInterval = time.Second * 10 + TelemetryStatsUpdateInterval = time.Second * 30 + TelemetryNonMediaStatsUpdateInterval = time.Minute * 5 ) var ( diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index 5ab5855ab..ffb223d55 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -20,6 +20,7 @@ import ( "go.uber.org/atomic" + "github.com/frostbyte73/core" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" @@ -53,7 +54,7 @@ type BytesTrackStats struct { totalSendBytes, totalRecvBytes atomic.Uint64 totalSendMessages, totalRecvMessages atomic.Uint32 telemetry TelemetryService - isStopped atomic.Bool + done core.Fuse } func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, telemetry TelemetryService) *BytesTrackStats { @@ -61,6 +62,7 @@ func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, tele trackID: trackID, pID: pID, telemetry: telemetry, + done: core.NewFuse(), } go s.reporter() return s @@ -91,7 +93,7 @@ func (s *BytesTrackStats) GetTrafficTotals() *TrafficTotals { } func (s *BytesTrackStats) Stop() { - s.isStopped.Store(true) + s.done.Break() } func (s *BytesTrackStats) report() { @@ -119,15 +121,20 @@ func (s *BytesTrackStats) report() { } func (s *BytesTrackStats) reporter() { - ticker := time.NewTicker(config.TelemetryStatsUpdateInterval) - defer ticker.Stop() - - for !s.isStopped.Load() { - <-ticker.C + ticker := time.NewTicker(config.TelemetryNonMediaStatsUpdateInterval) + defer func() { + ticker.Stop() s.report() - } + }() - s.report() + for { + select { + case <-s.done.Watch(): + return + case <-ticker.C: + s.report() + } + } } // ----------------------------------------------------------------------- From f5eb6c8a95af283a18ae9f478781087747670c42 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Wed, 28 Feb 2024 03:48:58 +0200 Subject: [PATCH 22/73] Update usage of core.Fuse. (#2519) --- .github/workflows/buildtest.yaml | 2 +- .github/workflows/docker.yaml | 2 +- .github/workflows/release.yaml | 2 +- go.mod | 9 ++++---- go.sum | 23 +++++++++++++++----- pkg/rtc/participant_traffic_load.go | 6 ++--- pkg/sfu/connectionquality/connectionstats.go | 4 ++-- pkg/sfu/streamtrackermanager.go | 6 ++--- pkg/telemetry/signalanddatastats.go | 4 ++-- 9 files changed, 35 insertions(+), 23 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 0b906e6f0..b48afc075 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.21" - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@v2.4.1 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 967b1f734..9b52542d7 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,7 +41,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.18' + go-version: '>=1.21' - name: Download Go modules run: go mod download diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2f37c7a87..ef0960712 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.18' + go-version: '>=1.21' - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 diff --git a/go.mod b/go.mod index 2695ebd61..246915a12 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/livekit/livekit-server -go 1.20 +go 1.21 require ( github.com/avast/retry-go/v4 v4.5.1 @@ -9,7 +9,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.3 - github.com/frostbyte73/core v0.0.9 + github.com/frostbyte73/core v0.0.10 github.com/gammazero/deque v0.2.1 github.com/gammazero/workerpool v1.1.3 github.com/google/wire v0.6.0 @@ -19,8 +19,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 - github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175 - github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58 + github.com/livekit/protocol v1.10.0 + github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -91,6 +91,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/puzpuzpuz/xsync v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect diff --git a/go.sum b/go.sum index ef9cde1da..4c4f84a4c 100644 --- a/go.sum +++ b/go.sum @@ -5,7 +5,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= @@ -34,8 +36,8 @@ github.com/florianl/go-tc v0.4.3 h1:xpobG2gFNvEqbclU07zjddALSjqTQTWJkxg5/kRYDpw= github.com/florianl/go-tc v0.4.3/go.mod h1:uvp6pIlOw7Z8hhfnT5M4+V1hHVgZWRZwwMS8Z0JsRxc= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frostbyte73/core v0.0.9 h1:AmE9GjgGpPsWk9ZkmY3HsYUs2hf2tZt+/W6r49URBQI= -github.com/frostbyte73/core v0.0.9/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= +github.com/frostbyte73/core v0.0.10 h1:D4DQXdPb8ICayz0n75rs4UYTXrUSdxzUfeleuNJORsU= +github.com/frostbyte73/core v0.0.10/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= @@ -117,6 +119,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -127,10 +130,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 h1:Zw88krOHni51OzDUlrduYb3m7VcsaKw06TnnDhsQpjg= github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175 h1:xX29sbgPxTmwUexxmDvNr+NgREHJEIvMOwuW870DatI= -github.com/livekit/protocol v1.9.10-0.20240224082149-454dff3b8175/go.mod h1:kT55fgJZgd1CAIO2w8tcc78ugw5xVjrSliy8pHGUeCw= -github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58 h1:yH55rBGLRO+ict2mu6bKZ5iPwTIrIwU1i0ydgThi4+k= -github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= +github.com/livekit/protocol v1.10.0 h1:HKBCitK7+Nuezktqv/h9h5AOllttsmNnZFpwIlAIRRw= +github.com/livekit/protocol v1.10.0/go.mod h1:NnlGwusu/SvwBxFe9Fpi9P2IKCA/V+kIqObZ3USWq0g= +github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 h1:bvjzDR+Rvdf3JgzQMtLiGVHBQ8KoOWM7x7sHj79jevQ= +github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -180,6 +183,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -238,16 +242,20 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= +github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= +github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -276,11 +284,13 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= @@ -442,6 +452,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/pkg/rtc/participant_traffic_load.go b/pkg/rtc/participant_traffic_load.go index 87b8b9ef2..e0fa7a4e5 100644 --- a/pkg/rtc/participant_traffic_load.go +++ b/pkg/rtc/participant_traffic_load.go @@ -19,10 +19,11 @@ import ( "time" "github.com/frostbyte73/core" - "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/telemetry" ) const ( @@ -51,7 +52,6 @@ func NewParticipantTrafficLoad(params ParticipantTrafficLoadParams) *Participant p := &ParticipantTrafficLoad{ params: params, tracksStatsMedia: make(map[livekit.TrackID]*livekit.RTPStats), - closed: core.NewFuse(), } go p.reporter() return p diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 1a977c650..16ac76d1d 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -23,9 +23,10 @@ import ( "github.com/pion/webrtc/v3" "go.uber.org/atomic" - "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( @@ -80,7 +81,6 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { IncludeJitter: params.IncludeJitter, Logger: params.Logger, }), - done: core.NewFuse(), } } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index a934b9e93..36ff551a2 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -25,11 +25,12 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" ) const ( @@ -98,7 +99,6 @@ func NewStreamTrackerManager( maxPublishedLayer: buffer.InvalidLayerSpatial, maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, - closed: core.NewFuse(), } s.trackInfo.Store(proto.Clone(trackInfo).(*livekit.TrackInfo)) diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index ffb223d55..f30f8bfe0 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -21,9 +21,10 @@ import ( "go.uber.org/atomic" "github.com/frostbyte73/core" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" + + "github.com/livekit/livekit-server/pkg/config" ) type BytesTrackType string @@ -62,7 +63,6 @@ func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, tele trackID: trackID, pID: pID, telemetry: telemetry, - done: core.NewFuse(), } go s.reporter() return s From 13151b24a04299a40ab9a685f20f574fc2c4550f Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 27 Feb 2024 17:58:56 -0800 Subject: [PATCH 23/73] Re-enable static check (#2520) --- .github/workflows/buildtest.yaml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index b48afc075..34a13ab6c 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -17,9 +17,9 @@ name: Test on: workflow_dispatch: push: - branches: [master] + branches: [ master ] pull_request: - branches: [master] + branches: [ master ] jobs: test: @@ -56,13 +56,11 @@ jobs: version: latest args: build - # - name: Static Check - # uses: dominikh/staticcheck-action@v1.3.0 - # with: - # checks: '["all", "-ST1000", "-ST1003", "-ST1020", "-ST1021", "-ST1022", "-SA1019"]' - # min-go-version: 1.18 - # version: 2022.1.3 - # install-go: false + - name: Static Check + uses: amarpal/staticcheck-action@master + with: + checks: '["all", "-ST1000", "-ST1003", "-ST1020", "-ST1021", "-ST1022", "-SA1019"]' + install-go: false - name: Test run: | From 5326263d445f7655bf10ccf8c0af4679518f355c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:33:15 -0800 Subject: [PATCH 24/73] Update module github.com/prometheus/client_golang to v1.19.0 (#2514) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 5 ++--- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 246915a12..2e84a9357 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pion/turn/v2 v2.1.5 github.com/pion/webrtc/v3 v3.2.28 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 github.com/rs/cors v1.10.1 github.com/stretchr/testify v1.8.4 @@ -75,7 +75,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect github.com/nats-io/nats.go v1.32.0 // indirect @@ -89,7 +88,7 @@ require ( github.com/pion/stun v0.6.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/puzpuzpuz/xsync v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 4c4f84a4c..1d8c2f69c 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,6 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= @@ -234,12 +232,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= From 20ac7fb4163ac2ca5fcda35f562a6bb040c22006 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 28 Feb 2024 12:16:34 +0800 Subject: [PATCH 25/73] Confirm migrated tracks info before declare migration completed (#2522) * Confirm migrated tracks info before declare migration completed * log expected mid --- pkg/rtc/participant.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index f037a46cb..3443ce2b1 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -709,7 +709,10 @@ func (p *ParticipantImpl) handleMigrateTracks() { if mt != nil { addedTracks = append(addedTracks, mt) } else { - p.pubLogger.Warnw("could not find migrated track", nil, "cid", cid) + p.pubLogger.Warnw("could not find migrated track, migration failed", nil, "cid", cid) + p.pendingTracksLock.Unlock() + p.IssueFullReconnect(types.ParticipantCloseReasonMigrateCodecMismatch) + return } } @@ -910,6 +913,9 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { } p.params.Logger.Debugw("SetMigrateState", "state", s) + if s == types.MigrateStateComplete { + p.handleMigrateTracks() + } p.migrateState.Store(s) p.dirty.Store(true) @@ -918,7 +924,7 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { p.TransportManager.ProcessPendingPublisherOffer() case types.MigrateStateComplete: - p.handleMigrateTracks() + p.TransportManager.ProcessPendingPublisherDataChannels() } @@ -1966,7 +1972,7 @@ func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *M p.pubLogger.Infow("add migrated track", "cid", cid, "trackID", ti.Sid, "track", logger.Proto(ti)) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { - p.pubLogger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid) + p.pubLogger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid, "mid", ti.Mid) return nil } From 098b12981fe01e4944e31f8e8c78852424142596 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 27 Feb 2024 20:22:38 -0800 Subject: [PATCH 26/73] fix pli throttle locking (#2521) * fix pli throttle locking * UpdatePliAndTime still used in cloud --- pkg/sfu/buffer/buffer.go | 10 +++++----- pkg/sfu/buffer/rtpstats_base.go | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 85f0b2ae9..2f920dd02 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -457,14 +457,14 @@ func (b *Buffer) SetPLIThrottle(duration int64) { func (b *Buffer) SendPLI(force bool) { b.RLock() - if (b.rtpStats == nil || b.rtpStats.TimeSinceLastPli() < b.pliThrottle) && !force { - b.RUnlock() + rtpStats := b.rtpStats + pliThrottle := b.pliThrottle + b.RUnlock() + + if (rtpStats == nil && !force) || !rtpStats.CheckAndUpdatePli(pliThrottle, force) { return } - b.rtpStats.UpdatePliAndTime(1) - b.RUnlock() - b.logger.Debugw("send pli", "ssrc", b.mediaSSRC, "force", force) pli := []rtcp.Packet{ &rtcp.PictureLossIndication{SenderSSRC: b.mediaSSRC, MediaSSRC: b.mediaSSRC}, diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index e2c406317..7fc6a5797 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -336,6 +336,18 @@ func (r *rtpStatsBase) UpdateNackProcessed(nackAckCount uint32, nackMissCount ui r.nackRepeated += nackRepeatedCount } +func (r *rtpStatsBase) CheckAndUpdatePli(throttle int64, force bool) bool { + r.lock.Lock() + defer r.lock.Unlock() + + if !r.endTime.IsZero() || (!force && time.Now().UnixNano()-r.lastPli.UnixNano() < throttle) { + return false + } + r.updatePliLocked(1) + r.updatePliTimeLocked() + return true +} + func (r *rtpStatsBase) UpdatePliAndTime(pliCount uint32) { r.lock.Lock() defer r.lock.Unlock() @@ -385,13 +397,6 @@ func (r *rtpStatsBase) LastPli() time.Time { return r.lastPli } -func (r *rtpStatsBase) TimeSinceLastPli() int64 { - r.lock.RLock() - defer r.lock.RUnlock() - - return time.Now().UnixNano() - r.lastPli.UnixNano() -} - func (r *rtpStatsBase) UpdateLayerLockPliAndTime(pliCount uint32) { r.lock.Lock() defer r.lock.Unlock() From 6519c7596a163bd1b176e328c535b9946814589b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 28 Feb 2024 12:11:40 +0530 Subject: [PATCH 27/73] Demote some logs. (#2523) When a fallback is not applied, it is due to signal interruption. ICE connection failing happens. And every time there is error, it is due to "no selected pair". Move all of it to `Debugw`. `setting ICE config` is the definitive log which says if a different ICE config was applied. --- pkg/rtc/transport.go | 8 +------- pkg/rtc/transportmanager.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 22983adf4..1765f865b 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -603,14 +603,8 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) { isShort, duration = t.IsShortConnection(time.Now()) if isShort { pair, err := t.getSelectedPair() - if err != nil { - t.params.Logger.Warnw("short ICE connection", err, "duration", duration) - } else { - t.params.Logger.Infow("short ICE connection", "pair", pair, "duration", duration) - } + t.params.Logger.Debugw("short ICE connection", "error", err, "pair", pair, "duration", duration) } - } else { - t.params.Logger.Infow("force short ICE connection") } t.params.Handler.OnFailed(isShort) diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 9dfbcce5f..cd1b23289 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -523,8 +523,11 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { if !t.hasRecentSignalLocked() || !signalValid { // the failed might cause by network interrupt because signal closed or we have not seen any signal in the time window, // so don't switch to next candidate type - t.params.Logger.Infow("ignoring prefer candidate check by ICE failure because signal connection interrupted", - "lastSignalSince", lastSignalSince, "signalValid", signalValid) + t.params.Logger.Debugw( + "ignoring prefer candidate check by ICE failure because signal connection interrupted", + "lastSignalSince", lastSignalSince, + "signalValid", signalValid, + ) t.failureCount = 0 t.lastFailure = time.Time{} t.lock.Unlock() @@ -572,13 +575,13 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { switch preferNext { case livekit.ICECandidateType_ICT_TCP: - t.params.Logger.Infow("prefer TCP transport on both peer connections") + t.params.Logger.Debugw("prefer TCP transport on both peer connections") case livekit.ICECandidateType_ICT_TLS: - t.params.Logger.Infow("prefer TLS transport both peer connections") + t.params.Logger.Debugw("prefer TLS transport both peer connections") case livekit.ICECandidateType_ICT_NONE: - t.params.Logger.Infow("allowing all transports on both peer connections") + t.params.Logger.Debugw("allowing all transports on both peer connections") } // irrespective of which one fails, force prefer candidate on both as the other one might From a4353682785d9b8568d5f728d6fd8b353496fc12 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 28 Feb 2024 16:24:23 +0800 Subject: [PATCH 28/73] use dynamic bucket size (#2524) --- go.mod | 2 +- go.sum | 4 +-- pkg/sfu/buffer/buffer.go | 68 +++++++++++++++++++++++------------ pkg/sfu/buffer/buffer_test.go | 38 ++++++++------------ pkg/sfu/buffer/factory.go | 42 ++++++++-------------- 5 files changed, 79 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 2e84a9357..1552338a6 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 + github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef github.com/livekit/protocol v1.10.0 github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index 1d8c2f69c..1f87a241b 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 h1:Zw88krOHni51OzDUlrduYb3m7VcsaKw06TnnDhsQpjg= -github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= +github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef h1:Db/UItb+Cvm1trBRJiEZOdRSyss+LDY4e8gU9aE4GRc= +github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= github.com/livekit/protocol v1.10.0 h1:HKBCitK7+Nuezktqv/h9h5AOllttsmNnZFpwIlAIRRw= github.com/livekit/protocol v1.10.0/go.mod h1:NnlGwusu/SvwBxFe9Fpi9P2IKCA/V+kIqObZ3USWq0g= github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 h1:bvjzDR+Rvdf3JgzQMtLiGVHBQ8KoOWM7x7sHj79jevQ= diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 2f920dd02..06c27b6bf 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -44,6 +44,9 @@ import ( const ( ReportDelta = time.Second + + InitPacketBufferSizeVideo = 300 + InitPacketBufferSizeAudio = 70 ) type pendingPacket struct { @@ -68,8 +71,8 @@ type Buffer struct { sync.RWMutex bucket *bucket.Bucket nacker *nack.NackQueue - videoPool *sync.Pool - audioPool *sync.Pool + maxVideoPkts int + maxAudioPkts int codecType webrtc.RTPCodecType payloadType uint8 extPackets deque.Deque[*ExtPacket] @@ -100,6 +103,7 @@ type Buffer struct { rtpStats *RTPStatsReceiver rrSnapshotId uint32 deltaStatsSnapshotId uint32 + ppsSnapshortId uint32 lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only @@ -126,18 +130,19 @@ type Buffer struct { extPacketTooMuchCount atomic.Uint32 primaryBufferForRTX *Buffer + rtxPktBuf []byte } // NewBuffer constructs a new Buffer -func NewBuffer(ssrc uint32, vp, ap *sync.Pool) *Buffer { +func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer { l := logger.GetLogger() // will be reset with correct context via SetLogger b := &Buffer{ - mediaSSRC: ssrc, - videoPool: vp, - audioPool: ap, - snRangeMap: utils.NewRangeMap[uint64, uint64](100), - pliThrottle: int64(500 * time.Millisecond), - logger: l.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU), + mediaSSRC: ssrc, + maxVideoPkts: maxVideoPkts, + maxAudioPkts: maxAudioPkts, + snRangeMap: utils.NewRangeMap[uint64, uint64](100), + pliThrottle: int64(500 * time.Millisecond), + logger: l.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU), } b.extPackets.SetMinCapacity(7) return b @@ -188,6 +193,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili }) b.rrSnapshotId = b.rtpStats.NewSnapshotId() b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() + b.ppsSnapshortId = b.rtpStats.NewSnapshotId() b.clockRate = codec.ClockRate b.lastReport = time.Now() @@ -225,10 +231,10 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili switch { case strings.HasPrefix(b.mime, "audio/"): b.codecType = webrtc.RTPCodecTypeAudio - b.bucket = bucket.NewBucket(b.audioPool.Get().(*[]byte)) + b.bucket = bucket.NewBucket(InitPacketBufferSizeAudio) case strings.HasPrefix(b.mime, "video/"): b.codecType = webrtc.RTPCodecTypeVideo - b.bucket = bucket.NewBucket(b.videoPool.Get().(*[]byte)) + b.bucket = bucket.NewBucket(InitPacketBufferSizeVideo) if b.frameRateCalculator[0] == nil { if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) { b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) @@ -347,22 +353,23 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) { return } - videoPktPtr := b.videoPool.Get().(*[]byte) - defer b.videoPool.Put(videoPktPtr) + if b.rtxPktBuf == nil { + b.rtxPktBuf = make([]byte, bucket.MaxPktSize) + } videoPkt := *rtxPkt videoPkt.PayloadType = b.payloadType videoPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) videoPkt.SSRC = b.mediaSSRC videoPkt.Payload = rtxPkt.Payload[2:] - n, err = videoPkt.MarshalTo((*videoPktPtr)) + n, err = videoPkt.MarshalTo(b.rtxPktBuf) if err != nil { b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", videoPkt.SequenceNumber) return } - b.calc((*videoPktPtr)[:n], &videoPkt, time.Now(), true) + b.calc(b.rtxPktBuf[:n], &videoPkt, time.Now(), true) return } @@ -417,13 +424,6 @@ func (b *Buffer) Close() error { defer b.Unlock() b.closeOnce.Do(func() { - if b.bucket != nil && b.codecType == webrtc.RTPCodecTypeVideo { - b.videoPool.Put(b.bucket.Src()) - } - if b.bucket != nil && b.codecType == webrtc.RTPCodecTypeAudio { - b.audioPool.Put(b.bucket.Src()) - } - b.closed.Store(true) if b.rtpStats != nil { @@ -776,6 +776,30 @@ func (b *Buffer) doReports(arrivalTime time.Time) { if pkts != nil && b.onRtcpFeedback != nil { b.onRtcpFeedback(pkts) } + + b.mayGrowBucket() +} + +func (b *Buffer) mayGrowBucket() { + cap := b.bucket.Capacity() + maxPkts := b.maxVideoPkts + if b.codecType == webrtc.RTPCodecTypeAudio { + maxPkts = b.maxAudioPkts + } + if cap >= maxPkts { + return + } + oldCap := cap + deltaInfo := b.rtpStats.DeltaInfo(b.deltaStatsSnapshotId) + if deltaInfo != nil && deltaInfo.Duration > 500*time.Millisecond { + pps := int(time.Duration(deltaInfo.Packets) * time.Second / deltaInfo.Duration) + for pps > cap && cap < maxPkts { + cap = b.bucket.Grow() + } + if cap > oldCap { + b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) + } + } } func (b *Buffer) buildNACKPacket() ([]rtcp.Packet, int) { diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 7f685186c..2125ca907 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -48,15 +48,8 @@ var opusCodec = webrtc.RTPCodecParameters{ } func TestNack(t *testing.T) { - pool := &sync.Pool{ - New: func() interface{} { - b := make([]byte, 1500) - return &b - }, - } - t.Run("nack normal", func(t *testing.T) { - buff := NewBuffer(123, pool, pool) + buff := NewBuffer(123, 1, 1) buff.codecType = webrtc.RTPCodecTypeVideo require.NotNil(t, buff) var wg sync.WaitGroup @@ -101,7 +94,7 @@ func TestNack(t *testing.T) { }) t.Run("nack with seq wrap", func(t *testing.T) { - buff := NewBuffer(123, pool, pool) + buff := NewBuffer(123, 1, 1) buff.codecType = webrtc.RTPCodecTypeVideo require.NotNil(t, buff) var wg sync.WaitGroup @@ -193,13 +186,7 @@ func TestNewBuffer(t *testing.T) { }, }, } - pool := &sync.Pool{ - New: func() interface{} { - b := make([]byte, 1500) - return &b - }, - } - buff := NewBuffer(123, pool, pool) + buff := NewBuffer(123, 1, 1) buff.codecType = webrtc.RTPCodecTypeVideo require.NotNil(t, buff) buff.OnRtcpFeedback(func(_ []rtcp.Packet) {}) @@ -219,13 +206,7 @@ func TestNewBuffer(t *testing.T) { } func TestFractionLostReport(t *testing.T) { - pool := &sync.Pool{ - New: func() interface{} { - b := make([]byte, 1500) - return &b - }, - } - buff := NewBuffer(123, pool, pool) + buff := NewBuffer(123, 1, 1) require.NotNil(t, buff) buff.codecType = webrtc.RTPCodecTypeVideo var wg sync.WaitGroup @@ -261,3 +242,14 @@ func TestFractionLostReport(t *testing.T) { } wg.Wait() } + +func BenchmarkMemcpu(b *testing.B) { + buf := make([]byte, 1500*1500*10) + buf2 := make([]byte, 1500*1500*20) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + copy(buf2, buf) + } + +} diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go index 2520c6cce..2785f95ae 100644 --- a/pkg/sfu/buffer/factory.go +++ b/pkg/sfu/buffer/factory.go @@ -19,49 +19,37 @@ import ( "sync" "github.com/pion/transport/v2/packetio" - - "github.com/livekit/mediatransportutil/pkg/bucket" ) type FactoryOfBufferFactory struct { - videoPool *sync.Pool - audioPool *sync.Pool + trackingPacketsVideo int + trackingPacketsAudio int } func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory { return &FactoryOfBufferFactory{ - videoPool: &sync.Pool{ - New: func() interface{} { - b := make([]byte, trackingPacketsVideo*bucket.MaxPktSize) - return &b - }, - }, - audioPool: &sync.Pool{ - New: func() interface{} { - b := make([]byte, trackingPacketsAudio*bucket.MaxPktSize) - return &b - }, - }, + trackingPacketsVideo: trackingPacketsVideo, + trackingPacketsAudio: trackingPacketsAudio, } } func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory { return &Factory{ - videoPool: f.videoPool, - audioPool: f.audioPool, - rtpBuffers: make(map[uint32]*Buffer), - rtcpReaders: make(map[uint32]*RTCPReader), - rtxPair: make(map[uint32]uint32), + trackingPacketsVideo: f.trackingPacketsVideo, + trackingPacketsAudio: f.trackingPacketsAudio, + rtpBuffers: make(map[uint32]*Buffer), + rtcpReaders: make(map[uint32]*RTCPReader), + rtxPair: make(map[uint32]uint32), } } type Factory struct { sync.RWMutex - videoPool *sync.Pool - audioPool *sync.Pool - rtpBuffers map[uint32]*Buffer - rtcpReaders map[uint32]*RTCPReader - rtxPair map[uint32]uint32 // repair -> base + trackingPacketsVideo int + trackingPacketsAudio int + rtpBuffers map[uint32]*Buffer + rtcpReaders map[uint32]*RTCPReader + rtxPair map[uint32]uint32 // repair -> base } func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser { @@ -84,7 +72,7 @@ func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io if reader, ok := f.rtpBuffers[ssrc]; ok { return reader } - buffer := NewBuffer(ssrc, f.videoPool, f.audioPool) + buffer := NewBuffer(ssrc, f.trackingPacketsVideo, f.trackingPacketsAudio) f.rtpBuffers[ssrc] = buffer for repair, base := range f.rtxPair { if repair == ssrc { From af79224e3055d1f95d76ba4edfbb62f6a6179967 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 29 Feb 2024 10:25:52 +0530 Subject: [PATCH 29/73] Check on copy. (#2525) Should not happen, but just being defensive. --- pkg/sfu/downtrack.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index a4d0b1be5..18589517e 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -89,9 +89,9 @@ var ( ErrOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") ErrPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") ErrDuplicatePacket = errors.New("duplicate packet") - ErrSequenceNumberOffsetNotFound = errors.New("sequence number offset not found") ErrPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") ErrDownTrackAlreadyBound = errors.New("already bound") + ErrPayloadOverflow = errors.New("payload overflow") ) var ( @@ -694,8 +694,13 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { PacketFactory.Put(poolEntity) return err } - copy(payload[outgoingHeaderSize:], extPkt.Packet.Payload[incomingHeaderSize:]) - payload = payload[:outgoingHeaderSize+len(extPkt.Packet.Payload)-incomingHeaderSize] + n := copy(payload[outgoingHeaderSize:], extPkt.Packet.Payload[incomingHeaderSize:]) + if n != len(extPkt.Packet.Payload[incomingHeaderSize:]) { + d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[incomingHeaderSize:]), "have", n) + PacketFactory.Put(poolEntity) + return ErrPayloadOverflow + } + payload = payload[:outgoingHeaderSize+n] hdr, err := d.getTranslatedRTPHeader(extPkt, &tp) if err != nil { From ea66eae9f565cc4b2f6b2d73a418867ea2b6db74 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 29 Feb 2024 14:35:19 +0530 Subject: [PATCH 30/73] Start moving things to structured logging (#2527) --- pkg/sfu/connectionquality/scorer.go | 18 +++++++++++++++ pkg/sfu/downtrack.go | 4 ++-- pkg/sfu/forwarder.go | 36 ++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index d5c3b019c..33421fcb9 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -23,6 +23,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "go.uber.org/zap/zapcore" ) const ( @@ -159,6 +160,23 @@ func (w *windowStat) String() string { ) } +func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error { + if w == nil { + return nil + } + + e.AddTime("startedAt", w.startedAt) + e.AddString("duration", w.duration.String()) + e.AddUint32("packetsExpected", w.packetsExpected) + e.AddUint32("packetsLost", w.packetsLost) + e.AddUint32("packetsMissing", w.packetsMissing) + e.AddUint32("packetsOutOfOrder", w.packetsOutOfOrder) + e.AddUint64("bytes", w.bytes) + e.AddUint32("rttMax", w.rttMax) + e.AddFloat64("jitterMax", w.jitterMax) + return nil +} + // ------------------------------------------ type qualityScorerParams struct { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 18589517e..f25cd408f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1211,7 +1211,7 @@ func (d *DownTrack) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b transition, availableLayers, brs := d.forwarder.ProvisionalAllocateGetCooperativeTransition(allowOvershoot) d.params.Logger.Debugw( "stream: cooperative transition", - "transition", transition, + "transition", &transition, "availableLayers", availableLayers, "bitrates", brs, ) @@ -1222,7 +1222,7 @@ func (d *DownTrack) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti transition, availableLayers, brs := d.forwarder.ProvisionalAllocateGetBestWeightedTransition() d.params.Logger.Debugw( "stream: best weighted transition", - "transition", transition, + "transition", &transition, "availableLayers", availableLayers, "bitrates", brs, ) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 7abc6ebb4..91596b723 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -25,6 +25,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v3" + "go.uber.org/zap/zapcore" "github.com/livekit/protocol/logger" @@ -92,7 +93,7 @@ type VideoAllocation struct { DistanceToDesired float64 } -func (v VideoAllocation) String() string { +func (v *VideoAllocation) String() string { return fmt.Sprintf("VideoAllocation{pause: %s, def: %+v, bwr: %d, del: %d, bwn: %d, rates: %+v, target: %s, req: %d, max: %s, dist: %0.2f}", v.PauseReason, v.IsDeficient, @@ -107,6 +108,24 @@ func (v VideoAllocation) String() string { ) } +func (v *VideoAllocation) MarshalLogObject(e zapcore.ObjectEncoder) error { + if v == nil { + return nil + } + + e.AddString("PauseReason", v.PauseReason.String()) + e.AddBool("IsDeficient", v.IsDeficient) + e.AddInt64("BandwidthRquested", v.BandwidthRequested) + e.AddInt64("BandwidthDelta", v.BandwidthDelta) + e.AddInt64("BandwidthNeeded", v.BandwidthNeeded) + e.AddReflected("Bitrates", v.Bitrates) + e.AddReflected("TargetLayer", v.TargetLayer) + e.AddInt32("RequestLayerSpatial", v.RequestLayerSpatial) + e.AddReflected("MaxLayer", v.MaxLayer) + e.AddFloat64("DistanceToDesired", v.DistanceToDesired) + return nil +} + var ( VideoAllocationDefault = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen @@ -137,10 +156,21 @@ type VideoTransition struct { BandwidthDelta int64 } -func (v VideoTransition) String() string { +func (v *VideoTransition) String() string { return fmt.Sprintf("VideoTransition{from: %s, to: %s, del: %d}", v.From, v.To, v.BandwidthDelta) } +func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error { + if v == nil { + return nil + } + + e.AddReflected("From", v.From) + e.AddReflected("To", v.To) + e.AddInt64("BandwidthDelta", v.BandwidthDelta) + return nil +} + // ------------------------------------------------------------------- type TranslationParams struct { @@ -1359,7 +1389,7 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video alloc.PauseReason != f.lastAllocation.PauseReason || alloc.TargetLayer != f.lastAllocation.TargetLayer || alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial { - f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", &alloc) } f.lastAllocation = alloc From d40041d01353974cd9c51a1a3d31bf65bf0d1c1c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 29 Feb 2024 22:48:36 +0530 Subject: [PATCH 31/73] Use the correct snapshot id for PPS. (#2528) * Use the correct snapshot id for PPS. That caused connection quality to operate on small windows. * remove debug --- pkg/sfu/buffer/buffer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 06c27b6bf..b2e6b75a7 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -103,7 +103,7 @@ type Buffer struct { rtpStats *RTPStatsReceiver rrSnapshotId uint32 deltaStatsSnapshotId uint32 - ppsSnapshortId uint32 + ppsSnapshotId uint32 lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only @@ -193,7 +193,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili }) b.rrSnapshotId = b.rtpStats.NewSnapshotId() b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() - b.ppsSnapshortId = b.rtpStats.NewSnapshotId() + b.ppsSnapshotId = b.rtpStats.NewSnapshotId() b.clockRate = codec.ClockRate b.lastReport = time.Now() @@ -790,7 +790,7 @@ func (b *Buffer) mayGrowBucket() { return } oldCap := cap - deltaInfo := b.rtpStats.DeltaInfo(b.deltaStatsSnapshotId) + deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId) if deltaInfo != nil && deltaInfo.Duration > 500*time.Millisecond { pps := int(time.Duration(deltaInfo.Packets) * time.Second / deltaInfo.Duration) for pps > cap && cap < maxPkts { From 00db7bc79b8454f6d593caf09dd1db672d1b1206 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 29 Feb 2024 21:11:24 -0800 Subject: [PATCH 32/73] Update README.md (#2530) * Update README.md Link to non-deprecated repo * Cleanup --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8207dc18..e79fb3d07 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ enabling you to build automations that behave like end-users. | JavaScript (TypeScript) | [server-sdk-js](https://github.com/livekit/server-sdk-js) | [docs](https://docs.livekit.io/server-sdk-js/) | | Ruby | [server-sdk-ruby](https://github.com/livekit/server-sdk-ruby) | | | Java (Kotlin) | [server-sdk-kotlin](https://github.com/livekit/server-sdk-kotlin) | | -| Python (community) | [tradablebits/livekit-server-sdk-python](https://github.com/tradablebits/livekit-server-sdk-python) | | +| Python (community) | [python-sdks](https://github.com/livekit/python-sdks) | | | PHP (community) | [agence104/livekit-server-sdk-php](https://github.com/agence104/livekit-server-sdk-php) | | ### Tools From 95011d64f8286987967d332f2161b3677735098c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Mar 2024 11:27:26 +0530 Subject: [PATCH 33/73] Annotate max subscribe layer change with events. (#2531) Using events to understand what is causing max subscribed layer changes. Seeing an edge case where all layers are disabled incorrectly. --- pkg/sfu/downtrack.go | 21 ++++++++++++--------- pkg/sfu/forwarder.go | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index f25cd408f..5cff248bb 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -274,7 +274,7 @@ type DownTrack struct { pacer pacer.Pacer maxLayerNotifierChMu sync.RWMutex - maxLayerNotifierCh chan struct{} + maxLayerNotifierCh chan string maxLayerNotifierChClosed bool keyFrameRequesterChMu sync.RWMutex @@ -308,7 +308,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { kind: kind, codec: codecs[0].RTPCodecCapability, pacer: params.Pacer, - maxLayerNotifierCh: make(chan struct{}, 1), + maxLayerNotifierCh: make(chan string, 1), keyFrameRequesterCh: make(chan struct{}, 1), } d.forwarder = NewForwarder( @@ -642,7 +642,7 @@ func (d *DownTrack) keyFrameRequester() { } } -func (d *DownTrack) postMaxLayerNotifierEvent() { +func (d *DownTrack) postMaxLayerNotifierEvent(event string) { if d.kind != webrtc.RTPCodecTypeVideo { return } @@ -650,24 +650,27 @@ func (d *DownTrack) postMaxLayerNotifierEvent() { d.maxLayerNotifierChMu.RLock() if !d.maxLayerNotifierChClosed { select { - case d.maxLayerNotifierCh <- struct{}{}: + case d.maxLayerNotifierCh <- event: default: + d.params.Logger.Debugw("max layer notifier channel busy", "event", event) } } d.maxLayerNotifierChMu.RUnlock() } func (d *DownTrack) maxLayerNotifierWorker() { + var event string more := true for more { - _, more = <-d.maxLayerNotifierCh + event, more = <-d.maxLayerNotifierCh maxLayerSpatial := buffer.InvalidLayerSpatial if more { maxLayerSpatial = d.forwarder.GetMaxSubscribedSpatial() + d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) } if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw("max subscribed layer changed", "maxLayerSpatial", maxLayerSpatial) + d.params.Logger.Debugw("notifying max subscribed layer", "layer", maxLayerSpatial, "event", event) onMaxSubscribedLayerChanged(d, maxLayerSpatial) } } @@ -903,7 +906,7 @@ func (d *DownTrack) handleMute(muted bool, changed bool) { // Note that while publisher mute is active, subscriber changes can also happen // and that could turn on/off layers on publisher side. // - d.postMaxLayerNotifierEvent() + d.postMaxLayerNotifierEvent("mute") if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) @@ -1015,7 +1018,7 @@ func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { return } - d.postMaxLayerNotifierEvent() + d.postMaxLayerNotifierEvent("max-subscribed") if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscribedLayerChanged(d, maxLayer) @@ -1961,7 +1964,7 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa if spmd.tp != nil { if spmd.tp.isSwitching { - d.postMaxLayerNotifierEvent() + d.postMaxLayerNotifierEvent("switching") } if spmd.tp.isResuming { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 91596b723..3e684e39d 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -456,7 +456,7 @@ func (f *Forwarder) PubMute(pubMuted bool) bool { return false } - f.logger.Debugw("setting forwarder pub mute", "pubMuted", pubMuted) + f.logger.Debugw("setting forwarder pub mute", "muted", pubMuted) f.pubMuted = pubMuted // resync when pub muted so that sequence numbers do not jump on unmute From 9e334c751099d142c1048331e43814ea10336dc4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Mar 2024 15:08:07 +0530 Subject: [PATCH 34/73] Refactoring channel handling (#2532) * Refactoring channel handling * Add a version to SubscriberID() --- pkg/rtc/mediatracksubscriptions.go | 2 +- pkg/sfu/downtrack.go | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 3f92e922d..652e2fcfb 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -190,7 +190,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * downTrack.OnMaxLayerChanged(func(dt *sfu.DownTrack, layer int32) { if t.onSubscriberMaxQualityChange != nil { - t.onSubscriberMaxQualityChange(subscriberID, dt.Codec(), layer) + t.onSubscriberMaxQualityChange(dt.SubscriberID(), dt.Codec(), layer) } }) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 5cff248bb..23f7433cb 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -286,6 +286,8 @@ type DownTrack struct { onMaxSubscribedLayerChanged func(dt *DownTrack, layer int32) onRttUpdate func(dt *DownTrack, rtt uint32) onCloseHandler func(willBeResumed bool) + + createdAt int64 } // NewDownTrack returns a DownTrack. @@ -310,6 +312,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { pacer: params.Pacer, maxLayerNotifierCh: make(chan string, 1), keyFrameRequesterCh: make(chan struct{}, 1), + createdAt: time.Now().UnixNano(), } d.forwarder = NewForwarder( d.kind, @@ -524,7 +527,10 @@ func (d *DownTrack) Codec() webrtc.RTPCodecCapability { return d.codec } // StreamID is the group this track belongs too. This must be unique func (d *DownTrack) StreamID() string { return d.params.StreamID } -func (d *DownTrack) SubscriberID() livekit.ParticipantID { return d.params.SubID } +func (d *DownTrack) SubscriberID() livekit.ParticipantID { + // add `createdAt` to ensure repeated subscriptions from same subscrober to same publisher does not collide + return livekit.ParticipantID(fmt.Sprintf("%s:%d", d.params.SubID, d.createdAt)) +} // Sets RTP header extensions for this track func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { @@ -659,21 +665,20 @@ func (d *DownTrack) postMaxLayerNotifierEvent(event string) { } func (d *DownTrack) maxLayerNotifierWorker() { - var event string - more := true - for more { - event, more = <-d.maxLayerNotifierCh + for event := range d.maxLayerNotifierCh { + maxLayerSpatial := d.forwarder.GetMaxSubscribedSpatial() + d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) - maxLayerSpatial := buffer.InvalidLayerSpatial - if more { - maxLayerSpatial = d.forwarder.GetMaxSubscribedSpatial() - d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) - } if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { d.params.Logger.Debugw("notifying max subscribed layer", "layer", maxLayerSpatial, "event", event) onMaxSubscribedLayerChanged(d, maxLayerSpatial) } } + + if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { + d.params.Logger.Debugw("notifying max subscribed layer", "layer", buffer.InvalidLayerSpatial, "event", "close") + onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) + } } // WriteRTP writes an RTP Packet to the DownTrack From 6a0b72c35746c247e732d2714bc566232df055ff Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Mar 2024 15:21:00 +0530 Subject: [PATCH 35/73] Use SubscriberID() is calls to `DeleteDownTrack`. (#2533) Thank you @cnderrauber for catching this. --- pkg/rtc/wrappedreceiver.go | 6 +++--- pkg/sfu/downtrack.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index a887ef832..677da8c3d 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -263,13 +263,13 @@ func (d *DummyReceiver) AddDownTrack(track sfu.TrackSender) error { return nil } -func (d *DummyReceiver) DeleteDownTrack(participantID livekit.ParticipantID) { +func (d *DummyReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) { d.downtrackLock.Lock() defer d.downtrackLock.Unlock() if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - r.DeleteDownTrack(participantID) + r.DeleteDownTrack(subscriberID) } else { - delete(d.downtracks, participantID) + delete(d.downtracks, subscriberID) } } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 23f7433cb..79abf6242 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -981,7 +981,7 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.onBindAndConnectedChange() d.params.Logger.Debugw("closing sender", "kind", d.kind) } - d.params.Receiver.DeleteDownTrack(d.params.SubID) + d.params.Receiver.DeleteDownTrack(d.SubscriberID()) if d.rtcpReader != nil && flush { d.params.Logger.Debugw("downtrack close rtcp reader") From 9a6d25674dbe970dfa0999d1adcded2cb3ac1e26 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Fri, 1 Mar 2024 19:29:32 +0200 Subject: [PATCH 36/73] Validate SIP trunks and rules when creating new ones. (#2535) --- go.mod | 6 ++--- go.sum | 14 +++++------ pkg/service/sip.go | 59 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 1552338a6..6d1482941 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef - github.com/livekit/protocol v1.10.0 + github.com/livekit/protocol v1.10.1 github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -47,6 +47,7 @@ require ( github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 + go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.32.0 @@ -61,7 +62,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -95,7 +96,6 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.21.0 // indirect diff --git a/go.sum b/go.sum index 1f87a241b..91b5c68b8 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= -github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= -github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.2 h1:2Edjn8Nrb44UvTdp84KU0bBPs1cO7noRCybtS3eJEUQ= +github.com/go-jose/go-jose/v3 v3.0.2/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -64,12 +64,12 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= @@ -130,8 +130,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef h1:Db/UItb+Cvm1trBRJiEZOdRSyss+LDY4e8gU9aE4GRc= github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.10.0 h1:HKBCitK7+Nuezktqv/h9h5AOllttsmNnZFpwIlAIRRw= -github.com/livekit/protocol v1.10.0/go.mod h1:NnlGwusu/SvwBxFe9Fpi9P2IKCA/V+kIqObZ3USWq0g= +github.com/livekit/protocol v1.10.1 h1:upe6pKRqH8wpsMuR2OLtgizEm94iia3pDYm3O4/2PRY= +github.com/livekit/protocol v1.10.1/go.mod h1:eWPz45pnxwpCwB84qqhHxG0bCRgasa2itN6GAHCDddc= github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 h1:bvjzDR+Rvdf3JgzQMtLiGVHBQ8KoOWM7x7sHj79jevQ= github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -260,7 +260,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -294,7 +294,6 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -407,6 +406,7 @@ golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 0ad76b207..dba99ecc4 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -16,11 +16,13 @@ package service import ( "context" + "fmt" "time" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/sip" "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" @@ -60,19 +62,34 @@ func (s *SIPService) CreateSIPTrunk(ctx context.Context, req *livekit.CreateSIPT if s.store == nil { return nil, ErrSIPNotConnected } - - info := &livekit.SIPTrunkInfo{ - SipTrunkId: utils.NewGuid(utils.SIPTrunkPrefix), - InboundAddresses: req.InboundAddresses, - OutboundAddress: req.OutboundAddress, - OutboundNumber: req.OutboundNumber, - InboundNumbersRegex: req.InboundNumbersRegex, - InboundUsername: req.InboundUsername, - InboundPassword: req.InboundPassword, - OutboundUsername: req.OutboundUsername, - OutboundPassword: req.OutboundPassword, + if len(req.InboundNumbersRegex) != 0 { + return nil, fmt.Errorf("Trunks with InboundNumbersRegex are deprecated. Use InboundNumbers instead.") } + // Keep ID empty, so that validation can print "" instead of a non-existent ID in the error. + info := &livekit.SIPTrunkInfo{ + InboundAddresses: req.InboundAddresses, + OutboundAddress: req.OutboundAddress, + OutboundNumber: req.OutboundNumber, + InboundNumbers: req.InboundNumbers, + InboundUsername: req.InboundUsername, + InboundPassword: req.InboundPassword, + OutboundUsername: req.OutboundUsername, + OutboundPassword: req.OutboundPassword, + } + + // Validate all trunks including the new one first. + list, err := s.store.ListSIPTrunk(ctx) + if err != nil { + return nil, err + } + list = append(list, info) + if err = sip.ValidateTrunks(list); err != nil { + return nil, err + } + + // Now we can generate ID and store. + info.SipTrunkId = utils.NewGuid(utils.SIPTrunkPrefix) if err := s.store.StoreSIPTrunk(ctx, info); err != nil { return nil, err } @@ -114,13 +131,25 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre return nil, ErrSIPNotConnected } + // Keep ID empty, so that validation can print "" instead of a non-existent ID in the error. info := &livekit.SIPDispatchRuleInfo{ - SipDispatchRuleId: utils.NewGuid(utils.SIPDispatchRulePrefix), - Rule: req.Rule, - TrunkIds: req.TrunkIds, - HidePhoneNumber: req.HidePhoneNumber, + Rule: req.Rule, + TrunkIds: req.TrunkIds, + HidePhoneNumber: req.HidePhoneNumber, } + // Validate all rules including the new one first. + list, err := s.store.ListSIPDispatchRule(ctx) + if err != nil { + return nil, err + } + list = append(list, info) + if err = sip.ValidateDispatchRules(list); err != nil { + return nil, err + } + + // Now we can generate ID and store. + info.SipDispatchRuleId = utils.NewGuid(utils.SIPDispatchRulePrefix) if err := s.store.StoreSIPDispatchRule(ctx, info); err != nil { return nil, err } From dbe2af38868759fe5ac5318ffdc0e0e5dd102a36 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 2 Mar 2024 23:12:20 +0530 Subject: [PATCH 37/73] Remove subscriber if track closed while adding subscriber. (#2537) * Remove subscriber if track closed while adding subscriber. It is possible that the track is closed when subscriber add is processed. That subscriber would have been dangling off a closed track. Check again after adding subscriber if track is closed. If it is, remove the subscriber and return error so that subscription manager re-resolves. * oops, wrong unlock --- pkg/rtc/mediatrackreceiver.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index b0bb43ed5..0283488a0 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -102,6 +102,7 @@ type MediaTrackReceiver struct { trackInfo *livekit.TrackInfo potentialCodecs []webrtc.RTPCodecParameters state mediaTrackReceiverState + willBeResumed bool onSetupReceiver func(mime string) onMediaLossFeedback func(dt *sfu.DownTrack, report *rtcp.ReceiverReport) @@ -275,6 +276,8 @@ func (t *MediaTrackReceiver) ClearAllReceivers(willBeResumed bool) { t.lock.Lock() receivers := t.receivers t.receivers = nil + + t.willBeResumed = willBeResumed t.lock.Unlock() for _, r := range receivers { @@ -482,7 +485,24 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su Logger: tLogger, DisableRed: t.trackInfo.GetDisableRed() || !t.params.AudioConfig.ActiveREDEncoding, }) - return t.MediaTrackSubscriptions.AddSubscriber(sub, wr) + subTrack, err := t.MediaTrackSubscriptions.AddSubscriber(sub, wr) + + // media track could have been closed while adding subscription + remove := false + willBeResumed := false + t.lock.RLock() + if t.state != mediaTrackReceiverStateOpen { + willBeResumed = t.willBeResumed + remove = true + } + t.lock.RUnlock() + + if remove { + _ = t.MediaTrackSubscriptions.RemoveSubscriber(sub.ID(), willBeResumed) + return nil, ErrNotOpen + } + + return subTrack, err } // RemoveSubscriber removes participant from subscription From cbc1d1b1cee79fb8718efe3ef8745e7abbd56164 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:03:57 -0800 Subject: [PATCH 38/73] Update go deps (#2529) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6d1482941..f803a07a2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/avast/retry-go/v4 v4.5.1 github.com/bep/debounce v1.2.1 - github.com/d5/tengo/v2 v2.16.1 + github.com/d5/tengo/v2 v2.17.0 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.3 @@ -40,7 +40,7 @@ require ( github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 github.com/rs/cors v1.10.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 diff --git a/go.sum b/go.sum index 91b5c68b8..80604eeea 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMt github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d5/tengo/v2 v2.16.1 h1:/N6dqiGu9toqANInZEOQMM8I06icdZnmb+81DG/lZdw= -github.com/d5/tengo/v2 v2.16.1/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= +github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= +github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -265,8 +265,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= From b58307f1449c6e9ed55bc4de03000e98c94d9c19 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Sun, 3 Mar 2024 12:34:49 +0800 Subject: [PATCH 39/73] Support XR request/response for rtt calculation (#2536) * Support XR request/response for rtt calculation * Update pkg/sfu/downtrack.go Co-authored-by: David Zhao --------- Co-authored-by: David Zhao --- go.mod | 4 +- go.sum | 8 +- pkg/rtc/mediatrack.go | 32 +++++++- pkg/rtc/mediatracksubscriptions.go | 1 + pkg/rtc/transport.go | 10 ++- pkg/rtc/types/interfaces.go | 2 + .../typesfakes/fake_local_participant.go | 79 +++++++++++++++++++ pkg/sfu/downtrack.go | 29 +++++++ 8 files changed, 155 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index f803a07a2..669c5ea85 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef + github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 github.com/livekit/protocol v1.10.1 github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 github.com/mackerelio/go-osstat v0.2.4 @@ -29,7 +29,7 @@ require ( github.com/pion/dtls/v2 v2.2.10 github.com/pion/ice/v2 v2.3.14 github.com/pion/interceptor v0.1.25 - github.com/pion/rtcp v1.2.13 + github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.3 github.com/pion/sctp v1.8.12 github.com/pion/sdp/v3 v3.0.6 diff --git a/go.sum b/go.sum index 80604eeea..dc3e827a7 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef h1:Db/UItb+Cvm1trBRJiEZOdRSyss+LDY4e8gU9aE4GRc= -github.com/livekit/mediatransportutil v0.0.0-20240228075855-6fbf3be6f6ef/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= +github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= +github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= github.com/livekit/protocol v1.10.1 h1:upe6pKRqH8wpsMuR2OLtgizEm94iia3pDYm3O4/2PRY= github.com/livekit/protocol v1.10.1/go.mod h1:eWPz45pnxwpCwB84qqhHxG0bCRgasa2itN6GAHCDddc= github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 h1:bvjzDR+Rvdf3JgzQMtLiGVHBQ8KoOWM7x7sHj79jevQ= @@ -200,8 +200,8 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtcp v1.2.13 h1:+EQijuisKwm/8VBs8nWllr0bIndR7Lf7cZG200mpbNo= -github.com/pion/rtcp v1.2.13/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= +github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 57141fb62..870c8c15c 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -16,8 +16,10 @@ package rtc import ( "context" + "math" "strings" "sync" + "time" "github.com/pion/rtcp" "github.com/pion/webrtc/v3" @@ -32,6 +34,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" "github.com/livekit/livekit-server/pkg/telemetry" + util "github.com/livekit/mediatransportutil" ) // MediaTrack represents a WebRTC track that needs to be forwarded @@ -47,6 +50,8 @@ type MediaTrack struct { dynacastManager *DynacastManager lock sync.RWMutex + + rttFromXR atomic.Bool } type MediaTrackParams struct { @@ -183,12 +188,14 @@ func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, mid string) bool { var newCodec bool - buff, rtcpReader := t.params.BufferFactory.GetBufferPair(uint32(track.SSRC())) + ssrc := uint32(track.SSRC()) + buff, rtcpReader := t.params.BufferFactory.GetBufferPair(ssrc) if buff == nil || rtcpReader == nil { t.params.Logger.Errorw("could not retrieve buffer pair", nil) return newCodec } + var lastRR uint32 rtcpReader.OnPacket(func(bytes []byte) { pkts, err := rtcp.Unmarshal(bytes) if err != nil { @@ -202,6 +209,25 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra // do nothing for now case *rtcp.SenderReport: buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime) + case *rtcp.ExtendedReport: + rttFromXR: + for _, report := range pkt.Reports { + if rr, ok := report.(*rtcp.DLRRReportBlock); ok { + for _, dlrrReport := range rr.Reports { + if dlrrReport.LastRR <= lastRR { + continue + } + nowNTP := util.ToNtpTime(time.Now()) + nowNTP32 := uint32(nowNTP >> 16) + ntpDiff := nowNTP32 - dlrrReport.LastRR - dlrrReport.DLRR + rtt := uint32(math.Ceil(float64(ntpDiff) * 1000.0 / 65536.0)) + buff.SetRTT(rtt) + t.rttFromXR.Store(true) + lastRR = dlrrReport.LastRR + break rttFromXR + } + } + } } } }) @@ -359,7 +385,9 @@ func (t *MediaTrack) GetConnectionScoreAndQuality() (float32, livekit.Connection } func (t *MediaTrack) SetRTT(rtt uint32) { - t.MediaTrackReceiver.SetRTT(rtt) + if !t.rttFromXR.Load() { + t.MediaTrackReceiver.SetRTT(rtt) + } } func (t *MediaTrack) HasPendingCodec() bool { diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 652e2fcfb..ee67987e4 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -138,6 +138,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * Pacer: sub.GetPacer(), Trailer: trailer, Logger: LoggerWithTrack(sub.GetLogger().WithComponent(sutils.ComponentSub), trackID, t.params.IsRelayed), + RTCPWriter: sub.WriteSubscriberRTCP, }) if err != nil { return nil, err diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 1765f865b..77b8de20b 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -38,13 +38,14 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" - lkinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" + sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/rtpextension" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" sutils "github.com/livekit/livekit-server/pkg/utils" + lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -322,6 +323,11 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } } } + } else { + // sfu only use interceptor to send XR but don't read response from it (use buffer instead), + // so use a empty callback here + ir.Add(lkinterceptor.NewRTTFromXRFactory(func(rtt uint32) { + })) } if len(params.SimTracks) > 0 { f, err := NewUnhandleSimulcastInterceptorFactory(UnhandleSimulcastTracks(params.SimTracks)) @@ -361,7 +367,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } } // put rtx interceptor behind unhandle simulcast interceptor so it can get the correct mid & rid - ir.Add(lkinterceptor.NewRTXInfoExtractorFactory(setTWCCForVideo, func(repair, base uint32) { + ir.Add(sfuinterceptor.NewRTXInfoExtractorFactory(setTWCCForVideo, func(repair, base uint32) { params.Logger.Debugw("rtx pair found from extension", "repair", repair, "base", base) params.Config.BufferFactory.SetRTXPair(repair, base) }, params.Logger)) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 8ad80ef24..88f624396 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -344,6 +344,8 @@ type LocalParticipant interface { AddTransceiverFromTrackToSubscriber(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) RemoveTrackFromSubscriber(sender *webrtc.RTPSender) error + WriteSubscriberRTCP(pkts []rtcp.Packet) error + // subscriptions SubscribeToTrack(trackID livekit.TrackID) UnsubscribeFromTrack(trackID livekit.TrackID) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 131e2f87c..4cc2d1db4 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -979,6 +979,17 @@ type FakeLocalParticipant struct { waitUntilSubscribedReturnsOnCall map[int]struct { result1 error } + WriteSubscriberRTCPStub func([]rtcp.Packet) error + writeSubscriberRTCPMutex sync.RWMutex + writeSubscriberRTCPArgsForCall []struct { + arg1 []rtcp.Packet + } + writeSubscriberRTCPReturns struct { + result1 error + } + writeSubscriberRTCPReturnsOnCall map[int]struct { + result1 error + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -6207,6 +6218,72 @@ func (fake *FakeLocalParticipant) WaitUntilSubscribedReturnsOnCall(i int, result }{result1} } +func (fake *FakeLocalParticipant) WriteSubscriberRTCP(arg1 []rtcp.Packet) error { + var arg1Copy []rtcp.Packet + if arg1 != nil { + arg1Copy = make([]rtcp.Packet, len(arg1)) + copy(arg1Copy, arg1) + } + fake.writeSubscriberRTCPMutex.Lock() + ret, specificReturn := fake.writeSubscriberRTCPReturnsOnCall[len(fake.writeSubscriberRTCPArgsForCall)] + fake.writeSubscriberRTCPArgsForCall = append(fake.writeSubscriberRTCPArgsForCall, struct { + arg1 []rtcp.Packet + }{arg1Copy}) + stub := fake.WriteSubscriberRTCPStub + fakeReturns := fake.writeSubscriberRTCPReturns + fake.recordInvocation("WriteSubscriberRTCP", []interface{}{arg1Copy}) + fake.writeSubscriberRTCPMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) WriteSubscriberRTCPCallCount() int { + fake.writeSubscriberRTCPMutex.RLock() + defer fake.writeSubscriberRTCPMutex.RUnlock() + return len(fake.writeSubscriberRTCPArgsForCall) +} + +func (fake *FakeLocalParticipant) WriteSubscriberRTCPCalls(stub func([]rtcp.Packet) error) { + fake.writeSubscriberRTCPMutex.Lock() + defer fake.writeSubscriberRTCPMutex.Unlock() + fake.WriteSubscriberRTCPStub = stub +} + +func (fake *FakeLocalParticipant) WriteSubscriberRTCPArgsForCall(i int) []rtcp.Packet { + fake.writeSubscriberRTCPMutex.RLock() + defer fake.writeSubscriberRTCPMutex.RUnlock() + argsForCall := fake.writeSubscriberRTCPArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) WriteSubscriberRTCPReturns(result1 error) { + fake.writeSubscriberRTCPMutex.Lock() + defer fake.writeSubscriberRTCPMutex.Unlock() + fake.WriteSubscriberRTCPStub = nil + fake.writeSubscriberRTCPReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) WriteSubscriberRTCPReturnsOnCall(i int, result1 error) { + fake.writeSubscriberRTCPMutex.Lock() + defer fake.writeSubscriberRTCPMutex.Unlock() + fake.WriteSubscriberRTCPStub = nil + if fake.writeSubscriberRTCPReturnsOnCall == nil { + fake.writeSubscriberRTCPReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.writeSubscriberRTCPReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -6430,6 +6507,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.verifySubscribeParticipantInfoMutex.RUnlock() fake.waitUntilSubscribedMutex.RLock() defer fake.waitUntilSubscribedMutex.RUnlock() + fake.writeSubscriberRTCPMutex.RLock() + defer fake.writeSubscriberRTCPMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 79abf6242..bf6d9970e 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -206,6 +206,7 @@ type DowntrackParams struct { Pacer pacer.Pacer Logger logger.Logger Trailer []byte + RTCPWriter func([]rtcp.Packet) error } // DownTrack implements TrackLocal, is the track used to write packets @@ -1567,6 +1568,34 @@ func (d *DownTrack) handleRTCP(bytes []byte) { sal.OnTransportCCFeedback(d, p) } } + + case *rtcp.ExtendedReport: + // SFU only responds with the DLRRReport for the track has the sender SSRC, the behavior is different with + // browser's implementation, which includes all sent tracks. It is ok since all the tracks + // use the same connection, and server-sdk-go can get the rtt from the first DLRRReport + // (libwebrtc/browsers don't send XR to calculate rtt, it only responds) + var lastRR uint32 + for _, report := range p.Reports { + if rr, ok := report.(*rtcp.ReceiverReferenceTimeReportBlock); ok { + lastRR = uint32(rr.NTPTimestamp >> 16) + break + } + } + + if lastRR > 0 { + d.params.RTCPWriter([]rtcp.Packet{&rtcp.ExtendedReport{ + SenderSSRC: d.ssrc, + Reports: []rtcp.ReportBlock{ + &rtcp.DLRRReportBlock{ + Reports: []rtcp.DLRRReport{{ + SSRC: p.SenderSSRC, + LastRR: lastRR, + DLRR: 0, // no delay + }}, + }, + }, + }}) + } } } From ad341d41f5c04bd628bd2d9dbcc12984dbe4e133 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 3 Mar 2024 02:47:51 -0800 Subject: [PATCH 40/73] start telemetry participant worker to collect signal stats (#2538) * start telemetry participant worker to collect signal stats * format * resolve room * tidy --- pkg/service/rtcservice.go | 42 ++++++++----------- pkg/telemetry/events.go | 29 ++++++-------- pkg/telemetry/signalanddatastats.go | 62 +++++++++++++++++++++++++++-- pkg/telemetry/telemetryservice.go | 19 ++++++--- 4 files changed, 101 insertions(+), 51 deletions(-) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index baef4aa30..dba8a1896 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -247,12 +247,10 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pi.ID = livekit.ParticipantID(initialResponse.GetJoin().GetParticipant().GetSid()) } - var signalStats *telemetry.BytesTrackStats - if pi.ID != "" { - signalStats = telemetry.NewBytesTrackStats( - telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeSignal, pi.ID), - pi.ID, - s.telemetry) + signalStats := telemetry.NewBytesSignalStats(r.Context(), s.telemetry) + if join := initialResponse.GetJoin(); join != nil { + signalStats.ResolveRoom(join.GetRoom()) + signalStats.ResolveParticipant(join.GetParticipant()) } pLogger := rtc.LoggerWithParticipant( @@ -274,9 +272,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { cr.RequestSink.Close() close(done) - if signalStats != nil { - signalStats.Stop() - } + signalStats.Stop() }() // upgrade only once the basics are good to go @@ -303,9 +299,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Warnw("could not write initial response", err) return } - if signalStats != nil { - signalStats.AddBytes(uint64(count), true) - } + signalStats.AddBytes(uint64(count), true) + pLogger.Debugw("new client WS connected", "connID", cr.ConnectionID, "reconnect", pi.Reconnect, @@ -349,20 +344,17 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Debugw("sending offer", "offer", m) case *livekit.SignalResponse_Answer: pLogger.Debugw("sending answer", "answer", m) - } - - if pi.ID == "" && res.GetJoin() != nil { - pi.ID = livekit.ParticipantID(res.GetJoin().GetParticipant().GetSid()) - signalStats = telemetry.NewBytesTrackStats( - telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeSignal, pi.ID), - pi.ID, - s.telemetry) + case *livekit.SignalResponse_Join: + signalStats.ResolveRoom(m.Join.GetRoom()) + signalStats.ResolveParticipant(m.Join.GetParticipant()) + case *livekit.SignalResponse_RoomUpdate: + signalStats.ResolveRoom(m.RoomUpdate.GetRoom()) } if count, err := sigConn.WriteResponse(res); err != nil { pLogger.Warnw("error writing to websocket", err) return - } else if signalStats != nil { + } else { signalStats.AddBytes(uint64(count), true) } } @@ -390,9 +382,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { } return } - if signalStats != nil { - signalStats.AddBytes(uint64(count), false) - } + signalStats.AddBytes(uint64(count), false) switch m := req.Message.(type) { case *livekit.SignalRequest_Ping: @@ -405,7 +395,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { Pong: time.Now().UnixMilli(), }, }) - if perr == nil && signalStats != nil { + if perr == nil { signalStats.AddBytes(uint64(count), true) } case *livekit.SignalRequest_PingReq: @@ -417,7 +407,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { }, }, }) - if perr == nil && signalStats != nil { + if perr == nil { signalStats.AddBytes(uint64(count), true) } } diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 35eaf668e..1d0a9c546 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -80,16 +80,17 @@ func (t *telemetryService) ParticipantJoined( shouldSendEvent bool, ) { t.enqueue(func() { - prometheus.IncrementParticipantRtcConnected(1) - prometheus.AddParticipant() - - t.createWorker( + _, found := t.getOrCreateWorker( ctx, livekit.RoomID(room.Sid), livekit.RoomName(room.Name), livekit.ParticipantID(participant.Sid), livekit.ParticipantIdentity(participant.Identity), ) + if !found { + prometheus.IncrementParticipantRtcConnected(1) + prometheus.AddParticipant() + } if shouldSendEvent { ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_JOINED, room, participant) @@ -117,18 +118,14 @@ func (t *telemetryService) ParticipantActive( }) } - worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)) - if !ok { - // in case of session migration, we may not have seen a Join event take place. - // we'd need to create the worker here before being able to process events - worker = t.createWorker( - ctx, - livekit.RoomID(room.Sid), - livekit.RoomName(room.Name), - livekit.ParticipantID(participant.Sid), - livekit.ParticipantIdentity(participant.Identity), - ) - + worker, found := t.getOrCreateWorker( + ctx, + livekit.RoomID(room.Sid), + livekit.RoomName(room.Name), + livekit.ParticipantID(participant.Sid), + livekit.ParticipantIdentity(participant.Identity), + ) + if !found { // need to also account for participant count prometheus.AddParticipant() } diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index f30f8bfe0..086192532 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -15,16 +15,17 @@ package telemetry import ( + "context" "fmt" + "sync" "time" + "github.com/frostbyte73/core" "go.uber.org/atomic" - "github.com/frostbyte73/core" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" - - "github.com/livekit/livekit-server/pkg/config" ) type BytesTrackType string @@ -139,6 +140,61 @@ func (s *BytesTrackStats) reporter() { // ----------------------------------------------------------------------- +type BytesSignalStats struct { + BytesTrackStats + ctx context.Context + + mu sync.Mutex + ri *livekit.Room + pi *livekit.ParticipantInfo +} + +func NewBytesSignalStats(ctx context.Context, telemetry TelemetryService) *BytesSignalStats { + return &BytesSignalStats{ + BytesTrackStats: BytesTrackStats{ + telemetry: telemetry, + }, + ctx: ctx, + } +} + +func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ri == nil && ri.GetSid() != "" { + s.ri = ri + s.maybeStart() + } +} + +func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) { + s.mu.Lock() + defer s.mu.Unlock() + if s.pi == nil { + s.pi = pi + s.maybeStart() + } +} + +func (s *BytesSignalStats) maybeStart() { + if s.ri == nil || s.pi == nil { + return + } + + s.pID = livekit.ParticipantID(s.pi.Sid) + s.trackID = BytesTrackIDForParticipantID(BytesTrackTypeSignal, s.pID) + + s.telemetry.ParticipantJoined(s.ctx, s.ri, s.pi, nil, nil, false) + go s.reporter() +} + +func (s *BytesSignalStats) reporter() { + s.BytesTrackStats.reporter() + s.telemetry.ParticipantLeft(s.ctx, s.ri, s.pi, false) +} + +// ----------------------------------------------------------------------- + func BytesTrackIDForParticipantID(typ BytesTrackType, participantID livekit.ParticipantID) livekit.TrackID { return livekit.TrackID(fmt.Sprintf("%s_%s%s", utils.TrackPrefix, string(typ), participantID)) } diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index c684d5842..7bf544167 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -19,12 +19,13 @@ import ( "sync" "time" + "golang.org/x/exp/maps" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/webhook" - "golang.org/x/exp/maps" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . TelemetryService @@ -153,12 +154,19 @@ func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worke return } -func (t *telemetryService) createWorker(ctx context.Context, +func (t *telemetryService) getOrCreateWorker(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, participantIdentity livekit.ParticipantIdentity, -) *StatsWorker { +) (*StatsWorker, bool) { + t.lock.Lock() + defer t.lock.Unlock() + + if worker, ok := t.workers[participantID]; ok { + return worker, true + } + worker := newStatsWorker( ctx, t, @@ -168,11 +176,10 @@ func (t *telemetryService) createWorker(ctx context.Context, participantIdentity, ) - t.lock.Lock() t.workers[participantID] = worker t.workersShadow = maps.Values(t.workers) - t.lock.Unlock() - return worker + + return worker, false } func (t *telemetryService) cleanupWorkers() { From 56dcd0b025c10b4ac3b1d606aeed84ec170fda20 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 4 Mar 2024 13:11:57 +0530 Subject: [PATCH 41/73] Process matching sender report only. (#2540) --- pkg/rtc/mediatrack.go | 5 +++-- pkg/rtc/transport.go | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 870c8c15c..c28c563c3 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -206,9 +206,10 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra for _, pkt := range pkts { switch pkt := pkt.(type) { case *rtcp.SourceDescription: - // do nothing for now case *rtcp.SenderReport: - buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime) + if pkt.SSRC == uint32(track.SSRC()) { + buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime) + } case *rtcp.ExtendedReport: rttFromXR: for _, report := range pkt.Reports { diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 77b8de20b..09e124bae 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -326,8 +326,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } else { // sfu only use interceptor to send XR but don't read response from it (use buffer instead), // so use a empty callback here - ir.Add(lkinterceptor.NewRTTFromXRFactory(func(rtt uint32) { - })) + ir.Add(lkinterceptor.NewRTTFromXRFactory(func(rtt uint32) {})) } if len(params.SimTracks) > 0 { f, err := NewUnhandleSimulcastInterceptorFactory(UnhandleSimulcastTracks(params.SimTracks)) From 8c5d8730b1befe0bfbaa9ed5a0d42636af6efc6a Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 4 Mar 2024 16:36:57 +0800 Subject: [PATCH 42/73] Check ssrc for rtcp packets (#2541) * Check ssrc for rtcp packets * Add warn log if rtcp fb has unknown ssrc --- pkg/sfu/downtrack.go | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index bf6d9970e..0eed08677 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1089,7 +1089,7 @@ func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32 } } -func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64, pauseReason VideoPauseReason) { +func (d *DownTrack) maybeAddTransition(_ int64, distance float64, pauseReason VideoPauseReason) { if d.kind == webrtc.RTPCodecTypeAudio { return } @@ -1506,12 +1506,20 @@ func (d *DownTrack) handleRTCP(bytes []byte) { for _, pkt := range pkts { switch p := pkt.(type) { case *rtcp.PictureLossIndication: - numPLIs++ - sendPliOnce() + if p.MediaSSRC == d.ssrc { + numPLIs++ + sendPliOnce() + } else { + d.params.Logger.Warnw("received PLI for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) + } case *rtcp.FullIntraRequest: - numFIRs++ - sendPliOnce() + if p.MediaSSRC == d.ssrc { + numFIRs++ + sendPliOnce() + } else { + d.params.Logger.Warnw("received FIR for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) + } case *rtcp.ReceiverEstimatedMaximumBitrate: if sal := d.getStreamAllocatorListener(); sal != nil { @@ -1554,13 +1562,17 @@ func (d *DownTrack) handleRTCP(bytes []byte) { } case *rtcp.TransportLayerNack: - var nacks []uint16 - for _, pair := range p.Nacks { - packetList := pair.PacketList() - numNACKs += uint32(len(packetList)) - nacks = append(nacks, packetList...) + if p.MediaSSRC == d.ssrc { + var nacks []uint16 + for _, pair := range p.Nacks { + packetList := pair.PacketList() + numNACKs += uint32(len(packetList)) + nacks = append(nacks, packetList...) + } + go d.retransmitPackets(nacks) + } else { + d.params.Logger.Warnw("received NACK for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) } - go d.retransmitPackets(nacks) case *rtcp.TransportLayerCC: if p.MediaSSRC == d.ssrc { From fdd76c8734ae737168c823ee8c767a5534a3a3aa Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 4 Mar 2024 14:41:35 +0530 Subject: [PATCH 43/73] Post a PLI check request on connected. (#2542) Otherwise, it gets delayed by timer and could wait upto to interval (which could be 200 ms when there is no RTT information yet). --- pkg/sfu/downtrack.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0eed08677..424e80356 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1858,6 +1858,9 @@ func (d *DownTrack) onBindAndConnectedChange() { if d.activePaddingOnMuteUpTrack.Load() { go d.sendPaddingOnMute() } + + // kick off PLI request if allocation is pending + d.postKeyFrameRequestEvent() } } From 46257c1d24bfd32ce8221e08c85db44c679497f8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 4 Mar 2024 19:12:53 +0530 Subject: [PATCH 44/73] Skip large RR intervals. (#2544) --- pkg/sfu/buffer/rtpstats_sender.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index ca4415140..33a22afbf 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -533,6 +533,33 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt s.maxJitter = r.jitterFromRR } + if int64(extReceivedRRSN-s.extLastRRSN) < 0 || (extReceivedRRSN-s.extLastRRSN) > (1<<15) { + timeSinceLastRR := time.Since(r.lastRRTime) + if r.lastRRTime.IsZero() { + timeSinceLastRR = time.Since(r.startTime) + } + r.logger.Infow( + "rr interval too big, skipping", + "lastRRTime", r.lastRRTime.String(), + "lastRR", r.lastRR, + "timeSinceLastRR", timeSinceLastRR.String(), + "receivedRR", rr, + "extStartSN", r.extStartSN, + "extHighestSN", r.extHighestSN, + "extStartTS", r.extStartTS, + "extHighestTS", r.extHighestTS, + "extLastRRSN", s.extLastRRSN, + "firstTime", r.firstTime.String(), + "startTime", r.startTime.String(), + "highestTime", r.highestTime.String(), + "extReceivedRRSN", extReceivedRRSN, + "packetsInInterval", extReceivedRRSN-s.extLastRRSN, + "extHighestSNFromRR", r.extHighestSNFromRR, + "packetsLostFromRR", r.packetsLostFromRR, + ) + continue + } + // on every RR, calculate delta since last RR using packet metadata cache is := r.getIntervalStats(s.extLastRRSN+1, extReceivedRRSN+1, r.extHighestSN) eis := &s.intervalStats From 108594a0b9affe01e48bd290ae57cef82d2495d9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 4 Mar 2024 19:43:54 +0530 Subject: [PATCH 45/73] Notify migration on migration out node. (#2543) It is possible that migration could trigger without migrating out node knowing about it. So, when a migration started notification comes in, set up migration timer if not already set. --- pkg/rtc/participant.go | 54 ++++++++++++------- pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 30 +++++++++++ 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3443ce2b1..21c4e1413 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -853,22 +853,7 @@ func (p *ParticipantImpl) clearMigrationTimer() { p.lock.Unlock() } -func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { - allTransportConnected := p.TransportManager.HasSubscriberEverConnected() - if p.IsPublisher() { - allTransportConnected = allTransportConnected && p.TransportManager.HasPublisherEverConnected() - } - if !force && !allTransportConnected { - return false - } - - if onStart != nil { - onStart() - } - - p.sendLeaveRequest(types.ParticipantCloseReasonMigrationRequested, true, false, true) - p.CloseSignalConnection(types.SignallingCloseReasonMigration) - +func (p *ParticipantImpl) setupMigrationTimerLocked() { // // On subscriber peer connection, remote side will try ICE on both // pre- and post-migration ICE candidates as the migrating out @@ -880,9 +865,6 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { // to try and succeed. If not, close the subscriber peer connection // and help the remote side to narrow down its ICE candidate pool. // - p.clearMigrationTimer() - - p.lock.Lock() p.migrationTimer = time.AfterFunc(migrationWaitDuration, func() { p.clearMigrationTimer() @@ -901,11 +883,45 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { p.TransportManager.SubscriberClose() }) +} + +func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { + allTransportConnected := p.TransportManager.HasSubscriberEverConnected() + if p.IsPublisher() { + allTransportConnected = allTransportConnected && p.TransportManager.HasPublisherEverConnected() + } + if !force && !allTransportConnected { + return false + } + + if onStart != nil { + onStart() + } + + p.sendLeaveRequest(types.ParticipantCloseReasonMigrationRequested, true, false, true) + p.CloseSignalConnection(types.SignallingCloseReasonMigration) + + p.clearMigrationTimer() + + p.lock.Lock() + p.setupMigrationTimerLocked() p.lock.Unlock() return true } +func (p *ParticipantImpl) NotifyMigration() { + p.lock.Lock() + defer p.lock.Unlock() + + if p.migrationTimer != nil { + // already set up + return + } + + p.setupMigrationTimerLocked() +} + func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { preState := p.MigrateState() if preState == types.MigrateStateComplete || preState == s { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 88f624396..b57a7e92a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -395,6 +395,7 @@ type LocalParticipant interface { // session migration MaybeStartMigration(force bool, onStart func()) bool + NotifyMigration() SetMigrateState(s MigrateState) MigrateState() MigrateState SetMigrateInfo(previousOffer, previousAnswer *webrtc.SessionDescription, mediaTracks []*livekit.TrackPublishedResponse, dataChannels []*livekit.DataChannelInfo) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 4cc2d1db4..6bbf2b5a6 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -557,6 +557,10 @@ type FakeLocalParticipant struct { negotiateArgsForCall []struct { arg1 bool } + NotifyMigrationStub func() + notifyMigrationMutex sync.RWMutex + notifyMigrationArgsForCall []struct { + } OnClaimsChangedStub func(func(types.LocalParticipant)) onClaimsChangedMutex sync.RWMutex onClaimsChangedArgsForCall []struct { @@ -3840,6 +3844,30 @@ func (fake *FakeLocalParticipant) NegotiateArgsForCall(i int) bool { return argsForCall.arg1 } +func (fake *FakeLocalParticipant) NotifyMigration() { + fake.notifyMigrationMutex.Lock() + fake.notifyMigrationArgsForCall = append(fake.notifyMigrationArgsForCall, struct { + }{}) + stub := fake.NotifyMigrationStub + fake.recordInvocation("NotifyMigration", []interface{}{}) + fake.notifyMigrationMutex.Unlock() + if stub != nil { + fake.NotifyMigrationStub() + } +} + +func (fake *FakeLocalParticipant) NotifyMigrationCallCount() int { + fake.notifyMigrationMutex.RLock() + defer fake.notifyMigrationMutex.RUnlock() + return len(fake.notifyMigrationArgsForCall) +} + +func (fake *FakeLocalParticipant) NotifyMigrationCalls(stub func()) { + fake.notifyMigrationMutex.Lock() + defer fake.notifyMigrationMutex.Unlock() + fake.NotifyMigrationStub = stub +} + func (fake *FakeLocalParticipant) OnClaimsChanged(arg1 func(types.LocalParticipant)) { fake.onClaimsChangedMutex.Lock() fake.onClaimsChangedArgsForCall = append(fake.onClaimsChangedArgsForCall, struct { @@ -6399,6 +6427,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.migrateStateMutex.RUnlock() fake.negotiateMutex.RLock() defer fake.negotiateMutex.RUnlock() + fake.notifyMigrationMutex.RLock() + defer fake.notifyMigrationMutex.RUnlock() fake.onClaimsChangedMutex.RLock() defer fake.onClaimsChangedMutex.RUnlock() fake.onCloseMutex.RLock() From 95001dfe546b552133a6afc48c52c463289425ce Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 5 Mar 2024 10:35:28 +0800 Subject: [PATCH 46/73] remove warning log of unexpected rtcp fb (#2546) * remove warning log of unexpected rtcp fb * Add log for high playout delay * don't create delay controller for audio --- pkg/sfu/downtrack.go | 21 +++++++-------------- pkg/sfu/playoutdelay.go | 5 +++++ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 424e80356..05da21add 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -328,14 +328,6 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { }) d.deltaStatsSenderSnapshotId = d.rtpStats.NewSenderSnapshotId() - if delay := params.PlayoutDelayLimit; delay.GetEnabled() { - var err error - d.playoutDelay, err = NewPlayoutDelayController(delay.GetMin(), delay.GetMax(), params.Logger, d.rtpStats) - if err != nil { - return nil, err - } - } - d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), @@ -349,6 +341,13 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { }) if d.kind == webrtc.RTPCodecTypeVideo { + if delay := params.PlayoutDelayLimit; delay.GetEnabled() { + var err error + d.playoutDelay, err = NewPlayoutDelayController(delay.GetMin(), delay.GetMax(), params.Logger, d.rtpStats) + if err != nil { + return nil, err + } + } go d.maxLayerNotifierWorker() go d.keyFrameRequester() } @@ -1509,16 +1508,12 @@ func (d *DownTrack) handleRTCP(bytes []byte) { if p.MediaSSRC == d.ssrc { numPLIs++ sendPliOnce() - } else { - d.params.Logger.Warnw("received PLI for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) } case *rtcp.FullIntraRequest: if p.MediaSSRC == d.ssrc { numFIRs++ sendPliOnce() - } else { - d.params.Logger.Warnw("received FIR for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) } case *rtcp.ReceiverEstimatedMaximumBitrate: @@ -1570,8 +1565,6 @@ func (d *DownTrack) handleRTCP(bytes []byte) { nacks = append(nacks, packetList...) } go d.retransmitPackets(nacks) - } else { - d.params.Logger.Warnw("received NACK for unknown SSRC", nil, "expected", d.ssrc, "media_ssrc", p.MediaSSRC, "sender_ssrc", p.SenderSSRC) } case *rtcp.TransportLayerCC: diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index cf95e3b5b..2dc756d39 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -33,6 +33,8 @@ const ( jitterLowMultiToDelay = 10 jitterHighMultiToDelay = 15 jitterHighThreshold = 15 + + targetDelayLogThreshold = 500 ) func (s PlayoutDelayState) String() string { @@ -110,6 +112,9 @@ func (c *PlayoutDelayController) SetJitter(jitter uint32) { c.lock.Unlock() return } + if targetDelay > targetDelayLogThreshold { + c.logger.Debugw("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay) + } c.currentDelay = targetDelay c.lock.Unlock() c.createExtData() From 0618cb39df47a381bbe8af77bdc61fe6932bc697 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 5 Mar 2024 20:31:44 +0530 Subject: [PATCH 47/73] Logging time and rtp diff for easier debugging (#2548) --- pkg/sfu/buffer/rtpstats_receiver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 093084dd8..347f98103 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -307,7 +307,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) "first", r.srFirst.ToString(), "last", r.srNewest.ToString(), "current", srDataCopy.ToString(), + "timeSinceFirst", timeSinceFirst, + "rtpDiffSinceFirst", rtpDiffSinceFirst, "calculatedFirst", calculatedClockRateFromFirst, + "timeSinceLast", timeSinceLast, + "rtpDiffSinceLast", rtpDiffSinceLast, "calculatedLast", calculatedClockRateFromLast, "count", r.clockSkewCount, ) From d87f8aa299a3d9b27dde14fcb2d586683b431084 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 5 Mar 2024 09:05:42 -0800 Subject: [PATCH 48/73] use departure timeout (#2549) * use departure timeout * deps * default * remove constants * deps * protoproxy cache * add sample --- config-sample.yaml | 4 +++- go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/config/config.go | 4 +++- pkg/rtc/room.go | 16 +++++++++------- pkg/rtc/room_test.go | 8 +++++--- pkg/service/roomallocator.go | 4 ++++ pkg/service/roomallocator_test.go | 1 + pkg/service/roommanager.go | 2 +- 9 files changed, 35 insertions(+), 22 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index da88700be..96eadb44b 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -170,8 +170,10 @@ keys: # room: # # allow rooms to be automatically created when participants join, defaults to true # # auto_create: false -# # number of seconds to leave a room open when it's empty +# # number of seconds to keep the room open if no one joins # empty_timeout: 300 +# # number of seconds to keep the room open after everyone leaves +# departure_timeout: 20 # # limit number of participants that can be in a room, 0 for no limit # max_participants: 0 # # only accept specific codecs for clients publishing to this room diff --git a/go.mod b/go.mod index 669c5ea85..7c98b8fd8 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.10.1 - github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 + github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268 + github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -97,7 +97,7 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.19.0 // indirect - golang.org/x/mod v0.15.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index dc3e827a7..5d24fdf19 100644 --- a/go.sum +++ b/go.sum @@ -130,10 +130,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.10.1 h1:upe6pKRqH8wpsMuR2OLtgizEm94iia3pDYm3O4/2PRY= -github.com/livekit/protocol v1.10.1/go.mod h1:eWPz45pnxwpCwB84qqhHxG0bCRgasa2itN6GAHCDddc= -github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3 h1:bvjzDR+Rvdf3JgzQMtLiGVHBQ8KoOWM7x7sHj79jevQ= -github.com/livekit/psrpc v0.5.3-0.20240227154351-b7f99eaaf7b3/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268 h1:cEox8Kzv0Ib+je6OtaKsx3DBlpZF45xZnkev2L4sQv4= +github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268/go.mod h1:XpJ2t2wFnnQghPpkxXAzMZhYMDnm8wWxdxYJK4fP9gM= +github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 h1:253WtQ2VGVHzIIzW9MUZj7vUDDILESU3zsEbiRdxYF0= +github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -312,8 +312,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/config/config.go b/pkg/config/config.go index 0530fb045..82f338f9a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -232,6 +232,7 @@ type RoomConfig struct { EnabledCodecs []CodecSpec `yaml:"enabled_codecs,omitempty"` MaxParticipants uint32 `yaml:"max_participants,omitempty"` EmptyTimeout uint32 `yaml:"empty_timeout,omitempty"` + DepartureTimeout uint32 `yaml:"departure_timeout,omitempty"` EnableRemoteUnmute bool `yaml:"enable_remote_unmute,omitempty"` MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` PlayoutDelay PlayoutDelayConfig `yaml:"playout_delay,omitempty"` @@ -482,7 +483,8 @@ var DefaultConfig = Config{ {Mime: webrtc.MimeTypeVP9}, {Mime: webrtc.MimeTypeAV1}, }, - EmptyTimeout: 5 * 60, + EmptyTimeout: 5 * 60, + DepartureTimeout: 20, }, Logging: LoggingConfig{ PionLevel: "error", diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 082f91f71..0a659ae94 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -47,8 +47,7 @@ import ( ) const ( - DefaultEmptyTimeout = 5 * 60 // 5m - AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal + AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal invAudioLevelQuantization = 1.0 / AudioLevelQuantization subscriberUpdateInterval = 3 * time.Second @@ -59,8 +58,7 @@ const ( var ( // var to allow unit test override - RoomDepartureGrace uint32 = 20 - roomUpdateInterval = 5 * time.Second // frequency to update room participant counts + roomUpdateInterval = 5 * time.Second // frequency to update room participant counts ) type broadcastOptions struct { @@ -139,6 +137,7 @@ func NewRoom( room *livekit.Room, internal *livekit.RoomInternal, config WebRTCConfig, + roomConfig config.RoomConfig, audioConfig *config.AudioConfig, serverInfo *livekit.ServerInfo, telemetry telemetry.TelemetryService, @@ -172,13 +171,16 @@ func NewRoom( disconnectSignalOnResumeNoMessagesParticipants: make(map[livekit.ParticipantIdentity]*disconnectSignalOnResumeNoMessages), } - r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) if r.protoRoom.EmptyTimeout == 0 { - r.protoRoom.EmptyTimeout = DefaultEmptyTimeout + r.protoRoom.EmptyTimeout = roomConfig.EmptyTimeout + } + if r.protoRoom.DepartureTimeout == 0 { + r.protoRoom.DepartureTimeout = roomConfig.DepartureTimeout } if r.protoRoom.CreationTime == 0 { r.protoRoom.CreationTime = time.Now().Unix() } + r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) if agentClient != nil { go func() { @@ -774,7 +776,7 @@ func (r *Room) CloseIfEmpty() { if r.FirstJoinedAt() > 0 && r.LastLeftAt() > 0 { elapsed = time.Now().Unix() - r.LastLeftAt() // need to give time in case participant is reconnecting - timeout = RoomDepartureGrace + timeout = r.protoRoom.DepartureTimeout } else { elapsed = time.Now().Unix() - r.protoRoom.CreationTime timeout = r.protoRoom.EmptyTimeout diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 0febf2989..ea09a91d0 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -48,8 +48,6 @@ const ( func init() { config.InitLoggerFromConfig(&config.DefaultConfig.Logging) - // allow immediate closure in testing - RoomDepartureGrace = 1 roomUpdateInterval = defaultDelay } @@ -377,7 +375,7 @@ func TestRoomClosure(t *testing.T) { rm.lock.Unlock() rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave) - time.Sleep(time.Duration(RoomDepartureGrace)*time.Second + defaultDelay) + time.Sleep(time.Duration(rm.ToProto().DepartureTimeout)*time.Second + defaultDelay) rm.CloseIfEmpty() require.Len(t, rm.GetParticipants(), 0) @@ -737,6 +735,10 @@ func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room { &livekit.Room{Name: "room"}, nil, WebRTCConfig{}, + config.RoomConfig{ + EmptyTimeout: 5 * 60, + DepartureTimeout: 1, + }, &config.AudioConfig{ UpdateInterval: audioUpdateInterval, SmoothIntervals: opts.audioSmoothIntervals, diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index cf88261c6..04543a59d 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -79,6 +79,9 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre if req.EmptyTimeout > 0 { rm.EmptyTimeout = req.EmptyTimeout } + if req.DepartureTimeout > 0 { + rm.DepartureTimeout = req.DepartureTimeout + } if req.MaxParticipants > 0 { rm.MaxParticipants = req.MaxParticipants } @@ -157,6 +160,7 @@ func (r *StandardRoomAllocator) ValidateCreateRoom(ctx context.Context, roomName func applyDefaultRoomConfig(room *livekit.Room, internal *livekit.RoomInternal, conf *config.RoomConfig) { room.EmptyTimeout = conf.EmptyTimeout + room.DepartureTimeout = conf.DepartureTimeout room.MaxParticipants = conf.MaxParticipants for _, codec := range conf.EnabledCodecs { room.EnabledCodecs = append(room.EnabledCodecs, &livekit.Codec{ diff --git a/pkg/service/roomallocator_test.go b/pkg/service/roomallocator_test.go index 8c87dc8c2..dc917fbcf 100644 --- a/pkg/service/roomallocator_test.go +++ b/pkg/service/roomallocator_test.go @@ -42,6 +42,7 @@ func TestCreateRoom(t *testing.T) { room, _, err := ra.CreateRoom(context.Background(), &livekit.CreateRoomRequest{Name: "myroom"}) require.NoError(t, err) require.Equal(t, conf.Room.EmptyTimeout, room.EmptyTimeout) + require.Equal(t, conf.Room.DepartureTimeout, room.DepartureTimeout) require.NotEmpty(t, room.EnabledCodecs) }) diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index d64cceabb..8c5aa60d5 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -545,7 +545,7 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room } // construct ice servers - newRoom := rtc.NewRoom(ri, internal, *r.rtcConfig, &r.config.Audio, r.serverInfo, r.telemetry, r.agentClient, r.egressLauncher) + newRoom := rtc.NewRoom(ri, internal, *r.rtcConfig, r.config.Room, &r.config.Audio, r.serverInfo, r.telemetry, r.agentClient, r.egressLauncher) roomTopic := rpc.FormatRoomTopic(roomName) roomServer := must.Get(rpc.NewTypedRoomServer(r, r.bus)) From 6c94185aa65c50f0a1aabb7f6f3274d75ce8a457 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 7 Mar 2024 12:52:48 +0530 Subject: [PATCH 49/73] Bring back soft close. (#2553) Remote media track needs it. --- pkg/rtc/participant.go | 2 +- pkg/rtc/types/interfaces.go | 3 +- .../typesfakes/fake_local_media_track.go | 39 +++++++++++++++++++ .../typesfakes/fake_local_participant.go | 18 +++++---- pkg/rtc/types/typesfakes/fake_media_track.go | 39 +++++++++++++++++++ pkg/rtc/types/typesfakes/fake_participant.go | 18 +++++---- pkg/rtc/uptrackmanager.go | 8 +++- 7 files changed, 107 insertions(+), 20 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 21c4e1413..1146495de 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1406,7 +1406,7 @@ func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) err } func (p *ParticipantImpl) removePublishedTrack(track types.MediaTrack) { - p.RemovePublishedTrack(track, false) + p.RemovePublishedTrack(track, false, true) if p.ProtocolVersion().SupportsUnpublish() { p.sendTrackUnpublished(track.ID()) } else { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index b57a7e92a..b775c1c31 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -255,7 +255,7 @@ type Participant interface { IsPublisher() bool GetPublishedTrack(trackID livekit.TrackID) MediaTrack GetPublishedTracks() []MediaTrack - RemovePublishedTrack(track MediaTrack, willBeResumed bool) + RemovePublishedTrack(track MediaTrack, willBeResumed bool, shouldClose bool) GetAudioLevel() (smoothedLevel float64, active bool) @@ -485,6 +485,7 @@ type MediaTrack interface { GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime string) int32 Receivers() []sfu.TrackReceiver + ClearAllReceivers(willBeResumed bool) IsEncrypted() bool } diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index 0fbdcb6db..be86e74c4 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -28,6 +28,11 @@ type FakeLocalMediaTrack struct { result1 types.SubscribedTrack result2 error } + ClearAllReceiversStub func(bool) + clearAllReceiversMutex sync.RWMutex + clearAllReceiversArgsForCall []struct { + arg1 bool + } CloseStub func(bool) closeMutex sync.RWMutex closeArgsForCall []struct { @@ -437,6 +442,38 @@ func (fake *FakeLocalMediaTrack) AddSubscriberReturnsOnCall(i int, result1 types }{result1, result2} } +func (fake *FakeLocalMediaTrack) ClearAllReceivers(arg1 bool) { + fake.clearAllReceiversMutex.Lock() + fake.clearAllReceiversArgsForCall = append(fake.clearAllReceiversArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.ClearAllReceiversStub + fake.recordInvocation("ClearAllReceivers", []interface{}{arg1}) + fake.clearAllReceiversMutex.Unlock() + if stub != nil { + fake.ClearAllReceiversStub(arg1) + } +} + +func (fake *FakeLocalMediaTrack) ClearAllReceiversCallCount() int { + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() + return len(fake.clearAllReceiversArgsForCall) +} + +func (fake *FakeLocalMediaTrack) ClearAllReceiversCalls(stub func(bool)) { + fake.clearAllReceiversMutex.Lock() + defer fake.clearAllReceiversMutex.Unlock() + fake.ClearAllReceiversStub = stub +} + +func (fake *FakeLocalMediaTrack) ClearAllReceiversArgsForCall(i int) bool { + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() + argsForCall := fake.clearAllReceiversArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalMediaTrack) Close(arg1 bool) { fake.closeMutex.Lock() fake.closeArgsForCall = append(fake.closeArgsForCall, struct { @@ -2116,6 +2153,8 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.addOnCloseMutex.RUnlock() fake.addSubscriberMutex.RLock() defer fake.addSubscriberMutex.RUnlock() + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.getAllSubscribersMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 6bbf2b5a6..6528fda61 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -631,11 +631,12 @@ type FakeLocalParticipant struct { protocolVersionReturnsOnCall map[int]struct { result1 types.ProtocolVersion } - RemovePublishedTrackStub func(types.MediaTrack, bool) + RemovePublishedTrackStub func(types.MediaTrack, bool, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool + arg3 bool } RemoveTrackFromSubscriberStub func(*webrtc.RTPSender) error removeTrackFromSubscriberMutex sync.RWMutex @@ -4305,17 +4306,18 @@ func (fake *FakeLocalParticipant) ProtocolVersionReturnsOnCall(i int, result1 ty }{result1} } -func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { +func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2) + fake.RemovePublishedTrackStub(arg1, arg2, arg3) } } @@ -4325,17 +4327,17 @@ func (fake *FakeLocalParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { +func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { +func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalParticipant) RemoveTrackFromSubscriber(arg1 *webrtc.RTPSender) error { diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index 7f42687a2..d4bdfc17e 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -28,6 +28,11 @@ type FakeMediaTrack struct { result1 types.SubscribedTrack result2 error } + ClearAllReceiversStub func(bool) + clearAllReceiversMutex sync.RWMutex + clearAllReceiversArgsForCall []struct { + arg1 bool + } CloseStub func(bool) closeMutex sync.RWMutex closeArgsForCall []struct { @@ -373,6 +378,38 @@ func (fake *FakeMediaTrack) AddSubscriberReturnsOnCall(i int, result1 types.Subs }{result1, result2} } +func (fake *FakeMediaTrack) ClearAllReceivers(arg1 bool) { + fake.clearAllReceiversMutex.Lock() + fake.clearAllReceiversArgsForCall = append(fake.clearAllReceiversArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.ClearAllReceiversStub + fake.recordInvocation("ClearAllReceivers", []interface{}{arg1}) + fake.clearAllReceiversMutex.Unlock() + if stub != nil { + fake.ClearAllReceiversStub(arg1) + } +} + +func (fake *FakeMediaTrack) ClearAllReceiversCallCount() int { + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() + return len(fake.clearAllReceiversArgsForCall) +} + +func (fake *FakeMediaTrack) ClearAllReceiversCalls(stub func(bool)) { + fake.clearAllReceiversMutex.Lock() + defer fake.clearAllReceiversMutex.Unlock() + fake.ClearAllReceiversStub = stub +} + +func (fake *FakeMediaTrack) ClearAllReceiversArgsForCall(i int) bool { + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() + argsForCall := fake.clearAllReceiversArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeMediaTrack) Close(arg1 bool) { fake.closeMutex.Lock() fake.closeArgsForCall = append(fake.closeArgsForCall, struct { @@ -1702,6 +1739,8 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.addOnCloseMutex.RUnlock() fake.addSubscriberMutex.RLock() defer fake.addSubscriberMutex.RUnlock() + fake.clearAllReceiversMutex.RLock() + defer fake.clearAllReceiversMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.getAllSubscribersMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 14f5d4fcc..98fff9c80 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -158,11 +158,12 @@ type FakeParticipant struct { isRecorderReturnsOnCall map[int]struct { result1 bool } - RemovePublishedTrackStub func(types.MediaTrack, bool) + RemovePublishedTrackStub func(types.MediaTrack, bool, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool + arg3 bool } SetMetadataStub func(string) setMetadataMutex sync.RWMutex @@ -1007,17 +1008,18 @@ func (fake *FakeParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { +func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2) + fake.RemovePublishedTrackStub(arg1, arg2, arg3) } } @@ -1027,17 +1029,17 @@ func (fake *FakeParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { +func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { +func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeParticipant) SetMetadata(arg1 string) { diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 9ec2f6089..dd600c6a7 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -268,8 +268,12 @@ func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) { }) } -func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, willBeResumed bool) { - track.Close(willBeResumed) +func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, willBeResumed bool, shouldClose bool) { + if shouldClose { + track.Close(willBeResumed) + } else { + track.ClearAllReceivers(willBeResumed) + } u.lock.Lock() delete(u.publishedTracks, track.ID()) u.lock.Unlock() From 7d397110847b5a50305833f5c4439fa70c9a2416 Mon Sep 17 00:00:00 2001 From: Alexandr Rastopchin <47109336+rastop123g@users.noreply.github.com> Date: Thu, 7 Mar 2024 22:11:36 +0500 Subject: [PATCH 50/73] fix issue 2539, do not kill the keepaliveWorker task when the ping timeout occurs (#2555) --- pkg/routing/redisrouter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index 579671814..da40a85a9 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -246,7 +246,7 @@ func (r *RedisRouter) keepaliveWorker(startedChan chan error) { for ping := range pings.Channel() { if time.Since(time.Unix(ping.Timestamp, 0)) > statsUpdateInterval { logger.Infow("keep alive too old, skipping", "timestamp", ping.Timestamp) - break + continue } r.nodeMu.Lock() From 063b9d57cc9f8f3eb0a7605fdaab5ac4407612e1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:12:21 -0800 Subject: [PATCH 51/73] Update pion deps (#2534) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7c98b8fd8..40dc6e110 100644 --- a/go.mod +++ b/go.mod @@ -32,10 +32,10 @@ require ( github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.3 github.com/pion/sctp v1.8.12 - github.com/pion/sdp/v3 v3.0.6 + github.com/pion/sdp/v3 v3.0.8 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.28 + github.com/pion/webrtc/v3 v3.2.29 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index 5d24fdf19..c90d14785 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY= github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= -github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= -github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= +github.com/pion/sdp/v3 v3.0.8 h1:yd/wkrS0nzXEAb+uwv1TL3SG/gzsTiXHVOtXtD7EKl0= +github.com/pion/sdp/v3 v3.0.8/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -226,8 +226,8 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.28 h1:ienStxZ6HcjtH2UlmnFpMM0loENiYjaX437uIUpQSKo= -github.com/pion/webrtc/v3 v3.2.28/go.mod h1:PNRCEuQlibrmuBhOTnol9j6KkIbUG11aHLEfNpUYey0= +github.com/pion/webrtc/v3 v3.2.29 h1:flXjxjlqpp3FjkpSSBKwv7UOfbUvan9+gFY6A5ZaAn4= +github.com/pion/webrtc/v3 v3.2.29/go.mod h1:M+5YSvBDPAkHHRwGXlplIFBQI5mXm6Y4byns1OpiX68= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -257,6 +257,7 @@ github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4Qn github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= From 324c5f35cb1bcc92f6f0b4f660aa32b175854daf Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 8 Mar 2024 19:06:28 +0530 Subject: [PATCH 52/73] handle nil resolver. (#2556) --- pkg/rtc/room.go | 2 +- pkg/rtc/types/interfaces.go | 1 - .../typesfakes/fake_local_participant.go | 22 +++++++++---------- pkg/rtc/types/typesfakes/fake_participant.go | 22 +++++++++---------- pkg/rtc/uptrackmanager.go | 6 +++-- 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 0a659ae94..02343080f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -709,7 +709,7 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync } func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { - if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion{}, r.GetParticipant, r.GetParticipantByID); err != nil { + if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion{}, r.GetParticipantByID); err != nil { return err } for _, track := range participant.GetPublishedTracks() { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index b775c1c31..e4e6b61d3 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -276,7 +276,6 @@ type Participant interface { UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, timedVersion utils.TimedVersion, - resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) LocalParticipant, resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 6528fda61..464f4009c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -942,13 +942,12 @@ type FakeLocalParticipant struct { arg1 livekit.TrackID arg2 *livekit.UpdateTrackSettings } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission arg2 utils.TimedVersion - arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant - arg4 func(participantID livekit.ParticipantID) types.LocalParticipant + arg3 func(participantID livekit.ParticipantID) types.LocalParticipant } updateSubscriptionPermissionReturns struct { result1 error @@ -6029,21 +6028,20 @@ func (fake *FakeLocalParticipant) UpdateSubscribedTrackSettingsArgsForCall(i int return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission arg2 utils.TimedVersion - arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant - arg4 func(participantID livekit.ParticipantID) types.LocalParticipant - }{arg1, arg2, arg3, arg4}) + arg3 func(participantID livekit.ParticipantID) types.LocalParticipant + }{arg1, arg2, arg3}) stub := fake.UpdateSubscriptionPermissionStub fakeReturns := fake.updateSubscriptionPermissionReturns - fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2, arg3}) fake.updateSubscriptionPermissionMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -6057,17 +6055,17 @@ func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionReturns(result1 error) { diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 98fff9c80..9f6f97e14 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -207,13 +207,12 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission arg2 utils.TimedVersion - arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant - arg4 func(participantID livekit.ParticipantID) types.LocalParticipant + arg3 func(participantID livekit.ParticipantID) types.LocalParticipant } updateSubscriptionPermissionReturns struct { result1 error @@ -1268,21 +1267,20 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } -func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission arg2 utils.TimedVersion - arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant - arg4 func(participantID livekit.ParticipantID) types.LocalParticipant - }{arg1, arg2, arg3, arg4}) + arg3 func(participantID livekit.ParticipantID) types.LocalParticipant + }{arg1, arg2, arg3}) stub := fake.UpdateSubscriptionPermissionStub fakeReturns := fake.updateSubscriptionPermissionReturns - fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2, arg3}) fake.updateSubscriptionPermissionMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -1296,17 +1294,17 @@ func (fake *FakeParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeParticipant) UpdateSubscriptionPermissionReturns(result1 error) { diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index dd600c6a7..904b3ea66 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -157,7 +157,6 @@ func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack { func (u *UpTrackManager) UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, timedVersion utils.TimedVersion, - _ func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, // TODO: separate PR to remove this argument resolverBySid func(participantID livekit.ParticipantID) types.LocalParticipant, ) error { u.lock.Lock() @@ -203,7 +202,10 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( ) if err := u.parseSubscriptionPermissionsLocked(subscriptionPermission, func(pID livekit.ParticipantID) types.LocalParticipant { u.lock.Unlock() - p := resolverBySid(pID) + var p types.LocalParticipant + if resolverBySid != nil { + p = resolverBySid(pID) + } u.lock.Lock() return p }); err != nil { From 666b912dd3384c85dd9bf2e260c1be7a9c122526 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 8 Mar 2024 20:31:58 +0530 Subject: [PATCH 53/73] Fix test (#2557) Have no idea how last PR was ready to merge without this. --- pkg/rtc/uptrackmanager_test.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/rtc/uptrackmanager_test.go b/pkg/rtc/uptrackmanager_test.go index 46a8c96cf..5556bc023 100644 --- a/pkg/rtc/uptrackmanager_test.go +++ b/pkg/rtc/uptrackmanager_test.go @@ -49,14 +49,14 @@ func TestUpdateSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.Nil(t, um.subscriberPermissions) // nobody is allowed to subscribe subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.NotNil(t, um.subscriberPermissions) require.Equal(t, 0, len(um.subscriberPermissions)) @@ -92,7 +92,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), sidResolver) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -117,7 +117,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms3, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.Equal(t, 3, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -170,7 +170,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - err := um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) + err := um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), sidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) @@ -189,7 +189,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { return nil } - err = um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, badSidResolver) + err = um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), badSidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) @@ -202,16 +202,16 @@ func TestUpdateSubscriptionPermission(t *testing.T) { v0, v1, v2 := vg.Next(), vg.Next(), vg.Next() - um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v1, nil, nil) + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v1, nil) require.Equal(t, v1.Load(), um.subscriptionPermissionVersion.Load(), "first update should be applied") - um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v2, nil, nil) + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v2, nil) require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "ordered updates should be applied") - um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil, nil) + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil) require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "out of order updates should be ignored") - um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion{}, nil, nil) + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion{}, nil) require.True(t, um.subscriptionPermissionVersion.After(&v2), "zero version in updates should use next local version") }) } @@ -233,7 +233,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -241,7 +241,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.False(t, um.hasPermissionLocked("audio", "p1")) require.False(t, um.hasPermissionLocked("audio", "p2")) @@ -258,7 +258,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -293,7 +293,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("screen", "p1")) From 13f8191cc803bbac4b33027879dd6493f0d93113 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 8 Mar 2024 11:11:32 -0800 Subject: [PATCH 54/73] update protocol (#2560) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 40dc6e110..705123128 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268 + github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6 github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index c90d14785..3dec16926 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268 h1:cEox8Kzv0Ib+je6OtaKsx3DBlpZF45xZnkev2L4sQv4= -github.com/livekit/protocol v1.10.2-0.20240305164205-dde5199e5268/go.mod h1:XpJ2t2wFnnQghPpkxXAzMZhYMDnm8wWxdxYJK4fP9gM= +github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6 h1:jWWMJP1x96C/3oHO2asGFzwa7xDU4NrWcyt0b0irqOQ= +github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6/go.mod h1:XpJ2t2wFnnQghPpkxXAzMZhYMDnm8wWxdxYJK4fP9gM= github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 h1:253WtQ2VGVHzIIzW9MUZj7vUDDILESU3zsEbiRdxYF0= github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From a08b058abcca8b268be6d49287b1e3616edd1d91 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 10 Mar 2024 01:29:37 +0530 Subject: [PATCH 55/73] Structured logging for sender report data. (#2563) --- pkg/sfu/buffer/rtpstats_base.go | 13 +++++++++++++ pkg/sfu/buffer/rtpstats_receiver.go | 22 +++++++++++++++------- pkg/sfu/buffer/rtpstats_sender.go | 20 ++++++++++---------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 7fc6a5797..8cc73db9d 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/mediatransportutil" @@ -123,6 +124,18 @@ func (r *RTCPSenderReportData) ToString() string { return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s", r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, r.At.String()) } +func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r == nil { + return nil + } + + e.AddTime("NTPTimestamp", r.NTPTimestamp.Time()) + e.AddUint32("RTPTimestamp", r.RTPTimestamp) + e.AddUint64("RTPTimestampExt", r.RTPTimestampExt) + e.AddTime("At", r.At) + return nil +} + // ------------------------------------------------------------------ type RTPStatsParams struct { diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 347f98103..12ab315af 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -252,8 +252,9 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.srNewest != nil && r.srNewest.NTPTimestamp > srData.NTPTimestamp { r.logger.Infow( "received anachronous sender report", - "last", r.srNewest.ToString(), - "current", srData.ToString(), + "first", r.srFirst, + "last", r.srNewest, + "current", srData, ) return } @@ -264,6 +265,12 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { tsCycles += (1 << 32) } + + if tsCycles >= (1 << 32) { + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) >= (1<<31) && srData.RTPTimestamp > r.srNewest.RTPTimestamp { + tsCycles -= (1 << 32) + } + } } srDataCopy := *srData @@ -279,8 +286,9 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.outOfOrderSsenderReportCount%10 == 0 { r.logger.Infow( "received sender report, out-of-order, resetting", - "last", r.srNewest.ToString(), - "current", srDataCopy.ToString(), + "first", r.srFirst, + "last", r.srNewest, + "current", &srDataCopy, "count", r.outOfOrderSsenderReportCount, ) } @@ -304,9 +312,9 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.clockSkewCount%100 == 0 { r.logger.Infow( "clock rate skew", - "first", r.srFirst.ToString(), - "last", r.srNewest.ToString(), - "current", srDataCopy.ToString(), + "first", r.srFirst, + "last", r.srNewest, + "current", &srDataCopy, "timeSinceFirst", timeSinceFirst, "rtpDiffSinceFirst", rtpDiffSinceFirst, "calculatedFirst", calculatedClockRateFromFirst, diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 33a22afbf..7e8b22510 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -682,11 +682,11 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui if r.clockSkewCount%10 == 0 { r.logger.Infow( "sending sender report, clock skew", - "first", r.srFirst.ToString(), - "last", r.srNewest.ToString(), - "curr", srData.ToString(), - "firstFeed", r.srFeedFirst.ToString(), - "lastFeed", r.srFeedNewest.ToString(), + "first", r.srFirst, + "last", r.srNewest, + "curr", srData, + "firstFeed", r.srFeedFirst, + "lastFeed", r.srFeedNewest, "timeNow", time.Now().String(), "extStartTS", r.extStartTS, "extHighestTS", r.extHighestTS, @@ -722,11 +722,11 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui if r.outOfOrderSenderReportCount%10 == 0 { r.logger.Infow( "sending sender report, out-of-order, repairing", - "first", r.srFirst.ToString(), - "last", r.srNewest.ToString(), - "curr", srData.ToString(), - "firstFeed", r.srFeedFirst.ToString(), - "lastFeed", r.srFeedNewest.ToString(), + "first", r.srFirst, + "last", r.srNewest, + "curr", srData, + "firstFeed", r.srFeedFirst, + "lastFeed", r.srFeedNewest, "timeNow", time.Now().String(), "extStartTS", r.extStartTS, "extHighestTS", r.extHighestTS, From bdbc9dcbc707639d534d41c099f7780db1803fe0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 10 Mar 2024 23:18:54 +0530 Subject: [PATCH 56/73] Use start time stamp to calculate down stream sender report. (#2564) * Use start time stamp to calculate down stream sender report. With first packet time adjustment, using the first time stamp is more accurate. This still suffers if the up stream clock rate changes (happens in cases like noise suppression which is not well understood). Will be looking at pass through of sender report from publisher to subscriber. * similar log strings * avoid early sender reports * log messages * Reduce first packet adjustment threshold to 15 seconds --- pkg/sfu/buffer/rtpstats_base.go | 24 ++++---- pkg/sfu/buffer/rtpstats_receiver.go | 4 +- pkg/sfu/buffer/rtpstats_sender.go | 85 +++++++++++------------------ 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 8cc73db9d..7af312c11 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -34,7 +34,7 @@ const ( cFirstSnapshotID = 1 cFirstPacketTimeAdjustWindow = 2 * time.Minute - cFirstPacketTimeAdjustThreshold = 5 * time.Minute + cFirstPacketTimeAdjustThreshold = 15 * time.Second ) // ------------------------------------------------------- @@ -506,18 +506,8 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(ts uint32, startTS uint32) { now := r.firstTime.Add(timeSinceFirst) firstTime := now.Add(-samplesDuration) if firstTime.Before(r.firstTime) { - r.logger.Debugw( - "adjusting first packet time", - "startTime", r.startTime.String(), - "nowTime", now.String(), - "before", r.firstTime.String(), - "after", firstTime.String(), - "adjustment", r.firstTime.Sub(firstTime).String(), - "nowTS", ts, - "startTS", startTS, - ) if r.firstTime.Sub(firstTime) > cFirstPacketTimeAdjustThreshold { - r.logger.Infow("first packet time adjustment too big, ignoring", + r.logger.Infow("adjusting first packet time, too big, ignoring", "startTime", r.startTime.String(), "nowTime", now.String(), "before", r.firstTime.String(), @@ -527,6 +517,16 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(ts uint32, startTS uint32) { "startTS", startTS, ) } else { + r.logger.Debugw( + "adjusting first packet time", + "startTime", r.startTime.String(), + "nowTime", now.String(), + "before", r.firstTime.String(), + "after", firstTime.String(), + "adjustment", r.firstTime.Sub(firstTime).String(), + "nowTS", ts, + "startTS", startTS, + ) r.firstTime = firstTime } } diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 12ab315af..5b5e502ca 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -251,7 +251,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) // prevent against extreme case of anachronous sender reports if r.srNewest != nil && r.srNewest.NTPTimestamp > srData.NTPTimestamp { r.logger.Infow( - "received anachronous sender report", + "received sender report, anachronous, dropping", "first", r.srFirst, "last", r.srNewest, "current", srData, @@ -311,7 +311,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) (timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) { if r.clockSkewCount%100 == 0 { r.logger.Infow( - "clock rate skew", + "received sender report, clock skew", "first", r.srFirst, "last", r.srNewest, "current", &srDataCopy, diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 7e8b22510..c748b483a 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -29,6 +29,8 @@ import ( const ( cSnInfoSize = 4096 cSnInfoMask = cSnInfoSize - 1 + + cSenderReportInitialWait = time.Second ) type snInfoFlag byte @@ -157,9 +159,8 @@ type RTPStatsSender struct { nextSenderSnapshotID uint32 senderSnapshots []senderSnapshot - clockSkewCount int - outOfOrderSenderReportCount int - metadataCacheOverflowCount int + clockSkewCount int + metadataCacheOverflowCount int srFeedFirst *RTCPSenderReportData srFeedNewest *RTCPSenderReportData @@ -648,13 +649,13 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui // construct current time based on monotonic clock timeSinceFirst := time.Since(r.firstTime) + if timeSinceFirst < cSenderReportInitialWait { + return nil + } now := r.firstTime.Add(timeSinceFirst) nowNTP := mediatransportutil.ToNtpTime(now) - - timeSinceHighest := now.Sub(r.highestTime) - nowRTPExt := r.extHighestTS + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) - nowRTPExtUsingTime := nowRTPExt - nowRTP := uint32(nowRTPExt) + nowRTPExtUsingTime := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) + nowRTPExt := nowRTPExtUsingTime // It is possible that publisher is pacing at a slower rate. // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. @@ -664,13 +665,12 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) if nowRTPExtUsingRate > nowRTPExt { nowRTPExt = nowRTPExtUsingRate - nowRTP = uint32(nowRTPExt) } } srData := &RTCPSenderReportData{ NTPTimestamp: nowNTP, - RTPTimestamp: nowRTP, + RTPTimestamp: uint32(nowRTPExt), RTPTimestampExt: nowRTPExt, At: now, } @@ -691,7 +691,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui "extStartTS", r.extStartTS, "extHighestTS", r.extHighestTS, "highestTime", r.highestTime.String(), - "timeSinceHighest", timeSinceHighest.String(), + "timeSinceHighest", now.Sub(r.highestTime).String(), "firstTime", r.firstTime.String(), "timeSinceFirst", timeSinceFirst.String(), "nowRTPExtUsingTime", nowRTPExtUsingTime, @@ -708,46 +708,27 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui } if r.srNewest != nil && nowRTPExt < r.srNewest.RTPTimestampExt { - // If report being generated is behind, use the time difference and - // clock rate of codec to produce next report. - // - // Current report could be behind due to the following - // - Publisher pacing - // - Due to above, report from publisher side is ahead of packet timestamps. - // Note that report will map wall clock to timestamp at capture time and happens before the pacer. - // - Pause/Mute followed by resume, some combination of events that could - // result in this module not having calculated clock rate of publisher side. - // - When the above happens, current will be generated using highestTS which could be behind. - // That could end up behind the last report's timestamp in extreme cases - if r.outOfOrderSenderReportCount%10 == 0 { - r.logger.Infow( - "sending sender report, out-of-order, repairing", - "first", r.srFirst, - "last", r.srNewest, - "curr", srData, - "firstFeed", r.srFeedFirst, - "lastFeed", r.srFeedNewest, - "timeNow", time.Now().String(), - "extStartTS", r.extStartTS, - "extHighestTS", r.extHighestTS, - "highestTime", r.highestTime.String(), - "timeSinceHighest", timeSinceHighest.String(), - "firstTime", r.firstTime.String(), - "timeSinceFirst", timeSinceFirst.String(), - "nowRTPExtUsingTime", nowRTPExtUsingTime, - "calculatedClockRate", calculatedClockRate, - "nowRTPExtUsingRate", nowRTPExtUsingRate, - "count", r.outOfOrderSenderReportCount, - ) - } - r.outOfOrderSenderReportCount++ - - ntpDiffSinceLast := nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) - nowRTPExt = r.srNewest.RTPTimestampExt + uint64(ntpDiffSinceLast.Seconds()*float64(r.params.ClockRate)) - nowRTP = uint32(nowRTPExt) - - srData.RTPTimestamp = nowRTP - srData.RTPTimestampExt = nowRTPExt + // If report being generated is behind the last report, skip it. + // Should not happen. + r.logger.Infow( + "sending sender report, out-of-order, skipping", + "first", r.srFirst, + "last", r.srNewest, + "curr", srData, + "firstFeed", r.srFeedFirst, + "lastFeed", r.srFeedNewest, + "timeNow", time.Now().String(), + "extStartTS", r.extStartTS, + "extHighestTS", r.extHighestTS, + "highestTime", r.highestTime.String(), + "timeSinceHighest", now.Sub(r.highestTime).String(), + "firstTime", r.firstTime.String(), + "timeSinceFirst", timeSinceFirst.String(), + "nowRTPExtUsingTime", nowRTPExtUsingTime, + "calculatedClockRate", calculatedClockRate, + "nowRTPExtUsingRate", nowRTPExtUsingRate, + ) + return nil } r.srNewest = srData @@ -758,7 +739,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui return &rtcp.SenderReport{ SSRC: ssrc, NTPTime: uint64(nowNTP), - RTPTime: nowRTP, + RTPTime: uint32(nowRTPExt), PacketCount: uint32(r.getTotalPacketsPrimary(r.extStartSN, r.extHighestSN) + r.packetsDuplicate + r.packetsPadding), OctetCount: uint32(r.bytes + r.bytesDuplicate + r.bytesPadding), } From 93c7d1f4fb6246bac7393cb6e7f90441491ff7df Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 11 Mar 2024 00:40:16 +0530 Subject: [PATCH 57/73] Adjust first packet time on down track resume. (#2566) Allows subscriber sender report to line up better quicker. --- pkg/rtc/wrappedreceiver.go | 8 ++++++++ pkg/sfu/buffer/rtpstats_sender.go | 16 ++++++++++++---- pkg/sfu/downtrack.go | 23 +++++++++++++++++------ pkg/sfu/receiver.go | 5 +++++ pkg/sfu/streamtrackermanager.go | 17 +++++++++++++++++ 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 677da8c3d..3abd3d584 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -26,6 +26,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -330,6 +331,13 @@ func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, re return 0, errors.New("receiver not available") } +func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + return r.GetRTCPSenderReportData(layer) + } + return nil, nil +} + func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetTrackStats() diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index c748b483a..e568bd6f7 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -615,11 +615,19 @@ func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(srFirst *RTCPSenderReportDat r.lock.Lock() defer r.lock.Unlock() - srFirstCopy := *srFirst - r.srFeedFirst = &srFirstCopy + if !r.initialized { + return + } - srNewestCopy := *srNewest - r.srFeedNewest = &srNewestCopy + if srFirst != nil { + srFirstCopy := *srFirst + r.srFeedFirst = &srFirstCopy + } + + if srNewest != nil { + srNewestCopy := *srNewest + r.srFeedNewest = &srNewestCopy + } r.maybeAdjustFirstPacketTime(ts, uint32(r.extStartTS)) } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 05da21add..1ed09223c 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1951,16 +1951,24 @@ func (d *DownTrack) HandleRTCPSenderReportData( srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData, ) error { - if (layer == d.forwarder.GetReferenceLayerSpatial() || (layer == 0 && isSVC)) && srNewest != nil { - d.rtpStats.MaybeAdjustFirstPacketTime( - srFirst, - srNewest, - srNewest.RTPTimestamp+uint32(d.forwarder.GetReferenceTimestampOffset()), - ) + if layer == d.forwarder.GetReferenceLayerSpatial() || (layer == 0 && isSVC) { + d.handleRTCPSenderReportData(srFirst, srNewest) } return nil } +func (d *DownTrack) handleRTCPSenderReportData(srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { + if srNewest == nil { + return + } + + d.rtpStats.MaybeAdjustFirstPacketTime( + srFirst, + srNewest, + srNewest.RTPTimestamp+uint32(d.forwarder.GetReferenceTimestampOffset()), + ) +} + type sendPacketMetadata struct { layer int32 packetTime time.Time @@ -2010,6 +2018,9 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa } if spmd.tp.isResuming { + // adjust first packet time on a resumption so that sender reports can lock in quicker + d.handleRTCPSenderReportData(d.params.Receiver.GetRTCPSenderReportData(d.forwarder.GetReferenceLayerSpatial())) + if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnResume(d) } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index e33f06eb4..9407303ef 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -85,6 +85,7 @@ type TrackReceiver interface { GetCalculatedClockRate(layer int32) uint32 GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) + GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) GetTrackStats() *livekit.RTPStats } @@ -790,6 +791,10 @@ func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, r return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } +func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + return w.streamTrackerManager.GetRTCPSenderReportData(layer) +} + // closes all track senders in parallel, returns when all are closed func closeTrackSenders(senders []TrackSender) { wg := sync.WaitGroup{} diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 36ff551a2..8acb64518 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -626,6 +626,23 @@ func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buf } } +func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + s.senderReportMu.Lock() + defer s.senderReportMu.Unlock() + + if layer < 0 || int(layer) >= len(s.senderReports) { + return nil, nil + } + + // SVC-TODO: better SVC detection + if s.isSVC { + // there is only one stream in SVC + layer = 0 + } + + return s.senderReports[layer].first, s.senderReports[layer].newest +} + func (s *StreamTrackerManager) GetCalculatedClockRate(layer int32) uint32 { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() From 50c48ff29d0913cc940113d12905f8b70e41529f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 11 Mar 2024 11:30:01 +0530 Subject: [PATCH 58/73] Ignore out-of-order receiver side sender reports. (#2567) --- pkg/sfu/buffer/rtpstats_receiver.go | 14 ++++++-------- pkg/sfu/buffer/rtpstats_sender.go | 27 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 5b5e502ca..2a789c443 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -276,16 +276,14 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) srDataCopy := *srData srDataCopy.RTPTimestampExt = uint64(srDataCopy.RTPTimestamp) + tsCycles - r.maybeAdjustFirstPacketTime(srDataCopy.RTPTimestamp, r.timestamp.GetStart()) - if r.srNewest != nil && srDataCopy.RTPTimestampExt < r.srNewest.RTPTimestampExt { // This can happen when a track is replaced with a null and then restored - // i. e. muting replacing with null and unmute restoring the original track. - // Under such a condition reset the sender reports to start from this point. - // Resetting will ensure sample rate calculations do not go haywire due to negative time. + // Or it could be due bad report generation. + // In any case, ignore out-of-order reports. if r.outOfOrderSsenderReportCount%10 == 0 { r.logger.Infow( - "received sender report, out-of-order, resetting", + "received sender report, out-of-order, skipping", "first", r.srFirst, "last", r.srNewest, "current", &srDataCopy, @@ -293,11 +291,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) ) } r.outOfOrderSsenderReportCount++ - - r.srFirst = nil - r.srNewest = nil + return } + r.maybeAdjustFirstPacketTime(srDataCopy.RTPTimestamp, r.timestamp.GetStart()) + if r.srNewest != nil { timeSinceLast := srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()).Seconds() rtpDiffSinceLast := srDataCopy.RTPTimestampExt - r.srNewest.RTPTimestampExt diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index e568bd6f7..585d9cb5c 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -665,16 +665,21 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui nowRTPExtUsingTime := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) nowRTPExt := nowRTPExtUsingTime - // It is possible that publisher is pacing at a slower rate. - // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. - // Check for that using calculated clock rate and use the later time stamp if applicable. - var nowRTPExtUsingRate uint64 - if calculatedClockRate != 0 { - nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) - if nowRTPExtUsingRate > nowRTPExt { - nowRTPExt = nowRTPExtUsingRate + /* + // TODO: Bad reports or unpaced publishing contorts the calculated clock rate a lot resulting in + // subsscriber sender reports jumping around. Needs more thinking. + // + // It is possible that publisher is pacing at a slower rate. + // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. + // Check for that using calculated clock rate and use the later time stamp if applicable. + var nowRTPExtUsingRate uint64 + if calculatedClockRate != 0 { + nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) + if nowRTPExtUsingRate > nowRTPExt { + nowRTPExt = nowRTPExtUsingRate + } } - } + */ srData := &RTCPSenderReportData{ NTPTimestamp: nowNTP, @@ -704,7 +709,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui "timeSinceFirst", timeSinceFirst.String(), "nowRTPExtUsingTime", nowRTPExtUsingTime, "calculatedClockRate", calculatedClockRate, - "nowRTPExtUsingRate", nowRTPExtUsingRate, + // TODO "nowRTPExtUsingRate", nowRTPExtUsingRate, "timeSinceLastReport", timeSinceLastReport.String(), "rtpDiffSinceLastReport", rtpDiffSinceLastReport, "windowClockRate", windowClockRate, @@ -734,7 +739,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui "timeSinceFirst", timeSinceFirst.String(), "nowRTPExtUsingTime", nowRTPExtUsingTime, "calculatedClockRate", calculatedClockRate, - "nowRTPExtUsingRate", nowRTPExtUsingRate, + // TODO "nowRTPExtUsingRate", nowRTPExtUsingRate, ) return nil } From 610d68a4098cd99e63f2cd0f4295450d130e26e5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 11 Mar 2024 12:25:07 +0530 Subject: [PATCH 59/73] Clean up using publisher side clock rate. (#2568) It is not used any more. --- pkg/rtc/wrappedreceiver.go | 7 ------- pkg/sfu/buffer/rtpstats_sender.go | 29 ++++----------------------- pkg/sfu/downtrack.go | 2 +- pkg/sfu/receiver.go | 5 ----- pkg/sfu/streamtrackermanager.go | 33 ------------------------------- 5 files changed, 5 insertions(+), 71 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 3abd3d584..e088c9eb3 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -317,13 +317,6 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetCalculatedClockRate(layer int32) uint32 { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetCalculatedClockRate(layer) - } - return 0 -} - func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 585d9cb5c..b78dc6b2a 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -647,7 +647,7 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui return } -func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) *rtcp.SenderReport { +func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() @@ -662,24 +662,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui } now := r.firstTime.Add(timeSinceFirst) nowNTP := mediatransportutil.ToNtpTime(now) - nowRTPExtUsingTime := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - nowRTPExt := nowRTPExtUsingTime - - /* - // TODO: Bad reports or unpaced publishing contorts the calculated clock rate a lot resulting in - // subsscriber sender reports jumping around. Needs more thinking. - // - // It is possible that publisher is pacing at a slower rate. - // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. - // Check for that using calculated clock rate and use the later time stamp if applicable. - var nowRTPExtUsingRate uint64 - if calculatedClockRate != 0 { - nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) - if nowRTPExtUsingRate > nowRTPExt { - nowRTPExt = nowRTPExtUsingRate - } - } - */ + nowRTPExt := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) srData := &RTCPSenderReportData{ NTPTimestamp: nowNTP, @@ -707,9 +690,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui "timeSinceHighest", now.Sub(r.highestTime).String(), "firstTime", r.firstTime.String(), "timeSinceFirst", timeSinceFirst.String(), - "nowRTPExtUsingTime", nowRTPExtUsingTime, - "calculatedClockRate", calculatedClockRate, - // TODO "nowRTPExtUsingRate", nowRTPExtUsingRate, + "nowRTPExt", nowRTPExt, "timeSinceLastReport", timeSinceLastReport.String(), "rtpDiffSinceLastReport", rtpDiffSinceLastReport, "windowClockRate", windowClockRate, @@ -737,9 +718,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui "timeSinceHighest", now.Sub(r.highestTime).String(), "firstTime", r.firstTime.String(), "timeSinceFirst", timeSinceFirst.String(), - "nowRTPExtUsingTime", nowRTPExtUsingTime, - "calculatedClockRate", calculatedClockRate, - // TODO "nowRTPExtUsingRate", nowRTPExtUsingRate, + "nowRTPExt", nowRTPExt, ) return nil } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 1ed09223c..1bc07b8ff 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1307,7 +1307,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { if clockLayer == buffer.InvalidLayerSpatial { clockLayer = d.forwarder.GetReferenceLayerSpatial() } - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.params.Receiver.GetCalculatedClockRate(clockLayer)) + return d.rtpStats.GetRtcpSenderReport(d.ssrc) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 9407303ef..eec709dba 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -83,7 +83,6 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 - GetCalculatedClockRate(layer int32) uint32 GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) @@ -783,10 +782,6 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetCalculatedClockRate(layer int32) uint32 { - return w.streamTrackerManager.GetCalculatedClockRate(layer) -} - func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 8acb64518..2b80e7c25 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -35,8 +35,6 @@ import ( const ( senderReportThresholdSeconds = float64(60.0) - - minDurationForClockRateCalculation = 15 * time.Second ) // --------------------------------------------------- @@ -643,37 +641,6 @@ func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTC return s.senderReports[layer].first, s.senderReports[layer].newest } -func (s *StreamTrackerManager) GetCalculatedClockRate(layer int32) uint32 { - s.senderReportMu.RLock() - defer s.senderReportMu.RUnlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - // invalid layer - return 0 - } - - srFirst := s.senderReports[layer].first - srNewest := s.senderReports[layer].newest - if srFirst == nil || srFirst.NTPTimestamp == 0 || srNewest == nil || srNewest.NTPTimestamp == 0 || srFirst.RTPTimestamp == srNewest.RTPTimestamp { - // sender reports invalid or same - return 0 - } - - if s.senderReports[layer].lastUpdated.IsZero() || time.Since(s.senderReports[layer].lastUpdated).Seconds() > senderReportThresholdSeconds { - // sender report updated too far back - return 0 - } - - tsf := srNewest.NTPTimestamp.Time().Sub(srFirst.NTPTimestamp.Time()) - if tsf < minDurationForClockRateCalculation { - // not enough time has elapsed to get a stable clock rate calculation - return 0 - } - - rdsf := srNewest.RTPTimestampExt - srFirst.RTPTimestampExt - return uint32(float64(rdsf) / tsf.Seconds()) -} - func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() From 1cf04e35110ab58246bb795168faa1e7bf992ad6 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 11 Mar 2024 18:53:12 -0700 Subject: [PATCH 60/73] add egress proxy proto (#2570) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 705123128..317336199 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6 + github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49 github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -62,7 +62,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.2 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -91,7 +91,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/puzpuzpuz/xsync v1.5.2 // indirect + github.com/puzpuzpuz/xsync/v3 v3.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect diff --git a/go.sum b/go.sum index 3dec16926..fc8764a1a 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= -github.com/go-jose/go-jose/v3 v3.0.2 h1:2Edjn8Nrb44UvTdp84KU0bBPs1cO7noRCybtS3eJEUQ= -github.com/go-jose/go-jose/v3 v3.0.2/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= +github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -130,8 +130,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6 h1:jWWMJP1x96C/3oHO2asGFzwa7xDU4NrWcyt0b0irqOQ= -github.com/livekit/protocol v1.11.1-0.20240308170839-fae28895efe6/go.mod h1:XpJ2t2wFnnQghPpkxXAzMZhYMDnm8wWxdxYJK4fP9gM= +github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49 h1:eI3blFvnnFvAaT5oTbVZBTQCufv9OEplW8ejYWDR6tY= +github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49/go.mod h1:x+QyergF26N374J0HxxDw6AiMEtf8wNtJS7yJrWGufA= github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 h1:253WtQ2VGVHzIIzW9MUZj7vUDDILESU3zsEbiRdxYF0= github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -240,8 +240,8 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= -github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg= +github.com/puzpuzpuz/xsync/v3 v3.1.0 h1:EewKT7/LNac5SLiEblJeUu8z5eERHrmRLnMQL2d7qX4= +github.com/puzpuzpuz/xsync/v3 v3.1.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 3e43f75143fa3d64573bb9163f1673eb6e4d759b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 13 Mar 2024 14:31:39 +0530 Subject: [PATCH 61/73] Forward publisher sender report. (#2572) * Forward publisher sender report. Publisher side RTCP sernfer report is rebased to SFU time base and used to send sender rerport to subscriber. Will wait to merge till previous versions are out as this will require a bunch of testing. * - Add rebased report drift - update protocol dep - fix path change check, it has to check against delta of propagation delay and not propagation delay as the two side clocks could be way off. --- go.mod | 21 ++--- go.sum | 40 +++++----- pkg/rtc/wrappedreceiver.go | 4 +- pkg/sfu/buffer/buffer.go | 4 +- pkg/sfu/buffer/rtpstats_base.go | 70 ++++++++++------- pkg/sfu/buffer/rtpstats_receiver.go | 115 +++++++++++++++++++++++++--- pkg/sfu/buffer/rtpstats_sender.go | 96 ++++++++--------------- pkg/sfu/downtrack.go | 35 +++------ pkg/sfu/forwarder.go | 13 ++-- pkg/sfu/receiver.go | 10 +-- pkg/sfu/rtpmunger.go | 4 + pkg/sfu/streamtrackermanager.go | 24 ++---- 12 files changed, 248 insertions(+), 188 deletions(-) diff --git a/go.mod b/go.mod index 317336199..63a5c4fcb 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49 - github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 + github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626 + github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -50,7 +50,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -62,9 +62,10 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -72,13 +73,13 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.32.0 // indirect + github.com/nats-io/nats.go v1.33.1 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -96,13 +97,13 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/grpc v1.62.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/grpc v1.62.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index fc8764a1a..8b2635d44 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/frostbyte73/core v0.0.10 h1:D4DQXdPb8ICayz0n75rs4UYTXrUSdxzUfeleuNJOR github.com/frostbyte73/core v0.0.10/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -58,8 +60,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -111,8 +113,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -130,10 +132,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49 h1:eI3blFvnnFvAaT5oTbVZBTQCufv9OEplW8ejYWDR6tY= -github.com/livekit/protocol v1.11.1-0.20240311174744-00c977ffbb49/go.mod h1:x+QyergF26N374J0HxxDw6AiMEtf8wNtJS7yJrWGufA= -github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4 h1:253WtQ2VGVHzIIzW9MUZj7vUDDILESU3zsEbiRdxYF0= -github.com/livekit/psrpc v0.5.3-0.20240228172457-3724cb4adbc4/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626 h1:FMN/mD/7e06gDi51sT0BdaTGIWIcdK140YTQNJI4HrY= +github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626/go.mod h1:znZpBU024XvKIpC9jkFzfrJMDuucDb4B1ON9lbUJMGA= +github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= +github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -163,8 +165,8 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0= -github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -304,8 +306,9 @@ golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIi golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -396,8 +399,9 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -434,10 +438,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -446,8 +450,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index e088c9eb3..442a8df83 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -324,11 +324,11 @@ func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, re return 0, errors.New("receiver not available") } -func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { +func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetRTCPSenderReportData(layer) } - return nil, nil + return nil } func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats { diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index b2e6b75a7..123472ad6 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -840,7 +840,7 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { } } -func (b *Buffer) GetSenderReportData() (*RTCPSenderReportData, *RTCPSenderReportData) { +func (b *Buffer) GetSenderReportData() *RTCPSenderReportData { b.RLock() defer b.RUnlock() @@ -848,7 +848,7 @@ func (b *Buffer) GetSenderReportData() (*RTCPSenderReportData, *RTCPSenderReport return b.rtpStats.GetRtcpSenderReportData() } - return nil, nil + return nil } func (b *Buffer) SetLastFractionLostReport(lost uint8) { diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 7af312c11..11c9517c6 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -484,7 +484,7 @@ func (r *rtpStatsBase) GetRtt() uint32 { return r.rtt } -func (r *rtpStatsBase) maybeAdjustFirstPacketTime(ts uint32, startTS uint32) { +func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *RTCPSenderReportData, tsOffset uint64, extStartTS uint64) { if time.Since(r.startTime) > cFirstPacketTimeAdjustWindow { return } @@ -495,7 +495,9 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(ts uint32, startTS uint32) { // abnormal delay (maybe due to pacing or maybe due to queuing // in some network element along the way), push back first time // to an earlier instance. - samplesDiff := int32(ts - startTS) + timeSinceReceive := time.Since(srData.At) + extNowTS := srData.RTPTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9) + samplesDiff := int64(extNowTS - extStartTS) if samplesDiff < 0 { // out-of-order, skip return @@ -505,28 +507,24 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(ts uint32, startTS uint32) { timeSinceFirst := time.Since(r.firstTime) now := r.firstTime.Add(timeSinceFirst) firstTime := now.Add(-samplesDuration) + + getFields := func() []interface{} { + return []interface{}{ + "startTime", r.startTime.String(), + "nowTime", now.String(), + "before", r.firstTime.String(), + "after", firstTime.String(), + "adjustment", r.firstTime.Sub(firstTime).String(), + "extNowTS", extNowTS, + "extStartTS", extStartTS, + } + } + if firstTime.Before(r.firstTime) { if r.firstTime.Sub(firstTime) > cFirstPacketTimeAdjustThreshold { - r.logger.Infow("adjusting first packet time, too big, ignoring", - "startTime", r.startTime.String(), - "nowTime", now.String(), - "before", r.firstTime.String(), - "after", firstTime.String(), - "adjustment", r.firstTime.Sub(firstTime).String(), - "nowTS", ts, - "startTS", startTS, - ) + r.logger.Infow("adjusting first packet time, too big, ignoring", getFields()...) } else { - r.logger.Debugw( - "adjusting first packet time", - "startTime", r.startTime.String(), - "nowTime", now.String(), - "before", r.firstTime.String(), - "after", firstTime.String(), - "adjustment", r.firstTime.Sub(firstTime).String(), - "nowTS", ts, - "startTS", startTS, - ) + r.logger.Debugw("adjusting first packet time", getFields()...) r.firstTime = firstTime } } @@ -678,7 +676,7 @@ func (r *rtpStatsBase) toString( str += ", rtt(ms):" str += fmt.Sprintf("%d|%d", p.RttCurrent, p.RttMax) - str += fmt.Sprintf(", pd: %s, rd: %s", RTPDriftToString(p.PacketDrift), RTPDriftToString(p.ReportDrift)) + str += fmt.Sprintf(", pd: %s, nrd: %s, rrd: %s", RTPDriftToString(p.PacketDrift), RTPDriftToString(p.ReportDrift), RTPDriftToString(p.RebasedReportDrift)) return str } @@ -719,7 +717,7 @@ func (r *rtpStatsBase) toProto( jitterTime := jitter / float64(r.params.ClockRate) * 1e6 maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 - packetDrift, reportDrift := r.getDrift(extStartTS, extHighestTS) + packetDrift, ntpReportDrift, rebasedReportDrift := r.getDrift(extStartTS, extHighestTS) p := &livekit.RTPStats{ StartTime: timestamppb.New(r.startTime), @@ -763,7 +761,8 @@ func (r *rtpStatsBase) toProto( RttCurrent: r.rtt, RttMax: r.maxRtt, PacketDrift: packetDrift, - ReportDrift: reportDrift, + ReportDrift: ntpReportDrift, + RebasedReportDrift: rebasedReportDrift, } gapsPresent := false @@ -845,7 +844,7 @@ func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, return &then, &now } -func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *livekit.RTPDrift, reportDrift *livekit.RTPDrift) { +func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *livekit.RTPDrift, ntpReportDrift *livekit.RTPDrift, rebasedReportDrift *livekit.RTPDrift) { if !r.firstTime.IsZero() { elapsed := r.highestTime.Sub(r.firstTime) rtpClockTicks := extHighestTS - extStartTS @@ -866,11 +865,12 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l } if r.srFirst != nil && r.srNewest != nil && r.srFirst.RTPTimestamp != r.srNewest.RTPTimestamp { - elapsed := r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) rtpClockTicks := r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt + + elapsed := r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) if elapsed.Seconds() > 0.0 { - reportDrift = &livekit.RTPDrift{ + ntpReportDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(r.srFirst.NTPTimestamp.Time()), EndTime: timestamppb.New(r.srNewest.NTPTimestamp.Time()), Duration: elapsed.Seconds(), @@ -882,6 +882,22 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), } } + + elapsed = r.srNewest.At.Sub(r.srFirst.At) + driftSamples = int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + if elapsed.Seconds() > 0.0 { + rebasedReportDrift = &livekit.RTPDrift{ + StartTime: timestamppb.New(r.srFirst.At), + EndTime: timestamppb.New(r.srNewest.At), + Duration: elapsed.Seconds(), + StartTimestamp: r.srFirst.RTPTimestampExt, + EndTimestamp: r.srNewest.RTPTimestampExt, + RtpClockTicks: rtpClockTicks, + DriftSamples: driftSamples, + DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate), + ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), + } + } } return } diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 2a789c443..c54dd53c8 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -28,6 +28,32 @@ import ( const ( cHistorySize = 4096 + + // RTCP Sender Reports are re-based to SFU time base so that all subscriber side + // can have the same time base (i. e. SFU time base). To convert publisher side + // RTCP Sender Reports to SFU timebase, a propagation delay is maintained. + // propagation_delay = time_of_report_reception - ntp_timestamp_in_report + // + // Propagation delay is adapted continuously. If it falls, adapt quickly to the + // lower value as that could be the real propagation delay. If it rises, adapt slowly + // as it might be a temporary change or slow drift. See below for handling of high deltas + // which could be a result of a path change. + cPropagationDelayFallFactor = float64(0.95) + cPropagationDelayRiseFactor = float64(0.05) + + // do not adapt to small OR large (outlier) changes + cPropagationDelayDeltaThresholdMin = 5 * time.Millisecond + cPropagationDelayDeltaThresholdMaxFactor = 2 + + // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. + // Reset at whichever of the below happens later. + // + // A smoothed version of delta of propagation delay is maintained and delta propagation delay exceeding + // a factor of the smoothed version is considered a sharp increase. That will trigger the start of the + // path change condition and if it persists, propagation delay will be reset. + cPropagationDelayDeltaAdaptationFactor = float64(0.1) + cPropagationDelayDeltaHighResetNumReports = 3 + cPropagationDelayDeltaHighResetWait = 10 * time.Second ) type RTPFlowState struct { @@ -53,6 +79,11 @@ type RTPStatsReceiver struct { history *protoutils.Bitmap[uint64] + propagationDelay time.Duration + smoothedDeltaPropagationDelay time.Duration + propagationDelayDeltaHighCount int + propagationDelayDeltaHighStartTime time.Time + clockSkewCount int outOfOrderSsenderReportCount int } @@ -294,8 +325,6 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) return } - r.maybeAdjustFirstPacketTime(srDataCopy.RTPTimestamp, r.timestamp.GetStart()) - if r.srNewest != nil { timeSinceLast := srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()).Seconds() rtpDiffSinceLast := srDataCopy.RTPTimestampExt - r.srNewest.RTPTimestampExt @@ -326,26 +355,88 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) } } - r.srNewest = &srDataCopy + var propagationDelay time.Duration + var deltaPropagationDelay time.Duration + getPropagationFields := func() []interface{} { + return []interface{}{ + "propagationDelay", r.propagationDelay.String(), + "receivedPropagationDelay", propagationDelay.String(), + "smoothedDeltaPropagationDelay", r.smoothedDeltaPropagationDelay.String(), + "receivedDeltaPropagationDelay", deltaPropagationDelay.String(), + "deltaHighCount", r.propagationDelayDeltaHighCount, + "sinceDeltaHighStart", time.Since(r.propagationDelayDeltaHighStartTime).String(), + "first", r.srFirst, + "last", r.srNewest, + "current", &srDataCopy, + } + } + initPropagationDelay := func(pd time.Duration) { + r.propagationDelay = pd + r.smoothedDeltaPropagationDelay = 0 + r.propagationDelayDeltaHighCount = 0 + r.propagationDelayDeltaHighStartTime = time.Time{} + } + + ntpTime := srDataCopy.NTPTimestamp.Time() + propagationDelay = srDataCopy.At.Sub(ntpTime) if r.srFirst == nil { r.srFirst = &srDataCopy + initPropagationDelay(propagationDelay) + r.logger.Debugw("initializing propagation delay", getPropagationFields()...) + } else { + deltaPropagationDelay = propagationDelay - r.propagationDelay + r.logger.Debugw("RAJA pd", getPropagationFields()...) // REMOVE + if r.smoothedDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.smoothedDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { + r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE + r.propagationDelayDeltaHighCount++ + if r.propagationDelayDeltaHighStartTime.IsZero() { + r.propagationDelayDeltaHighStartTime = time.Now() + } + + if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { + r.logger.Infow("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay)...) + initPropagationDelay(propagationDelay) + } + } else { + if r.smoothedDeltaPropagationDelay == 0 { + r.smoothedDeltaPropagationDelay = deltaPropagationDelay + } else { + r.smoothedDeltaPropagationDelay += time.Duration(cPropagationDelayDeltaAdaptationFactor * float64(deltaPropagationDelay-r.smoothedDeltaPropagationDelay)) + } + r.propagationDelayDeltaHighCount = 0 + r.propagationDelayDeltaHighStartTime = time.Time{} + + if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { + factor := cPropagationDelayFallFactor + if propagationDelay > r.propagationDelay { + factor = cPropagationDelayRiseFactor + } + fields := append( + getPropagationFields(), + "adjustedPropagationDelay", r.propagationDelay+time.Duration(factor*float64(propagationDelay-r.propagationDelay)), + ) // TODO-REMOVE + r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE + r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) + } + } } + // adjust receive time to estimated propagation delay + srDataCopy.At = ntpTime.Add(r.propagationDelay) + r.srNewest = &srDataCopy + + r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()) } -func (r *RTPStatsReceiver) GetRtcpSenderReportData() (srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) { +func (r *RTPStatsReceiver) GetRtcpSenderReportData() *RTCPSenderReportData { r.lock.RLock() defer r.lock.RUnlock() - if r.srFirst != nil { - srFirstCopy := *r.srFirst - srFirst = &srFirstCopy + if r.srNewest == nil { + return nil } - if r.srNewest != nil { - srNewestCopy := *r.srNewest - srNewest = &srNewestCopy - } - return + srNewestCopy := *r.srNewest + return &srNewestCopy } func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotID uint32) *rtcp.ReceptionReport { diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index b78dc6b2a..4fe39c0c5 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -161,9 +161,6 @@ type RTPStatsSender struct { clockSkewCount int metadataCacheOverflowCount int - - srFeedFirst *RTCPSenderReportData - srFeedNewest *RTCPSenderReportData } func NewRTPStatsSender(params RTPStatsParams) *RTPStatsSender { @@ -202,15 +199,6 @@ func (r *RTPStatsSender) Seed(from *RTPStatsSender) { r.nextSenderSnapshotID = from.nextSenderSnapshotID r.senderSnapshots = make([]senderSnapshot, cap(from.senderSnapshots)) copy(r.senderSnapshots, from.senderSnapshots) - - if from.srFeedFirst != nil { - srFeedFirst := *from.srFeedFirst - r.srFeedFirst = &srFeedFirst - } - if from.srFeedNewest != nil { - srFeedNewest := *from.srFeedNewest - r.srFeedNewest = &srFeedNewest - } } func (r *RTPStatsSender) NewSnapshotId() uint32 { @@ -611,25 +599,15 @@ func (r *RTPStatsSender) LastReceiverReportTime() time.Time { return r.lastRRTime } -func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData, ts uint32) { +func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(publisherSRData *RTCPSenderReportData, tsOffset uint64) { r.lock.Lock() defer r.lock.Unlock() - if !r.initialized { + if !r.initialized || publisherSRData == nil { return } - if srFirst != nil { - srFirstCopy := *srFirst - r.srFeedFirst = &srFirstCopy - } - - if srNewest != nil { - srNewestCopy := *srNewest - r.srFeedNewest = &srNewestCopy - } - - r.maybeAdjustFirstPacketTime(ts, uint32(r.extStartTS)) + r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS) } func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt uint64, err error) { @@ -647,22 +625,18 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui return } -func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { +func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPSenderReportData, tsOffset uint64) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() - if !r.initialized { + if !r.initialized || publisherSRData == nil { return nil } - // construct current time based on monotonic clock - timeSinceFirst := time.Since(r.firstTime) - if timeSinceFirst < cSenderReportInitialWait { - return nil - } - now := r.firstTime.Add(timeSinceFirst) + timeSincePublisherSR := time.Since(publisherSRData.At) + now := publisherSRData.At.Add(timeSincePublisherSR) nowNTP := mediatransportutil.ToNtpTime(now) - nowRTPExt := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) + nowRTPExt := publisherSRData.RTPTimestampExt - tsOffset + uint64(timeSincePublisherSR.Nanoseconds()*int64(r.params.ClockRate)/1e9) srData := &RTCPSenderReportData{ NTPTimestamp: nowNTP, @@ -670,32 +644,39 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { RTPTimestampExt: nowRTPExt, At: now, } + + getFields := func() []interface{} { + return []interface{}{ + "first", r.srFirst, + "last", r.srNewest, + "curr", srData, + "feed", publisherSRData, + "tsOffset", tsOffset, + "timeNow", time.Now().String(), + "extStartTS", r.extStartTS, + "extHighestTS", r.extHighestTS, + "highestTime", r.highestTime.String(), + "timeSinceHighest", now.Sub(r.highestTime).String(), + "firstTime", r.firstTime.String(), + "timeSinceFirst", now.Sub(r.firstTime).String(), + "timeSincePublisherSR", timeSincePublisherSR.String(), + "nowRTPExt", nowRTPExt, + } + } if r.srNewest != nil && nowRTPExt >= r.srNewest.RTPTimestampExt { timeSinceLastReport := nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RTPTimestampExt windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds() if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) { if r.clockSkewCount%10 == 0 { - r.logger.Infow( - "sending sender report, clock skew", - "first", r.srFirst, - "last", r.srNewest, - "curr", srData, - "firstFeed", r.srFeedFirst, - "lastFeed", r.srFeedNewest, - "timeNow", time.Now().String(), - "extStartTS", r.extStartTS, - "extHighestTS", r.extHighestTS, - "highestTime", r.highestTime.String(), - "timeSinceHighest", now.Sub(r.highestTime).String(), - "firstTime", r.firstTime.String(), - "timeSinceFirst", timeSinceFirst.String(), - "nowRTPExt", nowRTPExt, + fields := append( + getFields(), "timeSinceLastReport", timeSinceLastReport.String(), "rtpDiffSinceLastReport", rtpDiffSinceLastReport, "windowClockRate", windowClockRate, "count", r.clockSkewCount, ) + r.logger.Infow("sending sender report, clock skew", fields...) } r.clockSkewCount++ } @@ -704,22 +685,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { if r.srNewest != nil && nowRTPExt < r.srNewest.RTPTimestampExt { // If report being generated is behind the last report, skip it. // Should not happen. - r.logger.Infow( - "sending sender report, out-of-order, skipping", - "first", r.srFirst, - "last", r.srNewest, - "curr", srData, - "firstFeed", r.srFeedFirst, - "lastFeed", r.srFeedNewest, - "timeNow", time.Now().String(), - "extStartTS", r.extStartTS, - "extHighestTS", r.extHighestTS, - "highestTime", r.highestTime.String(), - "timeSinceHighest", now.Sub(r.highestTime).String(), - "firstTime", r.firstTime.String(), - "timeSinceFirst", timeSinceFirst.String(), - "nowRTPExt", nowRTPExt, - ) + r.logger.Infow("sending sender report, out-of-order, skipping", getFields()...) return nil } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 1bc07b8ff..41e6eb67f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -59,8 +59,7 @@ type TrackSender interface { payloadType webrtc.PayloadType, isSVC bool, layer int32, - srFirst *buffer.RTCPSenderReportData, - srNewest *buffer.RTCPSenderReportData, + publisherSRData *buffer.RTCPSenderReportData, ) error } @@ -1303,11 +1302,8 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - clockLayer := d.forwarder.CurrentLayer().Spatial - if clockLayer == buffer.InvalidLayerSpatial { - clockLayer = d.forwarder.GetReferenceLayerSpatial() - } - return d.rtpStats.GetRtcpSenderReport(d.ssrc) + layer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() + return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.params.Receiver.GetRTCPSenderReportData(layer), tsOffset) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { @@ -1948,25 +1944,17 @@ func (d *DownTrack) HandleRTCPSenderReportData( _payloadType webrtc.PayloadType, isSVC bool, layer int32, - srFirst *buffer.RTCPSenderReportData, - srNewest *buffer.RTCPSenderReportData, + publisherSRData *buffer.RTCPSenderReportData, ) error { - if layer == d.forwarder.GetReferenceLayerSpatial() || (layer == 0 && isSVC) { - d.handleRTCPSenderReportData(srFirst, srNewest) + currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() + if layer == currentLayer || (layer == 0 && isSVC) { + d.handleRTCPSenderReportData(publisherSRData, tsOffset) } return nil } -func (d *DownTrack) handleRTCPSenderReportData(srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { - if srNewest == nil { - return - } - - d.rtpStats.MaybeAdjustFirstPacketTime( - srFirst, - srNewest, - srNewest.RTPTimestamp+uint32(d.forwarder.GetReferenceTimestampOffset()), - ) +func (d *DownTrack) handleRTCPSenderReportData(publisherSRData *buffer.RTCPSenderReportData, tsOffset uint64) { + d.rtpStats.MaybeAdjustFirstPacketTime(publisherSRData, tsOffset) } type sendPacketMetadata struct { @@ -2018,8 +2006,9 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa } if spmd.tp.isResuming { - // adjust first packet time on a resumption so that sender reports can lock in quicker - d.handleRTCPSenderReportData(d.params.Receiver.GetRTCPSenderReportData(d.forwarder.GetReferenceLayerSpatial())) + // adjust first packet time on a resumption so that subsequent switches get a more accurate expected time stamp + currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() + d.handleRTCPSenderReportData(d.params.Receiver.GetRTCPSenderReportData(currentLayer), tsOffset) if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnResume(d) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 3e684e39d..e2bc0f1c9 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -556,18 +556,15 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { return layer } -func (f *Forwarder) GetReferenceLayerSpatial() int32 { +func (f *Forwarder) GetCurrentSpatialAndTSOffset() (int32, uint64) { f.lock.RLock() defer f.lock.RUnlock() - return f.referenceLayerSpatial -} + if f.kind == webrtc.RTPCodecTypeAudio { + return 0, f.rtpMunger.GetTSOffset() + } -func (f *Forwarder) GetReferenceTimestampOffset() uint64 { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.refTSOffset + return f.vls.GetCurrent().Spatial, f.rtpMunger.GetTSOffset() } func (f *Forwarder) isDeficientLocked() bool { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index eec709dba..3028e98e3 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -84,7 +84,7 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) - GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) + GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData GetTrackStats() *livekit.RTPStats } @@ -349,11 +349,11 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff }) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func() { - srFirst, srNewest := buff.GetSenderReportData() - w.streamTrackerManager.SetRTCPSenderReportData(layer, srFirst, srNewest) + srData := buff.GetSenderReportData() + w.streamTrackerManager.SetRTCPSenderReportData(layer, srData) w.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srFirst, srNewest) + _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData) }) }) @@ -786,7 +786,7 @@ func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, r return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } -func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { +func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { return w.streamTrackerManager.GetRTCPSenderReportData(layer) } diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 9ff94ea1a..49c918037 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -123,6 +123,10 @@ func (r *RTPMunger) GetLast() RTPMungerState { } } +func (r *RTPMunger) GetTSOffset() uint64 { + return r.tsOffset +} + func (r *RTPMunger) SeedLast(state RTPMungerState) { r.extLastSN = state.ExtLastSN r.extSecondLastSN = state.ExtSecondLastSN diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 2b80e7c25..b5f36dc97 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -50,12 +50,6 @@ type StreamTrackerManagerListener interface { // --------------------------------------------------- -type endsSenderReport struct { - first *buffer.RTCPSenderReportData - newest *buffer.RTCPSenderReportData - lastUpdated time.Time -} - type StreamTrackerManager struct { logger logger.Logger trackInfo atomic.Pointer[livekit.TrackInfo] @@ -76,7 +70,7 @@ type StreamTrackerManager struct { paused bool senderReportMu sync.RWMutex - senderReports [buffer.DefaultMaxLayerSpatial + 1]endsSenderReport + senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData layerOffsets [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerSpatial + 1]uint32 closed core.Fuse @@ -557,8 +551,8 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { - srRef := s.senderReports[ref].newest - srOther := s.senderReports[other].newest + srRef := s.senderReports[ref] + srOther := s.senderReports[other] if srRef == nil || srRef.NTPTimestamp == 0 || srOther == nil || srOther.NTPTimestamp == 0 { return } @@ -598,7 +592,7 @@ func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { s.layerOffsets[ref][other] = offset } -func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { +func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srData *buffer.RTCPSenderReportData) { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() @@ -606,9 +600,7 @@ func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buf return } - s.senderReports[layer].first = srFirst - s.senderReports[layer].newest = srNewest - s.senderReports[layer].lastUpdated = time.Now() + s.senderReports[layer] = srData // (re)fill offsets as necessary for received layer. for i := int32(0); i < buffer.DefaultMaxLayerSpatial+1; i++ { @@ -624,12 +616,12 @@ func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buf } } -func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { +func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() if layer < 0 || int(layer) >= len(s.senderReports) { - return nil, nil + return nil } // SVC-TODO: better SVC detection @@ -638,7 +630,7 @@ func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTC layer = 0 } - return s.senderReports[layer].first, s.senderReports[layer].newest + return s.senderReports[layer] } func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { From 4e96ad2e5bcc2f992f15421833de1d0f53eb6227 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 13 Mar 2024 23:01:19 +0530 Subject: [PATCH 62/73] Missed clean up in the last PR (#2576) * Missed clean up in the last PR * Infow -> Debugw --- pkg/sfu/buffer/rtpstats_receiver.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index c54dd53c8..c49fe69d0 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -385,7 +385,6 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("initializing propagation delay", getPropagationFields()...) } else { deltaPropagationDelay = propagationDelay - r.propagationDelay - r.logger.Debugw("RAJA pd", getPropagationFields()...) // REMOVE if r.smoothedDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.smoothedDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE r.propagationDelayDeltaHighCount++ @@ -394,7 +393,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) } if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { - r.logger.Infow("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay)...) + r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) initPropagationDelay(propagationDelay) } } else { From e376625a1365308ef823b7dbbb160dd3248352e0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 14 Mar 2024 04:36:19 +0530 Subject: [PATCH 63/73] Do not need to flush stream allocator events. (#2577) --- pkg/rtc/dynacastmanager.go | 8 +++++--- pkg/sfu/streamallocator/streamallocator.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 0fed210ce..20b35cb81 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -91,7 +91,7 @@ func (d *DynacastManager) Restart() { } func (d *DynacastManager) Close() { - d.qualityNotifyOpQueue.Stop() + <-d.qualityNotifyOpQueue.Stop() d.lock.Lock() dqs := d.getDynacastQualitiesLocked() @@ -305,9 +305,11 @@ func (d *DynacastManager) enqueueSubscribedQualityChange() { } } - d.params.Logger.Debugw("subscribedMaxQualityChange", + d.params.Logger.Debugw( + "subscribedMaxQualityChange", "subscribedCodecs", subscribedCodecs, - "maxSubscribedQualities", maxSubscribedQualities) + "maxSubscribedQualities", maxSubscribedQualities, + ) d.qualityNotifyOpQueue.Enqueue(func() { d.onSubscribedMaxQualityChange(subscribedCodecs, maxSubscribedQualities) }) diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 6370692b0..beda5f10a 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -180,7 +180,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { }), rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventsQueue: utils.NewOpsQueue("stream-allocator", 64, true), + eventsQueue: utils.NewOpsQueue("stream-allocator", 64, false), } s.probeController = NewProbeController(ProbeControllerParams{ From 14321f21bf2deeaf3d67ea7ac5f5e5f898cb2712 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 14 Mar 2024 10:27:24 +0530 Subject: [PATCH 64/73] Make OpsQueueParams to make it easier to understand args. (#2578) --- pkg/rtc/dynacastmanager.go | 7 ++++++- pkg/rtc/participant.go | 8 ++++++-- pkg/rtc/transport.go | 13 ++++++++---- pkg/sfu/streamallocator/streamallocator.go | 6 +++++- pkg/telemetry/telemetryservice.go | 11 +++++++--- pkg/utils/opsqueue.go | 24 ++++++++++++++-------- 6 files changed, 49 insertions(+), 20 deletions(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 20b35cb81..1de8d4d2a 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -59,7 +59,12 @@ func NewDynacastManager(params DynacastManagerParams) *DynacastManager { dynacastQuality: make(map[string]*DynacastQuality), maxSubscribedQuality: make(map[string]livekit.VideoQuality), committedMaxSubscribedQuality: make(map[string]livekit.VideoQuality), - qualityNotifyOpQueue: utils.NewOpsQueue("quality-notify", 64, true), + qualityNotifyOpQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + Name: "quality-notify", + MinSize: 64, + FlushOnStop: true, + Logger: params.Logger, + }), } if params.DynacastPauseDelay > 0 { d.maxSubscribedQualityDebounce = debounce.New(params.DynacastPauseDelay) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 1146495de..3c45151aa 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -241,8 +241,12 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { return nil, ErrMissingGrants } p := &ParticipantImpl{ - params: params, - pubRTCPQueue: sutils.NewOpsQueue("pub-rtcp", 64, false), + params: params, + pubRTCPQueue: sutils.NewOpsQueue(sutils.OpsQueueParams{ + Name: "pub-rtcp", + MinSize: 64, + Logger: params.Logger, + }), pendingTracks: make(map[string]*pendingTrackInfo), pendingPublishingTracks: make(map[livekit.TrackID]*pendingTrackInfo), connectedAt: time.Now(), diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 09e124bae..29558ae47 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -44,6 +44,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/streamallocator" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/livekit-server/pkg/utils" sutils "github.com/livekit/livekit-server/pkg/utils" lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" @@ -384,10 +385,14 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params.Logger = logger.GetLogger() } t := &PCTransport{ - params: params, - debouncedNegotiate: debounce.New(negotiationFrequency), - negotiationState: transport.NegotiationStateNone, - eventsQueue: sutils.NewOpsQueue("transport", 64, false), + params: params, + debouncedNegotiate: debounce.New(negotiationFrequency), + negotiationState: transport.NegotiationStateNone, + eventsQueue: sutils.NewOpsQueue(utils.OpsQueueParams{ + Name: "transport", + MinSize: 64, + Logger: params.Logger, + }), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, connectionDetails: types.NewICEConnectionDetails(params.Transport, params.Logger), diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index beda5f10a..352b50c2e 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -180,7 +180,11 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { }), rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventsQueue: utils.NewOpsQueue("stream-allocator", 64, false), + eventsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + Name: "stream-allocator", + MinSize: 64, + Logger: params.Logger, + }), } s.probeController = NewProbeController(ProbeControllerParams{ diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index 7bf544167..9695476ab 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -104,9 +104,14 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer t := &telemetryService{ AnalyticsService: analytics, - notifier: notifier, - jobsQueue: utils.NewOpsQueue("telemetry", jobsQueueMinSize, true), - workers: make(map[livekit.ParticipantID]*StatsWorker), + notifier: notifier, + jobsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + Name: "telemetry", + MinSize: jobsQueueMinSize, + FlushOnStop: true, + Logger: logger.GetLogger(), + }), + workers: make(map[livekit.ParticipantID]*StatsWorker), } t.jobsQueue.Start() diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 19f8d7ecc..bda4ceea1 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -19,12 +19,19 @@ import ( "sync" "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" ) +type OpsQueueParams struct { + Name string + MinSize uint + FlushOnStop bool + Logger logger.Logger +} + type OpsQueue struct { - name string - flushOnStop bool + params OpsQueueParams lock sync.Mutex ops deque.Deque[func()] @@ -34,14 +41,13 @@ type OpsQueue struct { isStopped bool } -func NewOpsQueue(name string, minSize uint, flushOnStop bool) *OpsQueue { +func NewOpsQueue(params OpsQueueParams) *OpsQueue { oq := &OpsQueue{ - name: name, - flushOnStop: flushOnStop, - wake: make(chan struct{}, 1), - doneChan: make(chan struct{}), + params: params, + wake: make(chan struct{}, 1), + doneChan: make(chan struct{}), } - oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(minSize-1)), 7))) + oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(oq.params.MinSize-1)), 7))) return oq } @@ -95,7 +101,7 @@ func (oq *OpsQueue) process() { <-oq.wake for { oq.lock.Lock() - if oq.isStopped && (!oq.flushOnStop || oq.ops.Len() == 0) { + if oq.isStopped && (!oq.params.FlushOnStop || oq.ops.Len() == 0) { oq.lock.Unlock() return } From 1d920ae4889fb2cbaf22832fb37caede62953165 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Thu, 14 Mar 2024 17:23:43 +0200 Subject: [PATCH 65/73] Support SIP DTMF data messages. (#2559) --- pkg/rtc/participant.go | 50 ++++-- pkg/rtc/room.go | 84 +++------ pkg/rtc/room_test.go | 164 ++++++++++++------ pkg/rtc/transport.go | 75 ++++---- pkg/rtc/transportmanager.go | 11 +- pkg/rtc/types/interfaces.go | 4 +- .../typesfakes/fake_local_participant.go | 24 +-- pkg/service/roommanager.go | 21 ++- test/client/client.go | 11 +- 9 files changed, 252 insertions(+), 192 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3c45151aa..7b76a3768 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -211,7 +211,7 @@ type ParticipantImpl struct { onStateChange func(p types.LocalParticipant, state livekit.ParticipantInfo_State) onMigrateStateChange func(p types.LocalParticipant, migrateState types.MigrateState) onParticipantUpdate func(types.LocalParticipant) - onDataPacket func(types.LocalParticipant, *livekit.DataPacket) + onDataPacket func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) migrateState atomic.Value // types.MigrateState @@ -625,7 +625,7 @@ func (p *ParticipantImpl) OnParticipantUpdate(callback func(types.LocalParticipa p.lock.Unlock() } -func (p *ParticipantImpl) OnDataPacket(callback func(types.LocalParticipant, *livekit.DataPacket)) { +func (p *ParticipantImpl) OnDataPacket(callback func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) { p.lock.Lock() p.onDataPacket = callback p.lock.Unlock() @@ -1472,8 +1472,8 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt p.dataChannelStats.AddBytes(uint64(len(data)), false) - dp := livekit.DataPacket{} - if err := proto.Unmarshal(data, &dp); err != nil { + dp := &livekit.DataPacket{} + if err := proto.Unmarshal(data, dp); err != nil { p.pubLogger.Warnw("could not parse data packet", err) return } @@ -1481,6 +1481,12 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt // trust the channel that it came in as the source of truth dp.Kind = kind + if p.Hidden() { + dp.ParticipantIdentity = "" + } else { + dp.ParticipantIdentity = string(p.params.Identity) + } + // only forward on user payloads switch payload := dp.Value.(type) { case *livekit.DataPacket_User: @@ -1488,14 +1494,34 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt onDataPacket := p.onDataPacket p.lock.RUnlock() if onDataPacket != nil { + u := payload.User if p.Hidden() { - payload.User.ParticipantSid = "" - payload.User.ParticipantIdentity = "" + u.ParticipantSid = "" + u.ParticipantIdentity = "" } else { - payload.User.ParticipantSid = string(p.params.SID) - payload.User.ParticipantIdentity = string(p.params.Identity) + u.ParticipantSid = string(p.params.SID) + u.ParticipantIdentity = string(p.params.Identity) + } + if dp.ParticipantIdentity != "" { + u.ParticipantIdentity = dp.ParticipantIdentity + } else { + dp.ParticipantIdentity = u.ParticipantIdentity + } + if len(dp.DestinationIdentities) != 0 { + u.DestinationIdentities = dp.DestinationIdentities + } else { + dp.DestinationIdentities = u.DestinationIdentities + } + onDataPacket(p, kind, dp) + } + case *livekit.DataPacket_SipDtmf: + if p.grants.GetParticipantKind() == livekit.ParticipantInfo_SIP { + p.lock.RLock() + onDataPacket := p.onDataPacket + p.lock.RUnlock() + if onDataPacket != nil { + onDataPacket(p, kind, dp) } - onDataPacket(p, &dp) } default: p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) @@ -2474,19 +2500,19 @@ func codecsFromMediaDescription(m *sdp.MediaDescription) (out []sdp.Codec, err e return out, nil } -func (p *ParticipantImpl) SendDataPacket(dp *livekit.DataPacket, data []byte) error { +func (p *ParticipantImpl) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { if p.State() != livekit.ParticipantInfo_ACTIVE { return ErrDataChannelUnavailable } - err := p.TransportManager.SendDataPacket(dp, data) + err := p.TransportManager.SendDataPacket(kind, encoded) if err != nil { if (errors.Is(err, sctp.ErrStreamClosed) || errors.Is(err, io.ErrClosedPipe)) && p.params.ReconnectOnDataChannelError { p.params.Logger.Infow("issuing full reconnect on data channel error", "error", err) p.IssueFullReconnect(types.ParticipantCloseReasonDataChannelError) } } else { - p.dataChannelStats.AddBytes(uint64(len(data)), true) + p.dataChannelStats.AddBytes(uint64(len(encoded)), true) } return err } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 02343080f..5cfe8f5da 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "math" + "slices" "sort" "strings" "sync" @@ -820,14 +821,8 @@ func (r *Room) OnParticipantChanged(f func(participant types.LocalParticipant)) r.onParticipantChanged = f } -func (r *Room) SendDataPacket(up *livekit.UserPacket, kind livekit.DataPacket_Kind) { - dp := &livekit.DataPacket{ - Kind: kind, - Value: &livekit.DataPacket_User{ - User: up, - }, - } - r.onDataPacket(nil, dp) +func (r *Room) SendDataPacket(dp *livekit.DataPacket, kind livekit.DataPacket_Kind) { + r.onDataPacket(nil, kind, dp) } func (r *Room) SetMetadata(metadata string) <-chan struct{} { @@ -1085,8 +1080,8 @@ func (r *Room) onParticipantUpdate(p types.LocalParticipant) { } } -func (r *Room) onDataPacket(source types.LocalParticipant, dp *livekit.DataPacket) { - BroadcastDataPacketForRoom(r, source, dp, r.Logger) +func (r *Room) onDataPacket(source types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket) { + BroadcastDataPacketForRoom(r, source, kind, dp, r.Logger) } func (r *Room) subscribeToExistingTracks(p types.LocalParticipant) { @@ -1171,33 +1166,6 @@ func (r *Room) sendParticipantUpdates(updates []*participantUpdate) { } } -// for protocol 2, send all active speakers -func (r *Room) sendActiveSpeakers(speakers []*livekit.SpeakerInfo) { - dp := &livekit.DataPacket{ - Kind: livekit.DataPacket_LOSSY, - Value: &livekit.DataPacket_Speaker{ - Speaker: &livekit.ActiveSpeakerUpdate{ - Speakers: speakers, - }, - }, - } - - var dpData []byte - for _, p := range r.GetParticipants() { - if p.ProtocolVersion().HandlesDataPackets() && !p.ProtocolVersion().SupportsSpeakerChanged() { - if dpData == nil { - var err error - dpData, err = proto.Marshal(dp) - if err != nil { - r.Logger.Errorw("failed to marshal ActiveSpeaker data packet", err) - return - } - } - _ = p.SendDataPacket(dp, dpData) - } - } -} - // for protocol 3, send only changed updates func (r *Room) sendSpeakerChanges(speakers []*livekit.SpeakerInfo) { for _, p := range r.GetParticipants() { @@ -1346,7 +1314,6 @@ func (r *Room) audioUpdateWorker() { // see if an update is needed if len(changedSpeakers) > 0 { - r.sendActiveSpeakers(activeSpeakers) r.sendSpeakerChanges(changedSpeakers) } @@ -1495,18 +1462,34 @@ func (r *Room) DebugInfo() map[string]interface{} { // ------------------------------------------------------------ -func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp *livekit.DataPacket, logger logger.Logger) { +func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket, logger logger.Logger) { + dp.Kind = kind // backward compatibility dest := dp.GetUser().GetDestinationSids() - var dpData []byte - destIdentities := dp.GetUser().GetDestinationIdentities() + if u := dp.GetUser(); u != nil { + if len(dp.DestinationIdentities) == 0 { + dp.DestinationIdentities = u.DestinationIdentities + } else { + u.DestinationIdentities = dp.DestinationIdentities + } + if dp.ParticipantIdentity != "" { + u.ParticipantIdentity = dp.ParticipantIdentity + } else { + dp.ParticipantIdentity = u.ParticipantIdentity + } + } + destIdentities := dp.DestinationIdentities participants := r.GetLocalParticipants() - capacity := len(dest) + capacity := len(destIdentities) + if capacity == 0 { + capacity = len(dest) + } if capacity == 0 { capacity = len(participants) } destParticipants := make([]types.LocalParticipant, 0, capacity) + var dpData []byte for _, op := range participants { if op.State() != livekit.ParticipantInfo_ACTIVE { continue @@ -1515,20 +1498,7 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp continue } if len(dest) > 0 || len(destIdentities) > 0 { - found := false - for _, dID := range dest { - if op.ID() == livekit.ParticipantID(dID) { - found = true - break - } - } - for _, dIdentity := range destIdentities { - if op.Identity() == livekit.ParticipantIdentity(dIdentity) { - found = true - break - } - } - if !found { + if !slices.Contains(dest, string(op.ID())) && !slices.Contains(destIdentities, string(op.Identity())) { continue } } @@ -1544,7 +1514,7 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp } utils.ParallelExec(destParticipants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { - err := op.SendDataPacket(dp, dpData) + err := op.SendDataPacket(kind, dpData) if err != nil && !errors.Is(err, io.ErrClosedPipe) && !errors.Is(err, sctp.ErrStreamClosed) && !errors.Is(err, ErrTransportFailure) && !errors.Is(err, ErrDataChannelBufferFull) { op.GetLogger().Infow("send data packet error", "error", err) diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index ea09a91d0..c73b5fa5c 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -22,10 +22,11 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/livekit/livekit-server/version" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/webhook" + "github.com/livekit/livekit-server/version" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" @@ -569,65 +570,126 @@ func TestActiveSpeakers(t *testing.T) { func TestDataChannel(t *testing.T) { t.Parallel() - t.Run("participants should receive data", func(t *testing.T) { - rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) - defer rm.Close(types.ParticipantCloseReasonNone) - participants := rm.GetParticipants() - p := participants[0].(*typesfakes.FakeLocalParticipant) + const ( + curAPI = iota + legacySID + legacyIdentity + ) + modes := []int{ + curAPI, legacySID, legacyIdentity, + } + modeNames := []string{ + "cur", "legacy sid", "legacy identity", + } - packet := livekit.DataPacket{ - Kind: livekit.DataPacket_RELIABLE, - Value: &livekit.DataPacket_User{ - User: &livekit.UserPacket{ - ParticipantSid: string(p.ID()), - Payload: []byte("message.."), - }, - }, + setSource := func(mode int, dp *livekit.DataPacket, p types.LocalParticipant) { + switch mode { + case curAPI: + dp.ParticipantIdentity = string(p.Identity()) + case legacySID: + dp.GetUser().ParticipantSid = string(p.ID()) + case legacyIdentity: + dp.GetUser().ParticipantIdentity = string(p.Identity()) } - p.OnDataPacketArgsForCall(0)(p, &packet) + } + setDest := func(mode int, dp *livekit.DataPacket, p types.LocalParticipant) { + switch mode { + case curAPI: + dp.DestinationIdentities = []string{string(p.Identity())} + case legacySID: + dp.GetUser().DestinationSids = []string{string(p.ID())} + case legacyIdentity: + dp.GetUser().DestinationIdentities = []string{string(p.Identity())} + } + } - // ensure everyone has received the packet - for _, op := range participants { - fp := op.(*typesfakes.FakeLocalParticipant) - if fp == p { - require.Zero(t, fp.SendDataPacketCallCount()) - continue - } - require.Equal(t, 1, fp.SendDataPacketCallCount()) - dp, _ := fp.SendDataPacketArgsForCall(0) - require.Equal(t, packet.Value, dp.Value) + t.Run("participants should receive data", func(t *testing.T) { + for _, mode := range modes { + mode := mode + t.Run(modeNames[mode], func(t *testing.T) { + rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) + defer rm.Close(types.ParticipantCloseReasonNone) + participants := rm.GetParticipants() + p := participants[0].(*typesfakes.FakeLocalParticipant) + + packet := &livekit.DataPacket{ + Kind: livekit.DataPacket_RELIABLE, + Value: &livekit.DataPacket_User{ + User: &livekit.UserPacket{ + Payload: []byte("message.."), + }, + }, + } + setSource(mode, packet, p) + + packetExp := proto.Clone(packet).(*livekit.DataPacket) + if mode != legacySID { + packetExp.ParticipantIdentity = string(p.Identity()) + packetExp.GetUser().ParticipantIdentity = string(p.Identity()) + } + + encoded, _ := proto.Marshal(packetExp) + p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet) + + // ensure everyone has received the packet + for _, op := range participants { + fp := op.(*typesfakes.FakeLocalParticipant) + if fp == p { + require.Zero(t, fp.SendDataPacketCallCount()) + continue + } + require.Equal(t, 1, fp.SendDataPacketCallCount()) + _, got := fp.SendDataPacketArgsForCall(0) + require.Equal(t, encoded, got) + } + }) } }) t.Run("only one participant should receive the data", func(t *testing.T) { - rm := newRoomWithParticipants(t, testRoomOpts{num: 4}) - defer rm.Close(types.ParticipantCloseReasonNone) - participants := rm.GetParticipants() - p := participants[0].(*typesfakes.FakeLocalParticipant) - p1 := participants[1].(*typesfakes.FakeLocalParticipant) + for _, mode := range modes { + mode := mode + t.Run(modeNames[mode], func(t *testing.T) { + rm := newRoomWithParticipants(t, testRoomOpts{num: 4}) + defer rm.Close(types.ParticipantCloseReasonNone) + participants := rm.GetParticipants() + p := participants[0].(*typesfakes.FakeLocalParticipant) + p1 := participants[1].(*typesfakes.FakeLocalParticipant) - packet := livekit.DataPacket{ - Kind: livekit.DataPacket_RELIABLE, - Value: &livekit.DataPacket_User{ - User: &livekit.UserPacket{ - ParticipantSid: string(p.ID()), - Payload: []byte("message to p1.."), - DestinationSids: []string{string(p1.ID())}, - }, - }, - } - p.OnDataPacketArgsForCall(0)(p, &packet) + packet := &livekit.DataPacket{ + Kind: livekit.DataPacket_RELIABLE, + Value: &livekit.DataPacket_User{ + User: &livekit.UserPacket{ + Payload: []byte("message to p1.."), + }, + }, + } + setSource(mode, packet, p) + setDest(mode, packet, p1) - // only p1 should receive the data - for _, op := range participants { - fp := op.(*typesfakes.FakeLocalParticipant) - if fp != p1 { - require.Zero(t, fp.SendDataPacketCallCount()) - } + packetExp := proto.Clone(packet).(*livekit.DataPacket) + if mode != legacySID { + packetExp.ParticipantIdentity = string(p.Identity()) + packetExp.GetUser().ParticipantIdentity = string(p.Identity()) + packetExp.DestinationIdentities = []string{string(p1.Identity())} + packetExp.GetUser().DestinationIdentities = []string{string(p1.Identity())} + } + + encoded, _ := proto.Marshal(packetExp) + p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet) + + // only p1 should receive the data + for _, op := range participants { + fp := op.(*typesfakes.FakeLocalParticipant) + if fp != p1 { + require.Zero(t, fp.SendDataPacketCallCount()) + } + } + require.Equal(t, 1, p1.SendDataPacketCallCount()) + _, got := p1.SendDataPacketArgsForCall(0) + require.Equal(t, encoded, got) + }) } - require.Equal(t, 1, p1.SendDataPacketCallCount()) - dp, _ := p1.SendDataPacketArgsForCall(0) - require.Equal(t, packet.Value, dp.Value) }) t.Run("publishing disallowed", func(t *testing.T) { @@ -646,7 +708,7 @@ func TestDataChannel(t *testing.T) { }, } if p.CanPublishData() { - p.OnDataPacketArgsForCall(0)(p, &packet) + p.OnDataPacketArgsForCall(0)(p, packet.Kind, &packet) } // no one should've been sent packet diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 29558ae47..1e0cd1a93 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -35,6 +35,13 @@ import ( "github.com/pkg/errors" "go.uber.org/atomic" + lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" + lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" + lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -46,12 +53,6 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" sutils "github.com/livekit/livekit-server/pkg/utils" - lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" - lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/logger/pionlogger" - lksdp "github.com/livekit/protocol/sdp" ) const ( @@ -799,16 +800,27 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni if err != nil { return err } + var ( + dcPtr **webrtc.DataChannel + dcReady *bool + ) + switch dc.Label() { + default: + // TODO: Appears that it's never called, so not sure what needs to be done here. We just keep the DC open? + // Maybe just add "reliable" parameter instead of checking the label. + t.params.Logger.Warnw("unknown data channel label", nil, "label", dc.Label()) + return nil + case ReliableDataChannel: + dcPtr = &t.reliableDC + dcReady = &t.reliableDCOpened + case LossyDataChannel: + dcPtr = &t.lossyDC + dcReady = &t.lossyDCOpened + } dcReadyHandler := func() { t.lock.Lock() - switch dc.Label() { - case ReliableDataChannel: - t.reliableDCOpened = true - - case LossyDataChannel: - t.lossyDCOpened = true - } + *dcReady = true t.lock.Unlock() t.params.Logger.Debugw(dc.Label() + " data channel open") @@ -826,30 +838,15 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni } t.lock.Lock() - switch dc.Label() { - case ReliableDataChannel: - t.reliableDC = dc - if t.params.DirectionConfig.StrictACKs { - t.reliableDC.OnOpen(dcReadyHandler) - } else { - t.reliableDC.OnDial(dcReadyHandler) - } - t.reliableDC.OnClose(dcCloseHandler) - t.reliableDC.OnError(dcErrorHandler) - case LossyDataChannel: - t.lossyDC = dc - if t.params.DirectionConfig.StrictACKs { - t.lossyDC.OnOpen(dcReadyHandler) - } else { - t.lossyDC.OnDial(dcReadyHandler) - } - t.lossyDC.OnClose(dcCloseHandler) - t.lossyDC.OnError(dcErrorHandler) - default: - t.params.Logger.Warnw("unknown data channel label", nil, "label", dc.Label()) + defer t.lock.Unlock() + *dcPtr = dc + if t.params.DirectionConfig.StrictACKs { + dc.OnOpen(dcReadyHandler) + } else { + dc.OnDial(dcReadyHandler) } - t.lock.Unlock() - + dc.OnClose(dcCloseHandler) + dc.OnError(dcErrorHandler) return nil } @@ -903,10 +900,10 @@ func (t *PCTransport) WriteRTCP(pkts []rtcp.Packet) error { return t.pc.WriteRTCP(pkts) } -func (t *PCTransport) SendDataPacket(dp *livekit.DataPacket, data []byte) error { +func (t *PCTransport) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { var dc *webrtc.DataChannel t.lock.RLock() - if dp.Kind == livekit.DataPacket_RELIABLE { + if kind == livekit.DataPacket_RELIABLE { dc = t.reliableDC } else { dc = t.lossyDC @@ -925,7 +922,7 @@ func (t *PCTransport) SendDataPacket(dp *livekit.DataPacket, data []byte) error return ErrDataChannelBufferFull } - return dc.Send(data) + return dc.Send(encoded) } func (t *PCTransport) Close() { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index cd1b23289..049b76a5e 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -28,14 +28,15 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/mediatransportutil/pkg/twcc" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/mediatransportutil/pkg/twcc" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" ) const ( @@ -240,9 +241,9 @@ func (t *TransportManager) RemoveSubscribedTrack(subTrack types.SubscribedTrack) t.subscriber.RemoveTrackFromStreamAllocator(subTrack) } -func (t *TransportManager) SendDataPacket(dp *livekit.DataPacket, data []byte) error { +func (t *TransportManager) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { // downstream data is sent via primary peer connection - return t.getTransport(true).SendDataPacket(dp, data) + return t.getTransport(true).SendDataPacket(kind, encoded) } func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels []*livekit.DataChannelInfo) error { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index e4e6b61d3..b0943d4f4 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -365,7 +365,7 @@ type LocalParticipant interface { SendJoinResponse(joinResponse *livekit.JoinResponse) error SendParticipantUpdate(participants []*livekit.ParticipantInfo) error SendSpeakerUpdate(speakers []*livekit.SpeakerInfo, force bool) error - SendDataPacket(packet *livekit.DataPacket, data []byte) error + SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error SendRoomUpdate(room *livekit.Room) error SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) @@ -384,7 +384,7 @@ type LocalParticipant interface { OnTrackUnpublished(callback func(LocalParticipant, MediaTrack)) // OnParticipantUpdate - metadata or permission is updated OnParticipantUpdate(callback func(LocalParticipant)) - OnDataPacket(callback func(LocalParticipant, *livekit.DataPacket)) + OnDataPacket(callback func(LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) OnClose(callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 464f4009c..37c03ef10 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -571,10 +571,10 @@ type FakeLocalParticipant struct { onCloseArgsForCall []struct { arg1 func(types.LocalParticipant) } - OnDataPacketStub func(func(types.LocalParticipant, *livekit.DataPacket)) + OnDataPacketStub func(func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) onDataPacketMutex sync.RWMutex onDataPacketArgsForCall []struct { - arg1 func(types.LocalParticipant, *livekit.DataPacket) + arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) } OnICEConfigChangedStub func(func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) onICEConfigChangedMutex sync.RWMutex @@ -660,10 +660,10 @@ type FakeLocalParticipant struct { sendConnectionQualityUpdateReturnsOnCall map[int]struct { result1 error } - SendDataPacketStub func(*livekit.DataPacket, []byte) error + SendDataPacketStub func(livekit.DataPacket_Kind, []byte) error sendDataPacketMutex sync.RWMutex sendDataPacketArgsForCall []struct { - arg1 *livekit.DataPacket + arg1 livekit.DataPacket_Kind arg2 []byte } sendDataPacketReturns struct { @@ -3932,10 +3932,10 @@ func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalPart return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnDataPacket(arg1 func(types.LocalParticipant, *livekit.DataPacket)) { +func (fake *FakeLocalParticipant) OnDataPacket(arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) { fake.onDataPacketMutex.Lock() fake.onDataPacketArgsForCall = append(fake.onDataPacketArgsForCall, struct { - arg1 func(types.LocalParticipant, *livekit.DataPacket) + arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) }{arg1}) stub := fake.OnDataPacketStub fake.recordInvocation("OnDataPacket", []interface{}{arg1}) @@ -3951,13 +3951,13 @@ func (fake *FakeLocalParticipant) OnDataPacketCallCount() int { return len(fake.onDataPacketArgsForCall) } -func (fake *FakeLocalParticipant) OnDataPacketCalls(stub func(func(types.LocalParticipant, *livekit.DataPacket))) { +func (fake *FakeLocalParticipant) OnDataPacketCalls(stub func(func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket))) { fake.onDataPacketMutex.Lock() defer fake.onDataPacketMutex.Unlock() fake.OnDataPacketStub = stub } -func (fake *FakeLocalParticipant) OnDataPacketArgsForCall(i int) func(types.LocalParticipant, *livekit.DataPacket) { +func (fake *FakeLocalParticipant) OnDataPacketArgsForCall(i int) func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) { fake.onDataPacketMutex.RLock() defer fake.onDataPacketMutex.RUnlock() argsForCall := fake.onDataPacketArgsForCall[i] @@ -4461,7 +4461,7 @@ func (fake *FakeLocalParticipant) SendConnectionQualityUpdateReturnsOnCall(i int }{result1} } -func (fake *FakeLocalParticipant) SendDataPacket(arg1 *livekit.DataPacket, arg2 []byte) error { +func (fake *FakeLocalParticipant) SendDataPacket(arg1 livekit.DataPacket_Kind, arg2 []byte) error { var arg2Copy []byte if arg2 != nil { arg2Copy = make([]byte, len(arg2)) @@ -4470,7 +4470,7 @@ func (fake *FakeLocalParticipant) SendDataPacket(arg1 *livekit.DataPacket, arg2 fake.sendDataPacketMutex.Lock() ret, specificReturn := fake.sendDataPacketReturnsOnCall[len(fake.sendDataPacketArgsForCall)] fake.sendDataPacketArgsForCall = append(fake.sendDataPacketArgsForCall, struct { - arg1 *livekit.DataPacket + arg1 livekit.DataPacket_Kind arg2 []byte }{arg1, arg2Copy}) stub := fake.SendDataPacketStub @@ -4492,13 +4492,13 @@ func (fake *FakeLocalParticipant) SendDataPacketCallCount() int { return len(fake.sendDataPacketArgsForCall) } -func (fake *FakeLocalParticipant) SendDataPacketCalls(stub func(*livekit.DataPacket, []byte) error) { +func (fake *FakeLocalParticipant) SendDataPacketCalls(stub func(livekit.DataPacket_Kind, []byte) error) { fake.sendDataPacketMutex.Lock() defer fake.sendDataPacketMutex.Unlock() fake.SendDataPacketStub = stub } -func (fake *FakeLocalParticipant) SendDataPacketArgsForCall(i int) (*livekit.DataPacket, []byte) { +func (fake *FakeLocalParticipant) SendDataPacketArgsForCall(i int) (livekit.DataPacket_Kind, []byte) { fake.sendDataPacketMutex.RLock() defer fake.sendDataPacketMutex.RUnlock() argsForCall := fake.sendDataPacketArgsForCall[i] diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 8c5aa60d5..96dd7036c 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -24,8 +24,6 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/maps" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" - "github.com/livekit/livekit-server/version" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -41,6 +39,8 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/livekit-server/version" ) const ( @@ -753,13 +753,18 @@ func (r *RoomManager) SendData(ctx context.Context, req *livekit.SendDataRequest } room.Logger.Debugw("api send data", "size", len(req.Data)) - up := &livekit.UserPacket{ - Payload: req.Data, - DestinationSids: req.DestinationSids, + room.SendDataPacket(&livekit.DataPacket{ + Kind: req.Kind, DestinationIdentities: req.DestinationIdentities, - Topic: req.Topic, - } - room.SendDataPacket(up, req.Kind) + Value: &livekit.DataPacket_User{ + User: &livekit.UserPacket{ + Payload: req.Data, + DestinationSids: req.DestinationSids, + DestinationIdentities: req.DestinationIdentities, + Topic: req.Topic, + }, + }, + }, req.Kind) return &livekit.SendDataResponse{}, nil } diff --git a/test/client/client.go b/test/client/client.go index e0be22e47..5d461a74a 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -679,19 +679,17 @@ func (c *RTCClient) PublishData(data []byte, kind livekit.DataPacket_Kind) error return err } - dp := &livekit.DataPacket{ + dpData, err := proto.Marshal(&livekit.DataPacket{ Kind: kind, Value: &livekit.DataPacket_User{ User: &livekit.UserPacket{Payload: data}, }, - } - - dpData, err := proto.Marshal(dp) + }) if err != nil { return err } - return c.publisher.SendDataPacket(dp, dpData) + return c.publisher.SendDataPacket(kind, dpData) } func (c *RTCClient) GetPublishedTrackIDs() []string { @@ -732,12 +730,13 @@ func (c *RTCClient) ensurePublisherConnected() error { } } -func (c *RTCClient) handleDataMessage(_ livekit.DataPacket_Kind, data []byte) { +func (c *RTCClient) handleDataMessage(kind livekit.DataPacket_Kind, data []byte) { dp := &livekit.DataPacket{} err := proto.Unmarshal(data, dp) if err != nil { return } + dp.Kind = kind if val, ok := dp.Value.(*livekit.DataPacket_User); ok { if c.OnDataReceived != nil { c.OnDataReceived(val.User.Payload, val.User.ParticipantSid) From 9e5f434cefd35aa055e37ea279c7d60a9fc72939 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 15 Mar 2024 10:59:37 +0530 Subject: [PATCH 66/73] Do not block as notification could make a network request. (#2581) --- pkg/rtc/dynacastmanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 1de8d4d2a..ca6ea7af3 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -96,7 +96,7 @@ func (d *DynacastManager) Restart() { } func (d *DynacastManager) Close() { - <-d.qualityNotifyOpQueue.Stop() + d.qualityNotifyOpQueue.Stop() d.lock.Lock() dqs := d.getDynacastQualitiesLocked() From e85922857a1f25fbb9456408797fee03bef130a9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 18 Mar 2024 14:55:05 +0530 Subject: [PATCH 67/73] Use cloned grants. (#2586) To ensure no pointer (i. e. mutable object) is held in returned proto. --- pkg/rtc/participant.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 7b76a3768..6617e056a 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -503,19 +503,20 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. p.timedVersion.Update(&piv) } + grants := p.ClaimGrants() p.lock.RLock() pi := &livekit.ParticipantInfo{ Sid: string(p.params.SID), Identity: string(p.params.Identity), - Name: p.grants.Name, + Name: grants.Name, State: p.State(), JoinedAt: p.ConnectedAt().Unix(), Version: v, - Permission: p.grants.Video.ToPermission(), - Metadata: p.grants.Metadata, + Permission: grants.Video.ToPermission(), + Metadata: grants.Metadata, Region: p.params.Region, IsPublisher: p.IsPublisher(), - Kind: p.grants.GetParticipantKind(), + Kind: grants.GetParticipantKind(), } p.lock.RUnlock() From bc1711c47631419ed29ec2a896ce3b7baf803874 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:00:28 -0700 Subject: [PATCH 68/73] Update golang.org/x/exp digest to a85f2c6 (#2551) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 63a5c4fcb..c2fe18108 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 @@ -99,10 +99,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.18.0 // indirect + golang.org/x/tools v0.19.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/grpc v1.62.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 8b2635d44..7ed6a89eb 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -346,8 +346,8 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -432,8 +432,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From c30c1be689e856dba90404dad42ca2a7c1eae1e6 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Mon, 18 Mar 2024 20:48:26 +0200 Subject: [PATCH 69/73] Push Docker images to the master tag. (#2579) --- .github/workflows/docker.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 9b52542d7..38258deb8 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -18,9 +18,10 @@ name: Release to Docker on: workflow_dispatch: push: - # only publish on version tags + branches: + - master # publish to 'master' tag tags: - - 'v*.*.*' + - 'v*.*.*' # publish on version tags, updates 'latest' tag jobs: docker: runs-on: ubuntu-latest @@ -35,6 +36,7 @@ jobs: livekit/livekit-server # generate Docker tags based on the following events/attributes tags: | + type=ref,event=branch type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}} @@ -56,6 +58,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -66,7 +69,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: true + push: ${{ github.event_name != 'pull_request' }} platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 03ada9ba76463c5a0d1f078b692170f6207f341f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 19 Mar 2024 11:59:24 +0530 Subject: [PATCH 70/73] Proper RTCP report past mute. (#2588) - When audio is muted, server injects silence frames which moves the time stamp forward and adjusts offset. That cannot be used against publisher side sender report. Use a pinned version. - Ignore small changes to propagation delay even while checking for sharp increase. That is spamming a lot for small changes, i.e. existing delta is 100 micro seconds or so and the new one is 300 micro seconds. Also rename to `longTerm` from `smoothed` as it is a slow varying long term estimate of propagation delay delta. And slow down that adaptation more. --- pkg/sfu/buffer/rtpstats_receiver.go | 74 +++++++++++++++-------------- pkg/sfu/forwarder.go | 4 +- pkg/sfu/rtpmunger.go | 9 +++- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index c49fe69d0..cbcb26794 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -48,10 +48,10 @@ const ( // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. // Reset at whichever of the below happens later. // - // A smoothed version of delta of propagation delay is maintained and delta propagation delay exceeding - // a factor of the smoothed version is considered a sharp increase. That will trigger the start of the + // A long term version of delta of propagation delay is maintained and delta propagation delay exceeding + // a factor of the long term version is considered a sharp increase. That will trigger the start of the // path change condition and if it persists, propagation delay will be reset. - cPropagationDelayDeltaAdaptationFactor = float64(0.1) + cPropagationDelayDeltaAdaptationFactor = float64(0.05) cPropagationDelayDeltaHighResetNumReports = 3 cPropagationDelayDeltaHighResetWait = 10 * time.Second ) @@ -80,7 +80,7 @@ type RTPStatsReceiver struct { history *protoutils.Bitmap[uint64] propagationDelay time.Duration - smoothedDeltaPropagationDelay time.Duration + longTermDeltaPropagationDelay time.Duration propagationDelayDeltaHighCount int propagationDelayDeltaHighStartTime time.Time @@ -361,7 +361,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) return []interface{}{ "propagationDelay", r.propagationDelay.String(), "receivedPropagationDelay", propagationDelay.String(), - "smoothedDeltaPropagationDelay", r.smoothedDeltaPropagationDelay.String(), + "longTermDeltaPropagationDelay", r.longTermDeltaPropagationDelay.String(), "receivedDeltaPropagationDelay", deltaPropagationDelay.String(), "deltaHighCount", r.propagationDelayDeltaHighCount, "sinceDeltaHighStart", time.Since(r.propagationDelayDeltaHighStartTime).String(), @@ -372,7 +372,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) } initPropagationDelay := func(pd time.Duration) { r.propagationDelay = pd - r.smoothedDeltaPropagationDelay = 0 + r.longTermDeltaPropagationDelay = 0 r.propagationDelayDeltaHighCount = 0 r.propagationDelayDeltaHighStartTime = time.Time{} } @@ -385,39 +385,41 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("initializing propagation delay", getPropagationFields()...) } else { deltaPropagationDelay = propagationDelay - r.propagationDelay - if r.smoothedDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.smoothedDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { - r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE - r.propagationDelayDeltaHighCount++ - if r.propagationDelayDeltaHighStartTime.IsZero() { - r.propagationDelayDeltaHighStartTime = time.Now() - } - - if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { - r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) - initPropagationDelay(propagationDelay) - } - } else { - if r.smoothedDeltaPropagationDelay == 0 { - r.smoothedDeltaPropagationDelay = deltaPropagationDelay - } else { - r.smoothedDeltaPropagationDelay += time.Duration(cPropagationDelayDeltaAdaptationFactor * float64(deltaPropagationDelay-r.smoothedDeltaPropagationDelay)) - } - r.propagationDelayDeltaHighCount = 0 - r.propagationDelayDeltaHighStartTime = time.Time{} - - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { - factor := cPropagationDelayFallFactor - if propagationDelay > r.propagationDelay { - factor = cPropagationDelayRiseFactor + if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { // ignore small changes + if r.longTermDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { + r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE + r.propagationDelayDeltaHighCount++ + if r.propagationDelayDeltaHighStartTime.IsZero() { + r.propagationDelayDeltaHighStartTime = time.Now() + } + + if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { + r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) + initPropagationDelay(propagationDelay) + } + } else { + r.propagationDelayDeltaHighCount = 0 + r.propagationDelayDeltaHighStartTime = time.Time{} + + if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { + factor := cPropagationDelayFallFactor + if propagationDelay > r.propagationDelay { + factor = cPropagationDelayRiseFactor + } + fields := append( + getPropagationFields(), + "adjustedPropagationDelay", r.propagationDelay+time.Duration(factor*float64(propagationDelay-r.propagationDelay)), + ) // TODO-REMOVE + r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE + r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } - fields := append( - getPropagationFields(), - "adjustedPropagationDelay", r.propagationDelay+time.Duration(factor*float64(propagationDelay-r.propagationDelay)), - ) // TODO-REMOVE - r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE - r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } } + if r.longTermDeltaPropagationDelay == 0 { + r.longTermDeltaPropagationDelay = deltaPropagationDelay + } else { + r.longTermDeltaPropagationDelay += time.Duration(cPropagationDelayDeltaAdaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + } } // adjust receive time to estimated propagation delay srDataCopy.At = ntpTime.Add(r.propagationDelay) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index e2bc0f1c9..de50a829b 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -561,10 +561,10 @@ func (f *Forwarder) GetCurrentSpatialAndTSOffset() (int32, uint64) { defer f.lock.RUnlock() if f.kind == webrtc.RTPCodecTypeAudio { - return 0, f.rtpMunger.GetTSOffset() + return 0, f.rtpMunger.GetPinnedTSOffset() } - return f.vls.GetCurrent().Spatial, f.rtpMunger.GetTSOffset() + return f.vls.GetCurrent().Spatial, f.rtpMunger.GetPinnedTSOffset() } func (f *Forwarder) isDeficientLocked() bool { diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 49c918037..47287e84a 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -83,6 +83,7 @@ type RTPMunger struct { extLastTS uint64 extSecondLastTS uint64 tsOffset uint64 + pinnedTSOffset uint64 lastMarker bool secondLastMarker bool @@ -107,6 +108,7 @@ func (r *RTPMunger) DebugInfo() map[string]interface{} { "ExtLastTS": r.extLastTS, "ExtSecondLastTS": r.extSecondLastTS, "TSOffset": r.tsOffset, + "PinnedTSOffset": r.pinnedTSOffset, "LastMarker": r.lastMarker, "SecondLastMarker": r.secondLastMarker, } @@ -123,8 +125,8 @@ func (r *RTPMunger) GetLast() RTPMungerState { } } -func (r *RTPMunger) GetTSOffset() uint64 { - return r.tsOffset +func (r *RTPMunger) GetPinnedTSOffset() uint64 { + return r.pinnedTSOffset } func (r *RTPMunger) SeedLast(state RTPMungerState) { @@ -146,6 +148,8 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.extLastTS = extPkt.ExtTimestamp r.extSecondLastTS = extPkt.ExtTimestamp + r.tsOffset = 0 + r.pinnedTSOffset = r.tsOffset } func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) { @@ -155,6 +159,7 @@ func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, r.updateSnOffset() r.tsOffset = extPkt.ExtTimestamp - r.extLastTS - tsAdjust + r.pinnedTSOffset = r.tsOffset } func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { From 8564329579734cd243f32286d76dc58cefc7fc7f Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Wed, 20 Mar 2024 18:59:25 +0200 Subject: [PATCH 71/73] Pass DTMF when creating SIP participants. (#2590) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/sip.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c2fe18108..135eb5bcf 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626 + github.com/livekit/protocol v1.12.0 github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 7ed6a89eb..2c88567ea 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626 h1:FMN/mD/7e06gDi51sT0BdaTGIWIcdK140YTQNJI4HrY= -github.com/livekit/protocol v1.11.1-0.20240313083005-cf54792d0626/go.mod h1:znZpBU024XvKIpC9jkFzfrJMDuucDb4B1ON9lbUJMGA= +github.com/livekit/protocol v1.12.0 h1:B7qsqq5xf9MmyG9WEk9/gMsfMVXuyLNxX5cO6TQil6s= +github.com/livekit/protocol v1.12.0/go.mod h1:G7Pa985GhZv2MCC3UnUocBhZfi3DsWA6WmlSkkpQYTM= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/sip.go b/pkg/service/sip.go index dba99ecc4..c3989e854 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -196,6 +196,7 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv CallTo: req.SipCallTo, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, + Dtmf: req.Dtmf, WsUrl: wsUrl, Token: token, } From 7945c01dbecb7315c303f3122e20d6e09fec3d27 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 21 Mar 2024 10:25:45 +0530 Subject: [PATCH 72/73] Reset sharp increase if received delta is small. (#2592) --- pkg/sfu/buffer/rtpstats_receiver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index cbcb26794..0a1b61a28 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -414,6 +414,9 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } } + } else { + r.propagationDelayDeltaHighCount = 0 + r.propagationDelayDeltaHighStartTime = time.Time{} } if r.longTermDeltaPropagationDelay == 0 { r.longTermDeltaPropagationDelay = deltaPropagationDelay From 0f597e7e469b978f3ee3ce3d495a8d043ee97fe7 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 21 Mar 2024 02:48:26 -0700 Subject: [PATCH 73/73] update protocol (#2593) * update protocol * cleanup * deps --- go.mod | 2 +- go.sum | 4 ++-- pkg/rtc/participant.go | 22 +++++++++++++--------- pkg/rtc/room.go | 2 +- pkg/rtc/subscribedtrack.go | 6 +++--- pkg/rtc/testutils.go | 2 +- pkg/rtc/uptrackmanager.go | 10 +++++----- pkg/rtc/uptrackmanager_test.go | 4 ++-- 8 files changed, 28 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 135eb5bcf..b15f7bf03 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.12.0 + github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 2c88567ea..670d00671 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.0 h1:B7qsqq5xf9MmyG9WEk9/gMsfMVXuyLNxX5cO6TQil6s= -github.com/livekit/protocol v1.12.0/go.mod h1:G7Pa985GhZv2MCC3UnUocBhZfi3DsWA6WmlSkkpQYTM= +github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e h1:XR7vPLN7c/R6R87UARoBW2csVKd7RuTXwG+XsjczbT0= +github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e/go.mod h1:G7Pa985GhZv2MCC3UnUocBhZfi3DsWA6WmlSkkpQYTM= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 6617e056a..5f8f7e60e 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -265,7 +265,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { } p.closeReason.Store(types.ParticipantCloseReasonNone) p.version.Store(params.InitialVersion) - p.timedVersion.Update(params.VersionGenerator.New()) + p.timedVersion.Update(params.VersionGenerator.Next()) p.migrateState.Store(types.MigrateStateInit) p.state.Store(livekit.ParticipantInfo_JOINING) p.grants = params.Grants @@ -495,16 +495,20 @@ func (p *ParticipantImpl) CanSkipBroadcast() bool { } func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { - v := p.version.Load() - piv := p.timedVersion.Load() - if p.dirty.Swap(false) { - v = p.version.Inc() - piv = p.params.VersionGenerator.Next() - p.timedVersion.Update(&piv) + if p.dirty.Load() { + p.lock.Lock() + if p.dirty.Swap(false) { + p.version.Inc() + p.timedVersion.Update(p.params.VersionGenerator.Next()) + } + p.lock.Unlock() } grants := p.ClaimGrants() p.lock.RLock() + v := p.version.Load() + piv := p.timedVersion + pi := &livekit.ParticipantInfo{ Sid: string(p.params.SID), Identity: string(p.params.Identity), @@ -1988,9 +1992,9 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } ti.MimeType = track.Codec().MimeType - if utils.NewTimedVersionFromProto(ti.Version).IsZero() { + if utils.TimedVersionFromProto(ti.Version).IsZero() { // only assign version on a fresh publish, i. e. avoid updating version in scenarios like migration - ti.Version = p.params.VersionGenerator.New().ToProto() + ti.Version = p.params.VersionGenerator.Next().ToProto() } mt = p.addMediaTrack(signalCid, track.ID(), ti) newTrack = true diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 5cfe8f5da..e50e84b83 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -710,7 +710,7 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync } func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { - if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion{}, r.GetParticipantByID); err != nil { + if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion(0), r.GetParticipantByID); err != nil { return err } for _, track := range participant.GetPublishedTracks() { diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 3cdf12d4e..eff9b8358 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -56,7 +56,7 @@ type SubscribedTrack struct { versionGenerator utils.TimedVersionGenerator settingsLock sync.Mutex settings *livekit.UpdateTrackSettings - settingsVersion *utils.TimedVersion + settingsVersion utils.TimedVersion bindLock sync.Mutex bound bool @@ -243,7 +243,7 @@ func (t *SubscribedTrack) applySettings() { } t.logger.Debugw("updating subscriber track settings", "settings", logger.Proto(t.settings)) - t.settingsVersion = t.versionGenerator.New() + t.settingsVersion = t.versionGenerator.Next() settingsVersion := t.settingsVersion t.settingsLock.Unlock() @@ -264,7 +264,7 @@ func (t *SubscribedTrack) applySettings() { } t.settingsLock.Lock() - if settingsVersion.Compare(t.settingsVersion) != 0 { + if settingsVersion != t.settingsVersion { // a newer settings has superceded this one t.settingsLock.Unlock() return diff --git a/pkg/rtc/testutils.go b/pkg/rtc/testutils.go index b63c372a8..e233fa5f3 100644 --- a/pkg/rtc/testutils.go +++ b/pkg/rtc/testutils.go @@ -45,7 +45,7 @@ func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro Identity: string(identity), State: livekit.ParticipantInfo_JOINED, IsPublisher: publisher, - }, utils.TimedVersion{}) + }, utils.TimedVersion(0)) p.SetMetadataCalls(func(m string) { var f func(participant types.LocalParticipant) diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 904b3ea66..57cf372ac 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -166,7 +166,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( // we do not want to initialize subscriptionPermissionVersion too early since if another machine is the // owner for the data, we'd prefer to use their TimedVersion // ignore older version - if !timedVersion.After(&u.subscriptionPermissionVersion) { + if !timedVersion.After(u.subscriptionPermissionVersion) { u.params.Logger.Debugw( "skipping older subscription permission version", "existingValue", logger.Proto(u.subscriptionPermission), @@ -177,10 +177,10 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( u.lock.Unlock() return nil } - u.subscriptionPermissionVersion.Update(&timedVersion) + u.subscriptionPermissionVersion.Update(timedVersion) } else { // for requests coming from the current node, use local versions - u.subscriptionPermissionVersion.Update(u.params.VersionGenerator.New()) + u.subscriptionPermissionVersion.Update(u.params.VersionGenerator.Next()) } // store as is for use when migrating @@ -188,7 +188,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( if subscriptionPermission == nil { u.params.Logger.Debugw( "updating subscription permission, setting to nil", - "version", &u.subscriptionPermissionVersion, + "version", u.subscriptionPermissionVersion, ) // possible to get a nil when migrating u.lock.Unlock() @@ -198,7 +198,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( u.params.Logger.Debugw( "updating subscription permission", "permissions", logger.Proto(u.subscriptionPermission), - "version", &u.subscriptionPermissionVersion, + "version", u.subscriptionPermissionVersion, ) if err := u.parseSubscriptionPermissionsLocked(subscriptionPermission, func(pID livekit.ParticipantID) types.LocalParticipant { u.lock.Unlock() diff --git a/pkg/rtc/uptrackmanager_test.go b/pkg/rtc/uptrackmanager_test.go index 5556bc023..2bee32e38 100644 --- a/pkg/rtc/uptrackmanager_test.go +++ b/pkg/rtc/uptrackmanager_test.go @@ -211,8 +211,8 @@ func TestUpdateSubscriptionPermission(t *testing.T) { um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil) require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "out of order updates should be ignored") - um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion{}, nil) - require.True(t, um.subscriptionPermissionVersion.After(&v2), "zero version in updates should use next local version") + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion(0), nil) + require.True(t, um.subscriptionPermissionVersion.After(v2), "zero version in updates should use next local version") }) }