Log ICE connection info on failure. (#3134)

- Truncate public remote IP
- Log only on short connection to avoid logging too much
This commit is contained in:
Raja Subramanian
2024-10-24 14:30:04 +05:30
committed by GitHub
parent ca77df8212
commit b8c6b1f1ec
6 changed files with 60 additions and 20 deletions
+1 -1
View File
@@ -1334,7 +1334,7 @@ type AnyTransportHandler struct {
p *ParticipantImpl
}
func (h AnyTransportHandler) OnFailed(isShortLived bool) {
func (h AnyTransportHandler) OnFailed(_isShortLived bool, _ici *types.ICEConnectionInfo) {
h.p.onAnyTransportFailed()
}
+25 -1
View File
@@ -18,6 +18,7 @@ import (
"context"
"fmt"
"math"
"net"
"slices"
"sort"
"strings"
@@ -54,6 +55,8 @@ const (
dataForwardLoadBalanceThreshold = 20
simulateDisconnectSignalTimeout = 5 * time.Second
minIPTruncateLen = 8
)
var (
@@ -1837,7 +1840,28 @@ func connectionDetailsFields(infos []*types.ICEConnectionInfo) []interface{} {
if c.Trickle {
cStr += "[trickle]"
}
cStr += " " + c.Remote.String()
remoteAddress := c.Remote.Address()
ipAddr := net.ParseIP(remoteAddress)
isPrivate := false
if ipAddr != nil {
isPrivate = ipAddr.IsPrivate()
}
if !isPrivate && len(remoteAddress) > minIPTruncateLen {
remoteAddress = remoteAddress[:len(remoteAddress)-3] + "..."
}
cStr += " " + fmt.Sprintf("%s %s %s:%d", c.Remote.NetworkType(), c.Remote.Type(), c.Remote.Address(), c.Remote.Port())
if relatedAddress := c.Remote.RelatedAddress(); relatedAddress != nil {
ipAddr = net.ParseIP(relatedAddress.Address)
if ipAddr != nil {
isPrivate = ipAddr.IsPrivate()
}
relatedAddressAddress := relatedAddress.Address
if !isPrivate && len(relatedAddressAddress) > minIPTruncateLen {
relatedAddressAddress = relatedAddressAddress[:len(relatedAddressAddress)-3] + "..."
}
cStr += " " + fmt.Sprintf(" related %s:%d", relatedAddressAddress, relatedAddress.Port)
}
candidates = append(candidates, cStr)
}
if len(candidates) > 0 {
+1 -1
View File
@@ -640,7 +640,7 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) {
}
}
t.params.Handler.OnFailed(isShort)
t.params.Handler.OnFailed(isShort, t.GetICEConnectionInfo())
}
func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState) {
+2 -1
View File
@@ -19,6 +19,7 @@ import (
"github.com/pion/webrtc/v3"
"github.com/livekit/livekit-server/pkg/rtc/types"
"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
"github.com/livekit/protocol/livekit"
)
@@ -36,7 +37,7 @@ type Handler interface {
OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error
OnInitialConnected()
OnFullyEstablished()
OnFailed(isShortLived bool)
OnFailed(isShortLived bool, iceConnectionInfo *types.ICEConnectionInfo)
OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver)
OnDataPacket(kind livekit.DataPacket_Kind, data []byte)
OnDataSendError(err error)
@@ -5,6 +5,7 @@ import (
"sync"
"github.com/livekit/livekit-server/pkg/rtc/transport"
"github.com/livekit/livekit-server/pkg/rtc/types"
"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
"github.com/livekit/protocol/livekit"
webrtc "github.com/pion/webrtc/v3"
@@ -33,10 +34,11 @@ type FakeHandler struct {
onDataSendErrorArgsForCall []struct {
arg1 error
}
OnFailedStub func(bool)
OnFailedStub func(bool, *types.ICEConnectionInfo)
onFailedMutex sync.RWMutex
onFailedArgsForCall []struct {
arg1 bool
arg2 *types.ICEConnectionInfo
}
OnFullyEstablishedStub func()
onFullyEstablishedMutex sync.RWMutex
@@ -230,16 +232,17 @@ func (fake *FakeHandler) OnDataSendErrorArgsForCall(i int) error {
return argsForCall.arg1
}
func (fake *FakeHandler) OnFailed(arg1 bool) {
func (fake *FakeHandler) OnFailed(arg1 bool, arg2 *types.ICEConnectionInfo) {
fake.onFailedMutex.Lock()
fake.onFailedArgsForCall = append(fake.onFailedArgsForCall, struct {
arg1 bool
}{arg1})
arg2 *types.ICEConnectionInfo
}{arg1, arg2})
stub := fake.OnFailedStub
fake.recordInvocation("OnFailed", []interface{}{arg1})
fake.recordInvocation("OnFailed", []interface{}{arg1, arg2})
fake.onFailedMutex.Unlock()
if stub != nil {
fake.OnFailedStub(arg1)
fake.OnFailedStub(arg1, arg2)
}
}
@@ -249,17 +252,17 @@ func (fake *FakeHandler) OnFailedCallCount() int {
return len(fake.onFailedArgsForCall)
}
func (fake *FakeHandler) OnFailedCalls(stub func(bool)) {
func (fake *FakeHandler) OnFailedCalls(stub func(bool, *types.ICEConnectionInfo)) {
fake.onFailedMutex.Lock()
defer fake.onFailedMutex.Unlock()
fake.OnFailedStub = stub
}
func (fake *FakeHandler) OnFailedArgsForCall(i int) bool {
func (fake *FakeHandler) OnFailedArgsForCall(i int) (bool, *types.ICEConnectionInfo) {
fake.onFailedMutex.RLock()
defer fake.onFailedMutex.RUnlock()
argsForCall := fake.onFailedArgsForCall[i]
return argsForCall.arg1
return argsForCall.arg1, argsForCall.arg2
}
func (fake *FakeHandler) OnFullyEstablished() {
+20 -8
View File
@@ -51,16 +51,24 @@ const (
udpLossUnstableCountThreshold = 20
)
// -------------------------------
type TransportManagerTransportHandler struct {
transport.Handler
t *TransportManager
t *TransportManager
logger logger.Logger
}
func (h TransportManagerTransportHandler) OnFailed(isShortLived bool) {
func (h TransportManagerTransportHandler) OnFailed(isShortLived bool, iceConnectionInfo *types.ICEConnectionInfo) {
if isShortLived {
h.logger.Infow("short ice connection", connectionDetailsFields([]*types.ICEConnectionInfo{iceConnectionInfo})...)
}
h.t.handleConnectionFailed(isShortLived)
h.Handler.OnFailed(isShortLived)
h.Handler.OnFailed(isShortLived, iceConnectionInfo)
}
// -------------------------------
type TransportManagerPublisherTransportHandler struct {
TransportManagerTransportHandler
}
@@ -70,6 +78,8 @@ func (h TransportManagerPublisherTransportHandler) OnAnswer(sd webrtc.SessionDes
return h.Handler.OnAnswer(sd)
}
// -------------------------------
type TransportManagerParams struct {
Identity livekit.ParticipantIdentity
SID livekit.ParticipantID
@@ -133,6 +143,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro
}
t.mediaLossProxy.OnMediaLossUpdate(t.onMediaLossUpdate)
lgr := LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER)
publisher, err := NewPCTransport(TransportParams{
ParticipantID: params.SID,
ParticipantIdentity: params.Identity,
@@ -142,11 +153,11 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro
DirectionConfig: params.Config.Publisher,
CongestionControlConfig: params.CongestionControlConfig,
EnabledCodecs: params.EnabledPublishCodecs,
Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER),
Logger: lgr,
SimTracks: params.SimTracks,
ClientInfo: params.ClientInfo,
Transport: livekit.SignalTarget_PUBLISHER,
Handler: TransportManagerPublisherTransportHandler{TransportManagerTransportHandler{params.PublisherHandler, t}},
Handler: TransportManagerPublisherTransportHandler{TransportManagerTransportHandler{params.PublisherHandler, t, lgr}},
DropRemoteICECandidates: params.DropRemoteICECandidates,
})
if err != nil {
@@ -154,6 +165,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro
}
t.publisher = publisher
lgr = LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER)
subscriber, err := NewPCTransport(TransportParams{
ParticipantID: params.SID,
ParticipantIdentity: params.Identity,
@@ -162,14 +174,14 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro
DirectionConfig: params.Config.Subscriber,
CongestionControlConfig: params.CongestionControlConfig,
EnabledCodecs: params.EnabledSubscribeCodecs,
Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER),
Logger: lgr,
ClientInfo: params.ClientInfo,
IsOfferer: true,
IsSendSide: true,
AllowPlayoutDelay: params.AllowPlayoutDelay,
DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount,
Transport: livekit.SignalTarget_SUBSCRIBER,
Handler: TransportManagerTransportHandler{params.SubscriberHandler, t},
Handler: TransportManagerTransportHandler{params.SubscriberHandler, t, lgr},
DropRemoteICECandidates: params.DropRemoteICECandidates,
})
if err != nil {
@@ -690,7 +702,7 @@ func (t *TransportManager) onMediaLossUpdate(loss uint8) {
t.lock.Unlock()
t.params.Logger.Infow("udp connection unstable, switch to tcp", "signalingRTT", t.signalingRTT)
t.params.SubscriberHandler.OnFailed(true)
t.params.SubscriberHandler.OnFailed(true, t.subscriber.GetICEConnectionInfo())
return
}
}