Fix race condition with Transport negotiations

This commit is contained in:
David Zhao
2021-06-04 12:26:23 -07:00
parent bf281b1994
commit c510ea2e1a
8 changed files with 78 additions and 55 deletions

View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/go-redis/redis/v8"
"github.com/livekit/livekit-server/pkg/testutils"
testclient "github.com/livekit/livekit-server/test/client"
"github.com/twitchtv/twirp"
@@ -30,8 +31,7 @@ const (
nodeId1 = "node-1"
nodeId2 = "node-2"
syncDelay = 100 * time.Millisecond
connectTimeout = 10 * time.Second
syncDelay = 100 * time.Millisecond
// if there are deadlocks, it's helpful to set a short test timeout (i.e. go test -timeout=30s)
// let connection timeout happen
//connectTimeout = 5000 * time.Second
@@ -95,7 +95,8 @@ func contextWithCreateRoomToken() context.Context {
func waitForServerToStart(s *service.LivekitServer) {
// wait till ready
ctx, _ := context.WithTimeout(context.Background(), connectTimeout)
ctx, cancel := context.WithTimeout(context.Background(), testutils.ConnectTimeout)
defer cancel()
for {
select {
case <-ctx.Done():
@@ -108,22 +109,6 @@ func waitForServerToStart(s *service.LivekitServer) {
}
}
func withTimeout(t *testing.T, description string, f func() bool) bool {
logger.Infow(description)
ctx, _ := context.WithTimeout(context.Background(), connectTimeout)
for {
select {
case <-ctx.Done():
t.Fatal("timed out: " + description)
return false
case <-time.After(10 * time.Millisecond):
if f() {
return true
}
}
}
}
func waitUntilConnected(t *testing.T, clients ...*testclient.RTCClient) {
logger.Infow("waiting for clients to become connected")
wg := sync.WaitGroup{}

View File

@@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/livekit/livekit-server/pkg/testutils"
livekit "github.com/livekit/livekit-server/proto"
"github.com/stretchr/testify/require"
)
@@ -35,7 +36,7 @@ func TestMultiNodeRouting(t *testing.T) {
defer t1.Stop()
}
withTimeout(t, "c2 should receive one track", func() bool {
testutils.WithTimeout(t, "c2 should receive one track", func() bool {
if len(c2.SubscribedTracks()) == 0 {
return false
}

View File

@@ -4,6 +4,7 @@ import (
"testing"
"time"
"github.com/livekit/livekit-server/pkg/testutils"
testclient "github.com/livekit/livekit-server/test/client"
"github.com/stretchr/testify/require"
@@ -26,7 +27,7 @@ func scenarioPublishingUponJoining(t *testing.T, ports ...int) {
defer stopWriters(writers...)
logger.Infow("waiting to receive tracks from c1 and c2")
success := withTimeout(t, "c3 should receive tracks from both clients", func() bool {
success := testutils.WithTimeout(t, "c3 should receive tracks from both clients", func() bool {
tracks := c3.SubscribedTracks()
if len(tracks[c1.ID()]) != 2 {
return false
@@ -46,7 +47,7 @@ func scenarioPublishingUponJoining(t *testing.T, ports ...int) {
c2.Stop()
logger.Infow("waiting for c2 tracks to be gone")
success = withTimeout(t, "c2 tracks should be gone", func() bool {
success = testutils.WithTimeout(t, "c2 tracks should be gone", func() bool {
tracks := c3.SubscribedTracks()
if len(tracks[c1.ID()]) != 2 {
return false
@@ -71,7 +72,7 @@ func scenarioPublishingUponJoining(t *testing.T, ports ...int) {
writers = publishTracksForClients(t, c2)
defer stopWriters(writers...)
success = withTimeout(t, "new c2 tracks should be published again", func() bool {
success = testutils.WithTimeout(t, "new c2 tracks should be published again", func() bool {
tracks := c3.SubscribedTracks()
if len(tracks[c2.ID()]) != 2 {
return false
@@ -98,7 +99,7 @@ func scenarioReceiveBeforePublish(t *testing.T) {
defer stopWriters(writers...)
// c2 should see some bytes flowing through
success := withTimeout(t, "waiting to receive bytes on c2", func() bool {
success := testutils.WithTimeout(t, "waiting to receive bytes on c2", func() bool {
return c2.BytesReceived() > 20
})
if !success {
@@ -109,7 +110,7 @@ func scenarioReceiveBeforePublish(t *testing.T) {
writers = publishTracksForClients(t, c2)
defer stopWriters(writers...)
success = withTimeout(t, "waiting to receive c2 tracks on c1", func() bool {
success = testutils.WithTimeout(t, "waiting to receive c2 tracks on c1", func() bool {
return len(c1.SubscribedTracks()[c2.ID()]) == 2
})
require.True(t, success)
@@ -117,7 +118,7 @@ func scenarioReceiveBeforePublish(t *testing.T) {
// now leave, and ensure that it's immediate
c2.Stop()
time.Sleep(connectTimeout)
time.Sleep(testutils.ConnectTimeout)
require.Empty(t, c1.RemoteParticipants())
}

View File

@@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/livekit/livekit-server/pkg/testutils"
testclient "github.com/livekit/livekit-server/test/client"
"github.com/stretchr/testify/require"
)
@@ -23,7 +24,7 @@ func TestClientCouldConnect(t *testing.T) {
waitUntilConnected(t, c1, c2)
// ensure they both see each other
withTimeout(t, "c1 and c2 could connect", func() bool {
testutils.WithTimeout(t, "c1 and c2 could connect", func() bool {
if len(c1.RemoteParticipants()) == 0 || len(c2.RemoteParticipants()) == 0 {
return false
}
@@ -56,7 +57,7 @@ func TestSinglePublisher(t *testing.T) {
// a new client joins and should get the initial stream
c3 := createRTCClient("c3", defaultServerPort, nil)
success := withTimeout(t, "c2 should receive two tracks", func() bool {
success := testutils.WithTimeout(t, "c2 should receive two tracks", func() bool {
if len(c2.SubscribedTracks()) == 0 {
return false
}
@@ -75,7 +76,7 @@ func TestSinglePublisher(t *testing.T) {
// ensure that new client that has joined also received tracks
waitUntilConnected(t, c3)
success = withTimeout(t, "c2 should receive two tracks", func() bool {
success = testutils.WithTimeout(t, "c2 should receive two tracks", func() bool {
if len(c3.SubscribedTracks()) == 0 {
return false
}
@@ -98,7 +99,7 @@ func TestSinglePublisher(t *testing.T) {
// when c3 disconnects.. ensure subscriber is cleaned up correctly
c3.Stop()
success = withTimeout(t, "c3 is cleaned up as a subscriber", func() bool {
success = testutils.WithTimeout(t, "c3 is cleaned up as a subscriber", func() bool {
room := s.RoomManager().GetRoom(testRoom)
require.NotNil(t, room)