livekit/test/singlenode_test.go

// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"net/http"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/jxskiss/base62"
	"github.com/pion/sdp/v3"
	"github.com/pion/stun/v3"
	"github.com/pion/turn/v5"
	"github.com/pion/webrtc/v4"
	"github.com/stretchr/testify/require"
	"github.com/thoas/go-funk"
	"github.com/twitchtv/twirp"
	"go.uber.org/atomic"

	"github.com/livekit/protocol/auth"
	"github.com/livekit/protocol/codecs/mime"
	"github.com/livekit/protocol/livekit"
	"github.com/livekit/protocol/logger"

	"github.com/livekit/livekit-server/pkg/config"
	"github.com/livekit/livekit-server/pkg/rtc"
	"github.com/livekit/livekit-server/pkg/rtc/types"
	"github.com/livekit/livekit-server/pkg/service"
	"github.com/livekit/livekit-server/pkg/sfu"
	"github.com/livekit/livekit-server/pkg/sfu/datachannel"
	"github.com/livekit/livekit-server/pkg/testutils"
	testclient "github.com/livekit/livekit-server/test/client"
)

const (
	waitTick    = 10 * time.Millisecond
	waitTimeout = 5 * time.Second
)

func TestClientCouldConnect(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestClientCouldConnect")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1, c2)

			// ensure they both see each other
			testutils.WithTimeout(t, func() string {
				if len(c1.RemoteParticipants()) == 0 {
					return "c1 did not see c2"
				}
				if len(c2.RemoteParticipants()) == 0 {
					return "c2 did not see c1"
				}
				return ""
			})
		})
	}
}

func TestClientConnectDuplicate(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestClientConnectDuplicate")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom}
			grant.SetCanPublish(true)
			grant.SetCanSubscribe(true)
			token := joinTokenWithGrant("c1", grant)
			c1 := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, nil)

			// publish 2 tracks
			t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
			require.NoError(t, err)
			defer t1.Stop()
			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
			require.NoError(t, err)
			defer t2.Stop()

			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1, c2)

			opts := &testclient.Options{
				Publish: "duplicate_connection",
			}
			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2 didn't subscribe to anything"
				}
				// should have received two tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
					return "c2 didn't subscribe to both tracks from c1"
				}

				// participant ID can be appended with '#..' . but should contain orig id as prefix
				tr1 := c2.SubscribedTracks()[c1.ID()][0]
				participantId1, _ := rtc.UnpackStreamID(tr1.StreamID())
				require.Equal(t, c1.ID(), participantId1)
				tr2 := c2.SubscribedTracks()[c1.ID()][1]
				participantId2, _ := rtc.UnpackStreamID(tr2.StreamID())
				require.Equal(t, c1.ID(), participantId2)
				return ""
			})

			c1Dup := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, opts)

			waitUntilConnected(t, c1Dup)

			t3, err := c1Dup.AddStaticTrack("video/vp8", "video", "webcam")
			require.NoError(t, err)
			defer t3.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()[c1Dup.ID()]) != 1 {
					return "c2 was not subscribed to track from duplicated c1"
				}

				tr3 := c2.SubscribedTracks()[c1Dup.ID()][0]
				participantId3, _ := rtc.UnpackStreamID(tr3.StreamID())
				require.Contains(t, c1Dup.ID(), participantId3)

				return ""
			})
		})
	}
}

func TestSinglePublisher(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	s, finish := setupSingleNodeTest("TestSinglePublisher")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1, c2)

			// publish an audio and video track and ensure clients receive it ok
			t1, err := c1.AddStaticTrack("audio/OPUS", "audio", "webcamaudio")
			require.NoError(t, err)
			defer t1.Stop()
			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcamvideo")
			require.NoError(t, err)
			defer t2.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2 was not subscribed to anything"
				}
				// should have received two tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
					return "c2 didn't subscribe to both tracks from c1"
				}

				tr1 := c2.SubscribedTracks()[c1.ID()][0]
				participantId, _ := rtc.UnpackStreamID(tr1.StreamID())
				require.Equal(t, c1.ID(), participantId)
				return ""
			})
			// ensure mime type is received
			remoteC1 := c2.GetRemoteParticipant(c1.ID())
			audioTrack := funk.Find(remoteC1.Tracks, func(ti *livekit.TrackInfo) bool {
				return ti.Name == "webcamaudio"
			}).(*livekit.TrackInfo)
			require.Equal(t, "audio/opus", audioTrack.MimeType)

			// a new client joins and should get the initial stream
			c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, nil)

			// ensure that new client that has joined also received tracks
			waitUntilConnected(t, c3)
			testutils.WithTimeout(t, func() string {
				if len(c3.SubscribedTracks()) == 0 {
					return "c3 didn't subscribe to anything"
				}
				// should have received two tracks
				if len(c3.SubscribedTracks()[c1.ID()]) != 2 {
					return "c3 didn't subscribe to tracks from c1"
				}
				return ""
			})

			// ensure that the track ids are generated by server
			tracks := c3.SubscribedTracks()[c1.ID()]
			for _, tr := range tracks {
				require.True(t, strings.HasPrefix(tr.ID(), "TR_"), "track should begin with TR")
			}

			// when c3 disconnects, ensure subscriber is cleaned up correctly
			c3.Stop()

			testutils.WithTimeout(t, func() string {
				room := s.RoomManager().GetRoom(context.Background(), testRoom)
				p := room.GetParticipant("c1")
				require.NotNil(t, p)

				for _, t := range p.GetPublishedTracks() {
					if t.IsSubscriber(c3.ID()) {
						return "c3 was not a subscriber of c1's tracks"
					}
				}
				return ""
			})
		})
	}
}

func TestConnectionStats(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	s, finish := setupSingleNodeTest("TestConnectionStats")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1, c2)
			defer func() {
				c1.Stop()
				c2.Stop()
			}()

			// both clients publish audio + video
			t1, err := c1.AddStaticTrack("audio/opus", "audio", "c1audio")
			require.NoError(t, err)
			defer t1.Stop()
			t2, err := c1.AddStaticTrack("video/vp8", "video", "c1video")
			require.NoError(t, err)
			defer t2.Stop()

			t3, err := c2.AddStaticTrack("audio/opus", "audio", "c2audio")
			require.NoError(t, err)
			defer t3.Stop()
			t4, err := c2.AddStaticTrack("video/vp8", "video", "c2video")
			require.NoError(t, err)
			defer t4.Stop()

			// wait for cross-subscriptions: each client should receive 2 tracks from the other
			testutils.WithTimeout(t, func() string {
				if len(c1.SubscribedTracks()[c2.ID()]) != 2 {
					return "c1 did not subscribe to both tracks from c2"
				}
				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
					return "c2 did not subscribe to both tracks from c1"
				}
				return ""
			})

			room := s.RoomManager().GetRoom(context.Background(), testRoom)
			require.NotNil(t, room)

			// hook the upstream WebRTCReceiver.OnStatsUpdate and downstream DownTrack.OnStatsUpdate
			// callbacks so we can verify the AnalyticsStat delivered through each carries valid
			// delta data. MediaTrack.Receivers() returns one entry per potential codec; only those
			// matching the actually published codec are *sfu.WebRTCReceiver, the rest are
			// placeholder *rtc.DummyReceiver instances that we skip.
			type statCapture struct {
				lock sync.Mutex
				stat *livekit.AnalyticsStat
			}
			receiverCaptures := make(map[livekit.TrackID]*statCapture)
			downTrackCaptures := make(map[livekit.ParticipantIdentity]map[livekit.TrackID]*statCapture)
			for _, identity := range []livekit.ParticipantIdentity{"c1", "c2"} {
				p := room.GetParticipant(identity)
				require.NotNil(t, p, "participant %s not found", identity)
				for _, mt := range p.GetPublishedTracks() {
					rc := &statCapture{}
					receiverCaptures[mt.ID()] = rc
					var hooked int
					for _, r := range mt.Receivers() {
						if dr, ok := r.(*rtc.DummyReceiver); ok {
							underlying := dr.Receiver()
							if underlying == nil {
								continue
							}
							r = underlying
						}
						wr, ok := r.(*sfu.WebRTCReceiver)
						if !ok {
							continue
						}
						wr.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) {
							rc.lock.Lock()
							rc.stat = stat
							rc.lock.Unlock()
						})
						hooked++
					}
					require.Greater(t, hooked, 0, "no live WebRTCReceiver found for published track %s", mt.ID())
				}

				dtCaps := make(map[livekit.TrackID]*statCapture)
				downTrackCaptures[identity] = dtCaps
				for _, st := range p.GetSubscribedTracks() {
					dt := st.DownTrack()
					require.NotNil(t, dt, "subscribed track %s has no DownTrack", st.ID())
					dc := &statCapture{}
					dtCaps[st.ID()] = dc
					dt.OnStatsUpdate(func(_ *sfu.DownTrack, stat *livekit.AnalyticsStat) {
						dc.lock.Lock()
						dc.stat = stat
						dc.lock.Unlock()
					})
				}
			}

			validateAnalyticsStat := func(stat *livekit.AnalyticsStat) string {
				if stat == nil {
					return "stat nil"
				}
				if len(stat.Streams) == 0 {
					return "stat has no streams"
				}
				var totalPackets uint32
				var totalBytes uint64
				for _, s := range stat.Streams {
					totalPackets += s.PrimaryPackets
					totalBytes += s.PrimaryBytes
				}
				if totalPackets == 0 {
					return "stat has no packets across streams"
				}
				if totalBytes == 0 {
					return "stat has no bytes across streams"
				}
				return ""
			}

			// wait for cumulative + delta + OnStatsUpdate-derived stats. the
			// connection-quality update interval is 5s, so allow plenty of time for
			// the receiver OnStatsUpdate callback to fire at least once and for
			// the downstream connection-quality scorer to compute a real score.
			testutils.WithTimeout(t, func() string {
				for _, identity := range []livekit.ParticipantIdentity{"c1", "c2"} {
					p := room.GetParticipant(identity)
					if p == nil {
						return fmt.Sprintf("participant %s not found", identity)
					}

					// upstream (publisher) cumulative stats
					published := p.GetPublishedTracks()
					if len(published) != 2 {
						return fmt.Sprintf("%s expected 2 published tracks, got %d", identity, len(published))
					}
					for _, mt := range published {
						lmt, ok := mt.(types.LocalMediaTrack)
						if !ok {
							return fmt.Sprintf("%s published track %s is not a LocalMediaTrack", identity, mt.ID())
						}
						stats := lmt.GetTrackStats()
						if stats == nil {
							return fmt.Sprintf("%s upstream cumulative stats nil for track %s", identity, mt.ID())
						}
						if stats.Packets == 0 {
							return fmt.Sprintf("%s upstream cumulative stats has no packets for track %s", identity, mt.ID())
						}
						if stats.Bytes == 0 {
							return fmt.Sprintf("%s upstream cumulative stats has no bytes for track %s", identity, mt.ID())
						}

						// upstream delta stats fed into the receiver OnStatsUpdate path
						rc, ok := receiverCaptures[mt.ID()]
						if !ok {
							return fmt.Sprintf("%s missing receiver capture for track %s", identity, mt.ID())
						}
						rc.lock.Lock()
						stat := rc.stat
						rc.lock.Unlock()
						if msg := validateAnalyticsStat(stat); msg != "" {
							return fmt.Sprintf("%s upstream OnStatsUpdate %s for track %s", identity, msg, mt.ID())
						}
					}

					// downstream (subscriber) cumulative stats and DownTrack OnStatsUpdate
					// delta stats captured from the listener path
					subscribed := p.GetSubscribedTracks()
					if len(subscribed) != 2 {
						return fmt.Sprintf("%s expected 2 subscribed tracks, got %d", identity, len(subscribed))
					}
					for _, st := range subscribed {
						dt := st.DownTrack()
						if dt == nil {
							return fmt.Sprintf("%s subscribed track %s has no DownTrack", identity, st.ID())
						}
						stats := dt.GetTrackStats()
						if stats == nil {
							return fmt.Sprintf("%s downstream cumulative stats nil for track %s", identity, st.ID())
						}
						if stats.Packets == 0 {
							return fmt.Sprintf("%s downstream cumulative stats has no packets for track %s", identity, st.ID())
						}
						if stats.Bytes == 0 {
							return fmt.Sprintf("%s downstream cumulative stats has no bytes for track %s", identity, st.ID())
						}

						// downstream delta stats fed into the DownTrack OnStatsUpdate path
						dc, ok := downTrackCaptures[identity][st.ID()]
						if !ok {
							return fmt.Sprintf("%s missing DownTrack capture for track %s", identity, st.ID())
						}
						dc.lock.Lock()
						stat := dc.stat
						dc.lock.Unlock()
						if msg := validateAnalyticsStat(stat); msg != "" {
							return fmt.Sprintf("%s downstream OnStatsUpdate %s for track %s", identity, msg, st.ID())
						}
					}
				}
				return ""
			}, 15*time.Second)
		})
	}
}

func Test_WhenAutoSubscriptionDisabled_ClientShouldNotReceiveAnyPublishedTracks(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("Test_WhenAutoSubscriptionDisabled_ClientShouldNotReceiveAnyPublishedTracks")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			opts := testclient.Options{AutoSubscribe: false}
			publisher := createRTCClient("publisher", defaultServerPort, testRTCServicePath, &opts)
			client := createRTCClient("client", defaultServerPort, testRTCServicePath, &opts)
			defer publisher.Stop()
			defer client.Stop()
			waitUntilConnected(t, publisher, client)

			track, err := publisher.AddStaticTrack("audio/opus", "audio", "webcam")
			require.NoError(t, err)
			defer track.Stop()

			time.Sleep(syncDelay)

			require.Empty(t, client.SubscribedTracks()[publisher.ID()])
		})
	}
}

func Test_RenegotiationWithDifferentCodecs(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestRenegotiationWithDifferentCodecs")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1, c2)

			// publish a vp8 video track and ensure clients receive it ok
			t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
			require.NoError(t, err)
			defer t1.Stop()
			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
			require.NoError(t, err)
			defer t2.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2 was not subscribed to anything"
				}
				// should have received two tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
					return "c2 was not subscribed to tracks from c1"
				}

				tracks := c2.SubscribedTracks()[c1.ID()]
				for _, t := range tracks {
					if mime.IsMimeTypeStringVP8(t.Codec().MimeType) {
						return ""

					}
				}
				return "did not receive track with vp8"
			})

			t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{
				MimeType:    "video/h264",
				ClockRate:   90000,
				SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
			}, "videoscreen", "screen")
			defer t3.Stop()
			require.NoError(t, err)

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2's not subscribed to anything"
				}
				// should have received three tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 3 {
					return "c2's not subscribed to 3 tracks from c1"
				}

				var vp8Found, h264Found bool
				tracks := c2.SubscribedTracks()[c1.ID()]
				for _, t := range tracks {
					if mime.IsMimeTypeStringVP8(t.Codec().MimeType) {
						vp8Found = true
					} else if mime.IsMimeTypeStringH264(t.Codec().MimeType) {
						h264Found = true
					}
				}
				if !vp8Found {
					return "did not receive track with vp8"
				}
				if !h264Found {
					return "did not receive track with h264"
				}
				return ""
			})
		})
	}
}

func TestSingleNodeRoomList(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	_, finish := setupSingleNodeTest("TestSingleNodeRoomList")
	defer finish()

	roomServiceListRoom(t)
}

func TestSingleNodeUpdateParticipant(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	_, finish := setupSingleNodeTest("TestSingleNodeRoomList")
	defer finish()

	adminCtx := contextWithToken(adminRoomToken(testRoom))
	t.Run("update nonexistent participant", func(t *testing.T) {
		_, err := roomClient.UpdateParticipant(adminCtx, &livekit.UpdateParticipantRequest{
			Room:     testRoom,
			Identity: "nonexistent",
			Permission: &livekit.ParticipantPermission{
				CanPublish: true,
			},
		})
		require.Error(t, err)
		var twErr twirp.Error
		require.True(t, errors.As(err, &twErr))
		require.Equal(t, twirp.NotFound, twErr.Code())
	})
}

// Ensure that CORS headers are returned
func TestSingleNodeCORS(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	s, finish := setupSingleNodeTest("TestSingleNodeCORS")
	defer finish()

	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d", s.HTTPPort()), nil)
	require.NoError(t, err)
	req.Header.Set("Authorization", "bearer xyz")
	req.Header.Set("Origin", "testhost.com")
	res, err := http.DefaultClient.Do(req)
	require.NoError(t, err)
	require.Equal(t, "testhost.com", res.Header.Get("Access-Control-Allow-Origin"))
}

func TestSingleNodeDoubleSlash(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	s, finish := setupSingleNodeTest("TestSingleNodeDoubleSlash")
	defer finish()
	// client contains trailing slash in URL, causing path to contain double //
	// without our middleware, this would cause a 302 redirect
	roomClient = livekit.NewRoomServiceJSONClient(fmt.Sprintf("http://localhost:%d/", s.HTTPPort()), &http.Client{})
	_, err := roomClient.ListRooms(contextWithToken(listRoomToken()), &livekit.ListRoomsRequest{})
	require.NoError(t, err)
}

func TestPingPong(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	_, finish := setupSingleNodeTest("TestPingPong")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, c1)

			require.NoError(t, c1.SendPing())
			require.Eventually(t, func() bool {
				return c1.PongReceivedAt() > 0
			}, time.Second, 10*time.Millisecond)
		})
	}
}

func TestSingleNodeJoinAfterClose(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestJoinAfterClose")
	defer finish()

	scenarioJoinClosedRoom(t)
}

func TestSingleNodeCloseNonRTCRoom(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("closeNonRTCRoom")
	defer finish()

	closeNonRTCRoom(t)
}

func TestAutoCreate(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	disableAutoCreate := func(conf *config.Config) {
		conf.Room.AutoCreate = false
	}
	t.Run("cannot join if room isn't created", func(t *testing.T) {
		s := createSingleNodeServer(disableAutoCreate)
		go func() {
			if err := s.Start(); err != nil {
				logger.Errorw("server returned error", err)
			}
		}()
		defer s.Stop(true)

		waitForServerToStart(s)

		for _, testRTCServicePath := range testRTCServicePaths {
			t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
				token := joinToken(testRoom, "start-before-create", nil)
				opts := &testclient.Options{}
				testRTCServicePathToTestClientOptions(testRTCServicePath, opts)
				_, err := testclient.NewWebSocketConn(
					fmt.Sprintf("ws://localhost:%d", defaultServerPort),
					token,
					opts,
				)
				require.Error(t, err)

				// second join should also fail
				token = joinToken(testRoom, "start-before-create-2", nil)
				_, err = testclient.NewWebSocketConn(
					fmt.Sprintf("ws://localhost:%d", defaultServerPort),
					token,
					opts,
				)
				require.Error(t, err)
			})
		}
	})

	t.Run("join with explicit createRoom", func(t *testing.T) {
		s := createSingleNodeServer(disableAutoCreate)
		go func() {
			if err := s.Start(); err != nil {
				logger.Errorw("server returned error", err)
			}
		}()
		defer s.Stop(true)

		waitForServerToStart(s)

		// explicitly create
		_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{Name: testRoom})
		require.NoError(t, err)

		for _, testRTCServicePath := range testRTCServicePaths {
			t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
				c1 := createRTCClient("join-after-create", defaultServerPort, testRTCServicePath, nil)
				waitUntilConnected(t, c1)

				c1.Stop()
			})
		}
	})
}

// don't give user subscribe permissions initially, and ensure autosubscribe is triggered afterwards
func TestSingleNodeUpdateSubscriptionPermissions(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	_, finish := setupSingleNodeTest("TestSingleNodeUpdateSubscriptionPermissions")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, nil)

			grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom}
			grant.SetCanSubscribe(false)
			at := auth.NewAccessToken(testApiKey, testApiSecret).
				AddGrant(grant).
				SetIdentity("sub")
			token, err := at.ToJWT()
			require.NoError(t, err)
			sub := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, &testclient.Options{
				AutoSubscribe:          true,
				AutoSubscribeDataTrack: true,
			})

			waitUntilConnected(t, pub, sub)

			writers := publishTracksForClients(t, pub)
			defer stopWriters(writers...)

			// publish a data track as well
			dtw, err := pub.PublishDataTrack()
			require.NoError(t, err)
			defer dtw.Stop()

			// wait sub receives tracks
			testutils.WithTimeout(t, func() string {
				pubRemote := sub.GetRemoteParticipant(pub.ID())
				if pubRemote == nil {
					return "could not find remote publisher"
				}
				if len(pubRemote.Tracks) != 2 {
					return "did not receive metadata for published tracks"
				}
				return ""
			})

			// no subscriptions should have been made while canSubscribe is false
			require.Empty(t, sub.SubscribedTracks()[pub.ID()])
			require.Empty(t, sub.SubscribedDataTracks()[pub.ID()])

			// set permissions out of band
			ctx := contextWithToken(adminRoomToken(testRoom))
			_, err = roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
				Room:     testRoom,
				Identity: "sub",
				Permission: &livekit.ParticipantPermission{
					CanSubscribe: true,
					CanPublish:   true,
				},
			})
			require.NoError(t, err)

			testutils.WithTimeout(t, func() string {
				tracks := sub.SubscribedTracks()[pub.ID()]
				if len(tracks) != 2 {
					return fmt.Sprintf("expected 2 tracks subscribed, actual: %d", len(tracks))
				}
				if len(sub.SubscribedDataTracks()[pub.ID()]) != 1 {
					return "expected data track to be subscribed"
				}
				return ""
			})
		})
	}
}

func TestSingleNodeAttributes(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}
	_, finish := setupSingleNodeTest("TestSingleNodeAttributes")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, &testclient.Options{
				Attributes: map[string]string{
					"b": "2",
					"c": "3",
				},
				TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
					T := true
					grants.CanUpdateOwnMetadata = &T
					token.SetAttributes(map[string]string{
						"a": "0",
						"b": "1",
					})
				},
			})

			grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom}
			grant.SetCanSubscribe(false)
			at := auth.NewAccessToken(testApiKey, testApiSecret).
				SetVideoGrant(grant).
				SetIdentity("sub")
			token, err := at.ToJWT()
			require.NoError(t, err)
			sub := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, nil)

			waitUntilConnected(t, pub, sub)

			// wait sub receives initial attributes
			testutils.WithTimeout(t, func() string {
				pubRemote := sub.GetRemoteParticipant(pub.ID())
				if pubRemote == nil {
					return "could not find remote publisher"
				}
				attrs := pubRemote.Attributes
				if !reflect.DeepEqual(attrs, map[string]string{
					"a": "0",
					"b": "2",
					"c": "3",
				}) {
					return fmt.Sprintf("did not receive expected attributes: %v", attrs)
				}
				return ""
			})
		})
	}
}

// TestDeviceCodecOverride checks that codecs that are incompatible with a device is not
// negotiated by the server
func TestDeviceCodecOverride(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestDeviceCodecOverride")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			// simulate device that isn't compatible with H.264
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, &testclient.Options{
				ClientInfo: &livekit.ClientInfo{
					Os:          "android",
					DeviceModel: "Xiaomi 2201117TI",
				},
			})
			defer c1.Stop()
			waitUntilConnected(t, c1)

			// it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs
			tw, err := c1.AddStaticTrack("video/h264", "video", "webcam")
			require.NoError(t, err)
			defer stopWriters(tw)

			var desc *sdp.MediaDescription
			require.Eventually(t, func() bool {
				lastAnswer := c1.LastAnswer()
				if lastAnswer == nil {
					return false
				}

				sd := webrtc.SessionDescription{
					Type: webrtc.SDPTypeAnswer,
					SDP:  lastAnswer.SDP,
				}
				answer, err := sd.Unmarshal()
				require.NoError(t, err)

				// video and data channel
				if len(answer.MediaDescriptions) < 2 {
					return false
				}

				for _, md := range answer.MediaDescriptions {
					if md.MediaName.Media == "video" {
						desc = md
						break
					}
				}
				return desc != nil
			}, waitTimeout, waitTick, "did not receive answer")

			hasSeenVP8 := false
			for _, a := range desc.Attributes {
				if a.Key == "rtpmap" {
					require.NotContains(t, a.Value, mime.MimeTypeCodecH264.String(), "should not contain H264 codec")
					if strings.Contains(a.Value, mime.MimeTypeCodecVP8.String()) {
						hasSeenVP8 = true
					}
				}
			}
			require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP")
		})
	}
}

func TestSubscribeToCodecUnsupported(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupSingleNodeTest("TestSubscribeToCodecUnsupported")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
			// create a client that doesn't support H264
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &testclient.Options{
				AutoSubscribe: true,
				DisabledCodecs: []webrtc.RTPCodecCapability{
					{MimeType: "video/H264"},
				},
			})
			waitUntilConnected(t, c1, c2)

			// publish a vp8 video track and ensure c2 receives it ok
			t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
			require.NoError(t, err)
			defer t1.Stop()
			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
			require.NoError(t, err)
			defer t2.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2 was not subscribed to anything"
				}
				// should have received two tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
					return "c2 was not subscribed to tracks from c1"
				}

				tracks := c2.SubscribedTracks()[c1.ID()]
				for _, t := range tracks {
					if mime.IsMimeTypeStringVP8(t.Codec().MimeType) {
						return ""
					}
				}
				return "did not receive track with vp8"
			})
			require.Nil(t, c2.GetSubscriptionResponseAndClear())

			// publish a h264 track and ensure c2 got subscription error
			t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{
				MimeType:    "video/h264",
				ClockRate:   90000,
				SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
			}, "videoscreen", "screen")
			defer t3.Stop()
			require.NoError(t, err)

			var h264TrackID string
			require.Eventually(t, func() bool {
				remoteC1 := c2.GetRemoteParticipant(c1.ID())
				require.NotNil(t, remoteC1)
				for _, track := range remoteC1.Tracks {
					if mime.IsMimeTypeStringH264(track.MimeType) {
						h264TrackID = track.Sid
						return true
					}
				}
				return false
			}, time.Second, 10*time.Millisecond, "did not receive track info with h264")

			require.Eventually(t, func() bool {
				sr := c2.GetSubscriptionResponseAndClear()
				if sr == nil {
					return false
				}
				require.Equal(t, h264TrackID, sr.TrackSid)
				require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err)
				return true
			}, 5*time.Second, 10*time.Millisecond, "did not receive subscription response")

			// publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it
			t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2")
			require.NoError(t, err)
			defer t4.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedTracks()) == 0 {
					return "c2 was not subscribed to anything"
				}
				// should have received two tracks
				if len(c2.SubscribedTracks()[c1.ID()]) != 3 {
					return "c2 was not subscribed to tracks from c1"
				}

				var vp8Count int
				tracks := c2.SubscribedTracks()[c1.ID()]
				for _, t := range tracks {
					if mime.IsMimeTypeStringVP8(t.Codec().MimeType) {
						vp8Count++
					}
				}
				if vp8Count == 2 {
					return ""
				}
				return "did not 2 receive track with vp8"
			})
			require.Nil(t, c2.GetSubscriptionResponseAndClear())
		})
	}
}

func TestDataPublishSlowSubscriber(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	dataChannelSlowThreshold := 21024

	logger.Infow("----------------STARTING TEST----------------", "test", t.Name())
	s := createSingleNodeServer(func(c *config.Config) {
		c.RTC.DatachannelSlowThreshold = dataChannelSlowThreshold
	})
	go func() {
		if err := s.Start(); err != nil {
			logger.Errorw("server returned error", err)
		}
	}()

	waitForServerToStart(s)

	defer func() {
		s.Stop(true)
		logger.Infow("----------------FINISHING TEST----------------", "test", t.Name())
	}()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, nil)
			fastSub := createRTCClient("fastSub", defaultServerPort, testRTCServicePath, nil)
			slowSubNotDrop := createRTCClient("slowSubNotDrop", defaultServerPort, testRTCServicePath, nil)
			slowSubDrop := createRTCClient("slowSubDrop", defaultServerPort, testRTCServicePath, nil)
			waitUntilConnected(t, pub, fastSub, slowSubDrop, slowSubNotDrop)
			defer func() {
				pub.Stop()
				fastSub.Stop()
				slowSubNotDrop.Stop()
				slowSubDrop.Stop()
			}()

			// no data should be dropped for fast subscriber
			var fastDataIndex atomic.Uint64
			fastSub.SetOnDataReceived(func(data []byte, sid string) {
				idx := binary.BigEndian.Uint64(data[len(data)-8:])
				require.Equal(t, fastDataIndex.Load()+1, idx)
				fastDataIndex.Store(idx)
			})

			// no data should be dropped for slow subscriber that is above threshold
			var slowNoDropDataIndex atomic.Uint64
			var drainSlowSubNotDrop atomic.Bool
			slowNoDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold * 2)
			slowSubNotDrop.SetOnDataReceived(func(data []byte, sid string) {
				idx := binary.BigEndian.Uint64(data[len(data)-8:])
				require.Equal(t, slowNoDropDataIndex.Load()+1, idx)
				slowNoDropDataIndex.Store(idx)
				if !drainSlowSubNotDrop.Load() {
					slowNoDropReader.Read(data, sid)
				}
			})

			// data should be dropped for slow subscriber that is below threshold
			var slowDropDataIndex atomic.Uint64
			dropped := make(chan struct{})
			slowDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold / 2)
			slowSubDrop.SetOnDataReceived(func(data []byte, sid string) {
				select {
				case <-dropped:
					return
				default:
				}
				idx := binary.BigEndian.Uint64(data[len(data)-8:])
				if idx != slowDropDataIndex.Load()+1 {
					close(dropped)
				}
				slowDropDataIndex.Store(idx)
				slowDropReader.Read(data, sid)
			})

			// publisher sends data as fast as possible, it will block by the slowest subscriber above the slow threshold
			var (
				blocked   atomic.Bool
				stopWrite atomic.Bool
				writeIdx  atomic.Uint64
			)
			writeStopped := make(chan struct{})
			go func() {
				defer close(writeStopped)
				var i int
				buf := make([]byte, 100)
				for !stopWrite.Load() {
					i++
					binary.BigEndian.PutUint64(buf[len(buf)-8:], uint64(i))
					if err := pub.PublishData(buf, livekit.DataPacket_RELIABLE); err != nil {
						if errors.Is(err, datachannel.ErrDataDroppedBySlowReader) {
							blocked.Store(true)
							i--
							continue
						} else {
							t.Log("error writing", err)
							break
						}
					}
					writeIdx.Store(uint64(i))
				}
			}()

			<-dropped

			time.Sleep(time.Second)
			blocked.Store(false)
			require.Eventually(t, func() bool { return blocked.Load() }, 30*time.Second, 100*time.Millisecond)
			stopWrite.Store(true)
			<-writeStopped
			drainSlowSubNotDrop.Store(true)
			require.Eventually(t, func() bool {
				return writeIdx.Load() == fastDataIndex.Load() &&
					writeIdx.Load() == slowNoDropDataIndex.Load()
			}, 10*time.Second, 50*time.Millisecond, "writeIdx %d, fast %d, slowNoDrop %d", writeIdx.Load(), fastDataIndex.Load(), slowNoDropDataIndex.Load())
		})
	}
}

func TestFireTrackBySdp(t *testing.T) {
	_, finish := setupSingleNodeTest("TestFireTrackBySdp")
	defer finish()

	var cases = []struct {
		name   string
		codecs []webrtc.RTPCodecCapability
		pubSDK livekit.ClientInfo_SDK
	}{
		{
			name: "js client could pub a/v tracks",
			codecs: []webrtc.RTPCodecCapability{
				{MimeType: mime.MimeTypeH264.String()},
				{MimeType: mime.MimeTypeOpus.String()},
			},
			pubSDK: livekit.ClientInfo_JS,
		},
		{
			name: "go client could pub audio tracks",
			codecs: []webrtc.RTPCodecCapability{
				{MimeType: "audio/opus"},
			},
			pubSDK: livekit.ClientInfo_GO,
		},
	}

	for _, c := range cases {
		codecs, sdk := c.codecs, c.pubSDK
		t.Run(c.name, func(t *testing.T) {
			for _, testRTCServicePath := range testRTCServicePaths {
				t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
					c1 := createRTCClient(c.name+"_c1", defaultServerPort, testRTCServicePath, &testclient.Options{
						ClientInfo: &livekit.ClientInfo{
							Sdk: sdk,
						},
					})
					c2 := createRTCClient(c.name+"_c2", defaultServerPort, testRTCServicePath, &testclient.Options{
						AutoSubscribe: true,
						ClientInfo: &livekit.ClientInfo{
							Sdk: livekit.ClientInfo_JS,
						},
					})
					waitUntilConnected(t, c1, c2)
					defer func() {
						c1.Stop()
						c2.Stop()
					}()

					// publish tracks and don't write any packets
					for _, codec := range codecs {
						_, err := c1.AddStaticTrackWithCodec(codec, codec.MimeType, codec.MimeType, testclient.AddTrackNoWriter())
						require.NoError(t, err)
					}

					require.Eventually(t, func() bool {
						return len(c2.SubscribedTracks()[c1.ID()]) == len(codecs)
					}, 5*time.Second, 10*time.Millisecond)

					var found int
					for _, pubTrack := range c1.GetPublishedTrackIDs() {
						t.Log("pub track", pubTrack)
						tracks := c2.SubscribedTracks()[c1.ID()]
						for _, track := range tracks {
							t.Log("sub track", track.ID(), track.Codec())
							if track.Codec().PayloadType == 0 && track.ID() == pubTrack {
								found++
								break
							}
						}
					}
					require.Equal(t, len(codecs), found)
				})
			}
		})
	}
}

func TestSinglePublisherDataTrack(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	s, finish := setupSingleNodeTest("TestSinglePublisherDataTrack")
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
			waitUntilConnected(t, c1, c2)

			// publish a couple of data tracks and ensure clients receive it ok
			dt1, err := c1.PublishDataTrack()
			require.NoError(t, err)
			defer dt1.Stop()

			dt2, err := c1.PublishDataTrack()
			require.NoError(t, err)
			defer dt2.Stop()

			testutils.WithTimeout(t, func() string {
				if len(c2.SubscribedDataTracks()) == 0 {
					return "c2 was not subscribed to any data tracks"
				}
				// should have received two data tracks
				if len(c2.SubscribedDataTracks()[c1.ID()]) != 2 {
					return "c2 didn't subscribe to both data tracks from c1"
				}
				return ""
			})

			// a new client joins and should get the initial stream
			c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})

			// ensure that new client that has joined also received data tracks
			waitUntilConnected(t, c3)
			testutils.WithTimeout(t, func() string {
				if len(c3.SubscribedDataTracks()) == 0 {
					return "c3 didn't subscribe to any data tracks"
				}
				// should have received two data tracks
				if len(c3.SubscribedDataTracks()[c1.ID()]) != 2 {
					return "c3 didn't subscribe to tracks from c1"
				}
				return ""
			})

			// ensure that the data track ids are generated by server
			tracks := c3.SubscribedDataTracks()[c1.ID()]
			for _, tr := range tracks {
				require.True(t, strings.HasPrefix(string(tr.ID()), "DTR_"), "data track should begin with DTR")
			}

			// when c3 disconnects, ensure subscriber is cleaned up correctly
			c3.Stop()

			testutils.WithTimeout(t, func() string {
				room := s.RoomManager().GetRoom(context.Background(), testRoom)
				p := room.GetParticipant("c1")
				require.NotNil(t, p)

				for _, t := range p.GetPublishedDataTracks() {
					if t.IsSubscriber(c3.ID()) {
						return "c3 was not a subscriber of c1's data tracks"
					}
				}
				return ""
			})
		})
	}
}

func TestTurnRelay(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	testCases := []struct {
		name                     string
		allowRestrictedPeerCIDRs []string
		denyPeerCIDRs            []string
		expectedToConnect        bool
	}{
		{
			"allow",
			[]string{"10.0.0.0/8", "192.168.0.0/16"},
			nil,
			true,
		},
		{
			"not-allowed",
			nil,
			nil,
			false,
		},
		{
			"denied-overrides-allowed",
			[]string{"10.0.0.0/8", "192.168.0.0/16"},
			[]string{"10.0.0.0/8", "192.168.0.0/16"},
			false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			s := createSingleNodeServer(func(c *config.Config) {
				c.TURN.Enabled = true
				c.TURN.UDPPort = 3478
				c.TURN.AllowRestrictedPeerCIDRs = tc.allowRestrictedPeerCIDRs
				c.TURN.DenyPeerCIDRs = tc.denyPeerCIDRs
			})
			go func() {
				if err := s.Start(); err != nil {
					logger.Errorw("server returned error", err)
				}
			}()
			defer s.Stop(true)

			waitForServerToStart(s)

			c1 := createRTCClient("relay_c1", defaultServerPort, testRTCServicePathv0, &testclient.Options{
				AutoSubscribe: true,
				ForceRelay:    true,
			})
			defer c1.Stop()

			if tc.expectedToConnect {
				waitUntilConnected(t, c1)

				testutils.WithTimeout(t, func() string {
					if !c1.IsLocalCandidateRelaySelected() {
						return "expected local candidate to be relay"
					}
					return ""
				})
			} else {
				ensureNotConnected(t, c1)
			}
		})
	}
}

func TestTurnAuthFailure(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	const turnUDPPort = 3478

	s := createSingleNodeServer(func(c *config.Config) {
		c.TURN.Enabled = true
		c.TURN.UDPPort = turnUDPPort
	})
	go func() {
		if err := s.Start(); err != nil {
			logger.Errorw("server returned error", err)
		}
	}()
	defer s.Stop(true)

	waitForServerToStart(s)

	// build a known-good username/password pair so individual cases can mutate
	// only the part they are exercising.
	pID := livekit.ParticipantID("PA_authfail")
	authHandler := service.NewTURNAuthHandler(auth.NewSimpleKeyProvider(testApiKey, testApiSecret))
	validUsername, validExpiry := authHandler.CreateUsername(testApiKey, pID, 300)
	validPassword, err := authHandler.CreatePassword(testApiKey, pID, validExpiry)
	require.NoError(t, err)

	// username encoded with an already-expired timestamp; constructed directly
	// because CreateUsername floors non-positive TTLs to the default.
	expiredUsername := base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", testApiKey, pID, time.Now().Add(-time.Minute).Unix()))

	// username encoded with an api key the server does not know about.
	unknownAPIKeyUsername, _ := authHandler.CreateUsername("unknown-api-key", pID, 300)

	// password whose hash was generated for an expiry that doesn't match the
	// one encoded in the username. The server reconstructs the password using
	// the username's expiry, so the integrity check fails.
	mismatchedExpiryPassword, err := authHandler.CreatePassword(testApiKey, pID, validExpiry+60)
	require.NoError(t, err)
	require.NotEqual(t, validPassword, mismatchedExpiryPassword)

	// username carrying expiry=0 must be rejected outright; constructed
	// directly because CreateUsername always stamps a real expiry.
	zeroExpiryUsername := base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", testApiKey, pID, 0))

	// username with only apiKey|pID (no expiry component) is the legacy
	// pre-expiry form and must be rejected.
	twoPartUsername := base62.EncodeToString(fmt.Appendf(nil, "%s|%s", testApiKey, pID))

	testCases := []struct {
		name     string
		username string
		password string
	}{
		{
			name:     "unparseable-username",
			username: "not-base62!!!",
			password: validPassword,
		},
		{
			name:     "wrong-password",
			username: validUsername,
			password: "wrongpassword",
		},
		{
			name:     "expired-username",
			username: expiredUsername,
			password: validPassword,
		},
		{
			name:     "unknown-api-key",
			username: unknownAPIKeyUsername,
			password: validPassword,
		},
		{
			name:     "password-expiry-mismatch",
			username: validUsername,
			password: mismatchedExpiryPassword,
		},
		{
			name:     "zero-expiry-username",
			username: zeroExpiryUsername,
			password: validPassword,
		},
		{
			name:     "two-part-username",
			username: twoPartUsername,
			password: validPassword,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			conn, err := net.ListenPacket("udp4", "0.0.0.0:0")
			require.NoError(t, err)
			defer conn.Close()

			client, err := turn.NewClient(&turn.ClientConfig{
				TURNServerAddr: fmt.Sprintf("127.0.0.1:%d", turnUDPPort),
				Username:       tc.username,
				Password:       tc.password,
				Realm:          service.LivekitRealm,
				Conn:           conn,
			})
			require.NoError(t, err)
			defer client.Close()
			require.NoError(t, client.Listen())

			_, allocErr := client.Allocate()
			require.Error(t, allocErr)

			// pion's TURN server replies with 400 Bad Request for any
			// authenticated-allocate failure (unknown user or integrity check
			// mismatch); the initial unauthenticated probe is what returns 401.
			var turnErr *stun.TurnError
			require.ErrorAs(t, allocErr, &turnErr)
			require.Equal(t, stun.CodeBadRequest, turnErr.ErrorCodeAttr.Code)
		})
	}
}

// dataBlobCapture buffers RequestResponse, StoreDataBlobResponse, and GetDataBlobResponse messages
// sent to a test client so they can be asserted on. Other messages flow through to the
// default handler.
type dataBlobCapture struct {
	mu               sync.Mutex
	requestResponses []*livekit.RequestResponse
	storeResponses   []*livekit.StoreDataBlobResponse
	blobResponses    []*livekit.GetDataBlobResponse
}

func (c *dataBlobCapture) interceptor() testclient.SignalResponseInterceptor {
	return func(msg *livekit.SignalResponse, next testclient.SignalResponseHandler) error {
		switch m := msg.Message.(type) {
		case *livekit.SignalResponse_RequestResponse:
			c.mu.Lock()
			c.requestResponses = append(c.requestResponses, m.RequestResponse)
			c.mu.Unlock()
		case *livekit.SignalResponse_StoreDataBlobResponse:
			c.mu.Lock()
			c.storeResponses = append(c.storeResponses, m.StoreDataBlobResponse)
			c.mu.Unlock()
		case *livekit.SignalResponse_GetDataBlobResponse:
			c.mu.Lock()
			c.blobResponses = append(c.blobResponses, m.GetDataBlobResponse)
			c.mu.Unlock()
		}
		return next(msg)
	}
}

func (c *dataBlobCapture) takeRequestResponse() *livekit.RequestResponse {
	c.mu.Lock()
	defer c.mu.Unlock()
	if len(c.requestResponses) == 0 {
		return nil
	}
	rr := c.requestResponses[0]
	c.requestResponses = c.requestResponses[1:]
	return rr
}

func (c *dataBlobCapture) takeStoreResponse() *livekit.StoreDataBlobResponse {
	c.mu.Lock()
	defer c.mu.Unlock()
	if len(c.storeResponses) == 0 {
		return nil
	}
	sr := c.storeResponses[0]
	c.storeResponses = c.storeResponses[1:]
	return sr
}

func (c *dataBlobCapture) takeBlobResponse() *livekit.GetDataBlobResponse {
	c.mu.Lock()
	defer c.mu.Unlock()
	if len(c.blobResponses) == 0 {
		return nil
	}
	sr := c.blobResponses[0]
	c.blobResponses = c.blobResponses[1:]
	return sr
}

func (c *dataBlobCapture) requestResponseCount() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return len(c.requestResponses)
}

func setupDataBlobServer(t *testing.T, name string, enable bool) (*service.LivekitServer, func()) {
	logger.Infow("----------------STARTING TEST----------------", "test", name)
	s := createSingleNodeServer(func(c *config.Config) {
		c.EnableParticipantDataBlob = enable
		c.Limit.MaxDataBlobSize = 1024
	})
	go func() {
		if err := s.Start(); err != nil {
			logger.Errorw("server returned error", err)
		}
	}()
	waitForServerToStart(s)
	return s, func() {
		s.Stop(true)
		logger.Infow("----------------FINISHING TEST----------------", "test", name)
	}
}

func TestSingleNodeDataBlob(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupDataBlobServer(t, "TestSingleNodeDataBlob", true)
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			pubCapture := &dataBlobCapture{}
			subCapture := &dataBlobCapture{}

			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, &testclient.Options{
				AutoSubscribe:             true,
				SignalResponseInterceptor: pubCapture.interceptor(),
			})
			sub := createRTCClient("sub", defaultServerPort, testRTCServicePath, &testclient.Options{
				AutoSubscribe:             true,
				SignalResponseInterceptor: subCapture.interceptor(),
			})
			waitUntilConnected(t, pub, sub)
			defer stopClients(pub, sub)

			key := &livekit.DataBlobKey{
				Key: &livekit.DataBlobKey_Generic{
					Generic: "blob-1",
				},
			}
			contents := []byte("definition-bytes")

			// publisher stores a blob
			require.NoError(t, pub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_StoreDataBlobRequest{
					StoreDataBlobRequest: &livekit.StoreDataBlobRequest{
						RequestId: 1,
						Blob: &livekit.DataBlob{
							Key:      key,
							Contents: contents,
						},
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				resp := pubCapture.takeStoreResponse()
				if resp == nil {
					return "publisher did not receive store response"
				}
				if resp.RequestId != 1 {
					return fmt.Sprintf("expected store response request id 1, got %d", resp.RequestId)
				}
				if resp.Key == nil {
					return "store response missing key"
				}
				if resp.Key.String() != key.String() {
					return fmt.Sprintf("expected stored blob key %s, got %s", key.String(), resp.Key.String())
				}
				return ""
			})
			require.Equal(t, 0, pubCapture.requestResponseCount(), "publisher should not receive an error response on success")

			// subscriber asks for the blob
			require.NoError(t, sub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_GetDataBlobRequest{
					GetDataBlobRequest: &livekit.GetDataBlobRequest{
						ParticipantIdentity: "pub",
						Key:                 key,
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				resp := subCapture.takeBlobResponse()
				if resp == nil {
					return "subscriber did not receive blob response"
				}
				if resp.Blob == nil {
					return "blob response missing blob"
				}
				if resp.Blob.Key.String() != key.String() {
					return fmt.Sprintf("expected blob key %s, got %s", key.String(), resp.Blob.Key.String())
				}
				if string(resp.Blob.Contents) != string(contents) {
					return fmt.Sprintf("expected contents %q, got %q", contents, resp.Blob.Contents)
				}
				return ""
			})

			// subscriber asks for an unknown blob on a known publisher
			require.NoError(t, sub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_GetDataBlobRequest{
					GetDataBlobRequest: &livekit.GetDataBlobRequest{
						ParticipantIdentity: "pub",
						Key: &livekit.DataBlobKey{
							Key: &livekit.DataBlobKey_Generic{
								Generic: "does-not-exist",
							},
						},
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				rr := subCapture.takeRequestResponse()
				if rr == nil {
					return "subscriber did not receive RequestResponse for missing blob"
				}
				if rr.Reason != livekit.RequestResponse_NOT_FOUND {
					return fmt.Sprintf("expected NOT_FOUND, got %s", rr.Reason)
				}
				return ""
			})

			// subscriber asks for a blob on an unknown publisher identity
			require.NoError(t, sub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_GetDataBlobRequest{
					GetDataBlobRequest: &livekit.GetDataBlobRequest{
						ParticipantIdentity: "unknown-publisher",
						Key:                 key,
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				rr := subCapture.takeRequestResponse()
				if rr == nil {
					return "subscriber did not receive RequestResponse for unknown publisher"
				}
				if rr.Reason != livekit.RequestResponse_NOT_FOUND {
					return fmt.Sprintf("expected NOT_FOUND, got %s", rr.Reason)
				}
				return ""
			})

			// publisher sends an invalid blob (empty key)
			require.NoError(t, pub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_StoreDataBlobRequest{
					StoreDataBlobRequest: &livekit.StoreDataBlobRequest{
						Blob: &livekit.DataBlob{
							Contents: contents,
						},
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				rr := pubCapture.takeRequestResponse()
				if rr == nil {
					return "publisher did not receive RequestResponse for invalid define"
				}
				if rr.Reason != livekit.RequestResponse_INVALID_REQUEST {
					return fmt.Sprintf("expected INVALID_REQUEST, got %s", rr.Reason)
				}
				return ""
			})
		})
	}
}

func TestSingleNodeDataBlobDisabled(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
		return
	}

	_, finish := setupDataBlobServer(t, "TestSingleNodeDataBlobDisabled", false)
	defer finish()

	for _, testRTCServicePath := range testRTCServicePaths {
		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
			pubCapture := &dataBlobCapture{}
			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, &testclient.Options{
				AutoSubscribe:             true,
				SignalResponseInterceptor: pubCapture.interceptor(),
			})
			waitUntilConnected(t, pub)
			defer stopClients(pub)

			require.NoError(t, pub.SendRequest(&livekit.SignalRequest{
				Message: &livekit.SignalRequest_StoreDataBlobRequest{
					StoreDataBlobRequest: &livekit.StoreDataBlobRequest{
						Blob: &livekit.DataBlob{
							Key: &livekit.DataBlobKey{
								Key: &livekit.DataBlobKey_Generic{
									Generic: "blob-1",
								},
							},
							Contents: []byte("definition-bytes"),
						},
					},
				},
			}))

			testutils.WithTimeout(t, func() string {
				rr := pubCapture.takeRequestResponse()
				if rr == nil {
					return "publisher did not receive RequestResponse"
				}
				if rr.Reason != livekit.RequestResponse_NOT_ALLOWED {
					return fmt.Sprintf("expected NOT_ALLOWED, got %s", rr.Reason)
				}
				return ""
			})
		})
	}
}