livekit/pkg/agent/agent_test.go

package agent_test

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"github.com/livekit/livekit-server/pkg/agent"
	"github.com/livekit/livekit-server/pkg/agent/testutil"
	"github.com/livekit/protocol/livekit"
	"github.com/livekit/protocol/rpc"
	"github.com/livekit/protocol/utils/guid"
	"github.com/livekit/protocol/utils/must"
	"github.com/livekit/psrpc"
)

func TestAgent(t *testing.T) {
	t.Run("dispatched jobs are assigned to a worker", func(t *testing.T) {
		bus := psrpc.NewLocalMessageBus()

		client := must.Get(rpc.NewAgentInternalClient(bus))
		server := testutil.NewTestServer(bus)
		t.Cleanup(server.Close)

		worker := server.SimulateAgentWorker()
		worker.Register("", "test", livekit.JobType_JT_ROOM)
		jobAssignments := worker.JobAssignments.Observe()

		job := &livekit.Job{
			Id:         guid.New(guid.AgentJobPrefix),
			DispatchId: guid.New(guid.AgentDispatchPrefix),
			Type:       livekit.JobType_JT_ROOM,
			Room:       &livekit.Room{},
			Namespace:  "test",
		}
		_, err := client.JobRequest(context.Background(), "test", agent.RoomAgentTopic, job)
		require.NoError(t, err)

		select {
		case a := <-jobAssignments.Events():
			require.EqualValues(t, job.Id, a.Job.Id)
		case <-time.After(time.Second):
			require.Fail(t, "job assignment timeout")
		}
	})
}

func TestAgentLoadBalancing(t *testing.T) {

	batchJobCreate := func(wg *sync.WaitGroup, batchSize int, totalJobs int, client rpc.AgentInternalClient) {
		for i := 0; i < totalJobs; i += batchSize {
			wg.Add(1)
			go func(start int) {
				defer wg.Done()
				for j := start; j < start+batchSize && j < totalJobs; j++ {
					job := &livekit.Job{
						Id:         guid.New(guid.AgentJobPrefix),
						DispatchId: guid.New(guid.AgentDispatchPrefix),
						Type:       livekit.JobType_JT_ROOM,
						Room:       &livekit.Room{},
						Namespace:  "test",
					}
					_, err := client.JobRequest(context.Background(), "test", agent.RoomAgentTopic, job)
					require.NoError(t, err)
				}
			}(i)
		}
	}

	t.Run("jobs are distributed normally with baseline worker load", func(t *testing.T) {
		totalWorkers := 5
		totalJobs := 100

		bus := psrpc.NewLocalMessageBus()

		client := must.Get(rpc.NewAgentInternalClient(bus))
		server := testutil.NewTestServer(bus)
		t.Cleanup(server.Close)

		agents := make([]*testutil.AgentWorker, totalWorkers)
		for i := 0; i < totalWorkers; i++ {
			agents[i] = server.SimulateAgentWorker()
			agents[i].Register(fmt.Sprintf("agent-%d", i), "test", livekit.JobType_JT_ROOM)
		}

		jobAssignments := make(chan *livekit.Job, totalJobs)
		for i := 0; i < totalWorkers; i++ {
			worker := agents[i]
			go func() {
				for a := range worker.JobAssignments.Observe().Events() {
					jobAssignments <- a.Job
				}
			}()
		}

		var wg sync.WaitGroup
		batchJobCreate(&wg, 10, totalJobs, client)
		wg.Wait()

		jobCount := make(map[string]int)
		for i := 0; i < totalJobs; i++ {
			select {
			case job := <-jobAssignments:
				jobCount[job.AgentName]++
			case <-time.After(time.Second):
				require.Fail(t, "job assignment timeout")
			}
		}

		assignedJobs := 0
		// check that jobs are distributed normally
		for i := 0; i < totalWorkers; i++ {
			agentName := fmt.Sprintf("agent-%d", i)
			assignedJobs += jobCount[agentName]
			require.GreaterOrEqual(t, jobCount[agentName], 0)
			require.Less(t, jobCount[agentName], 35) // three std deviations from the mean is 32
		}

		// ensure all jobs are assigned
		require.Equal(t, 100, assignedJobs)
	})

	t.Run("jobs are distributed with variable and overloaded worker load", func(t *testing.T) {
		totalWorkers := 4
		totalJobs := 15

		bus := psrpc.NewLocalMessageBus()

		client := must.Get(rpc.NewAgentInternalClient(bus))
		server := testutil.NewTestServer(bus)
		t.Cleanup(server.Close)

		agents := make([]*testutil.AgentWorker, totalWorkers)
		for i := 0; i < totalWorkers; i++ {
			if i%2 == 0 {
				// make sure we have some workers that can accept jobs
				agents[i] = server.SimulateAgentWorker()
			} else {
				agents[i] = server.SimulateAgentWorker(testutil.WithDefaultWorkerLoad(0.9))
			}
			agents[i].Register(fmt.Sprintf("agent-%d", i), "test", livekit.JobType_JT_ROOM)
		}

		jobAssignments := make(chan *livekit.Job, totalJobs)
		for i := 0; i < totalWorkers; i++ {
			worker := agents[i]
			go func() {
				for a := range worker.JobAssignments.Observe().Events() {
					jobAssignments <- a.Job
				}
			}()
		}

		var wg sync.WaitGroup
		batchJobCreate(&wg, 1, totalJobs, client)
		wg.Wait()

		jobCount := make(map[string]int)
		for i := 0; i < totalJobs; i++ {
			select {
			case job := <-jobAssignments:
				jobCount[job.AgentName]++
			case <-time.After(time.Second):
				require.Fail(t, "job assignment timeout")
			}
		}

		assignedJobs := 0
		for i := 0; i < totalWorkers; i++ {
			agentName := fmt.Sprintf("agent-%d", i)
			assignedJobs += jobCount[agentName]

			if i%2 == 0 {
				require.GreaterOrEqual(t, jobCount[agentName], 2)
			} else {
				require.Equal(t, 0, jobCount[agentName])
			}
			require.GreaterOrEqual(t, jobCount[agentName], 0)
		}

		// ensure all jobs are assigned
		require.Equal(t, 15, assignedJobs)
	})
}