Merge remote-tracking branch 'origin/master' into raja_min_packets

pkg/service/egress.go

@@ -23,7 +23,6 @@ import (
 	"github.com/twitchtv/twirp"
 
 	"github.com/livekit/livekit-server/pkg/rtc"
-	"github.com/livekit/livekit-server/pkg/telemetry"
 	"github.com/livekit/protocol/egress"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
@@ -33,48 +32,39 @@ import (
 
 type EgressService struct {
 	client      rpc.EgressClient
-	store       ServiceStore
-	es          EgressStore
+	io          IOClient
 	roomService livekit.RoomService
-	telemetry   telemetry.TelemetryService
+	store       ServiceStore
 	launcher    rtc.EgressLauncher
 }
 
 type egressLauncher struct {
-	client    rpc.EgressClient
-	es        EgressStore
-	telemetry telemetry.TelemetryService
+	client rpc.EgressClient
+	io     IOClient
 }
 
-func NewEgressLauncher(
-	client rpc.EgressClient,
-	es EgressStore,
-	ts telemetry.TelemetryService) rtc.EgressLauncher {
+func NewEgressLauncher(client rpc.EgressClient, io IOClient) rtc.EgressLauncher {
 	if client == nil {
 		return nil
 	}
-
 	return &egressLauncher{
-		client:    client,
-		es:        es,
-		telemetry: ts,
+		client: client,
+		io:     io,
 	}
 }
 
 func NewEgressService(
 	client rpc.EgressClient,
 	store ServiceStore,
-	es EgressStore,
+	io IOClient,
 	rs livekit.RoomService,
-	ts telemetry.TelemetryService,
 	launcher rtc.EgressLauncher,
 ) *EgressService {
 	return &EgressService{
 		client:      client,
 		store:       store,
-		es:          es,
+		io:          io,
 		roomService: rs,
-		telemetry:   ts,
 		launcher:    launcher,
 	}
 }
@@ -189,7 +179,6 @@ func (s *EgressService) startEgress(ctx context.Context, roomName livekit.RoomNa
 	} else if s.launcher == nil {
 		return nil, ErrEgressNotConnected
 	}
-
 	if roomName != "" {
 		room, _, err := s.store.LoadRoom(ctx, roomName, false)
 		if err != nil {
@@ -197,14 +186,18 @@ func (s *EgressService) startEgress(ctx context.Context, roomName livekit.RoomNa
 		}
 		req.RoomId = room.Sid
 	}
-
 	return s.launcher.StartEgress(ctx, req)
 }
 
 func (s *egressLauncher) StartEgress(ctx context.Context, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) {
 	return s.StartEgressWithClusterId(ctx, "", req)
 }
+
 func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId string, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) {
+	if s.client == nil {
+		return nil, ErrEgressNotConnected
+	}
+
 	// Ensure we have an Egress ID
 	if req.EgressId == "" {
 		req.EgressId = utils.NewGuid(utils.EgressPrefix)
@@ -215,12 +208,10 @@ func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId
 		return nil, err
 	}
 
-	s.telemetry.EgressStarted(ctx, info)
-	go func() {
-		if err := s.es.StoreEgress(ctx, info); err != nil {
-			logger.Errorw("could not write egress info", err)
-		}
-	}()
+	_, err = s.io.CreateEgress(ctx, info)
+	if err != nil {
+		logger.Errorw("failed to create egress", err)
+	}
 
 	return info, nil
 }
@@ -234,11 +225,8 @@ func (s *EgressService) UpdateLayout(ctx context.Context, req *livekit.UpdateLay
 	if err := EnsureRecordPermission(ctx); err != nil {
 		return nil, twirpAuthError(err)
 	}
-	if s.client == nil {
-		return nil, ErrEgressNotConnected
-	}
 
-	info, err := s.es.LoadEgress(ctx, req.EgressId)
+	info, err := s.io.GetEgress(ctx, &rpc.GetEgressRequest{EgressId: req.EgressId})
 	if err != nil {
 		return nil, err
 	}
@@ -277,7 +265,7 @@ func (s *EgressService) UpdateStream(ctx context.Context, req *livekit.UpdateStr
 	info, err := s.client.UpdateStream(ctx, req.EgressId, req)
 	if err != nil {
 		var loadErr error
-		info, loadErr = s.es.LoadEgress(ctx, req.EgressId)
+		info, loadErr = s.io.GetEgress(ctx, &rpc.GetEgressRequest{EgressId: req.EgressId})
 		if loadErr != nil {
 			return nil, loadErr
 		}
@@ -306,29 +294,7 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR
 	if err := EnsureRecordPermission(ctx); err != nil {
 		return nil, twirpAuthError(err)
 	}
-	if s.client == nil {
-		return nil, ErrEgressNotConnected
-	}
-
-	var items []*livekit.EgressInfo
-	if req.EgressId != "" {
-		info, err := s.es.LoadEgress(ctx, req.EgressId)
-		if err != nil {
-			return nil, err
-		}
-
-		if !req.Active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) {
-			items = []*livekit.EgressInfo{info}
-		}
-	} else {
-		var err error
-		items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName), req.Active)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return &livekit.ListEgressResponse{Items: items}, nil
+	return s.io.ListEgress(ctx, req)
 }
 
 func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressRequest) (*livekit.EgressInfo, error) {
@@ -344,7 +310,7 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR
 	info, err := s.client.StopEgress(ctx, req.EgressId, req)
 	if err != nil {
 		var loadErr error
-		info, loadErr = s.es.LoadEgress(ctx, req.EgressId)
+		info, loadErr = s.io.GetEgress(ctx, &rpc.GetEgressRequest{EgressId: req.EgressId})
 		if loadErr != nil {
 			return nil, loadErr
 		}
@@ -359,11 +325,5 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR
 		}
 	}
 
-	go func() {
-		if err := s.es.UpdateEgress(ctx, info); err != nil {
-			logger.Errorw("could not write egress info", err)
-		}
-	}()
-
 	return info, nil
 }

pkg/service/interfaces.go

@@ -18,7 +18,10 @@ import (
 	"context"
 	"time"
 
+	"google.golang.org/protobuf/types/known/emptypb"
+
 	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/rpc"
 )
 
 //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
@@ -71,6 +74,13 @@ type IngressStore interface {
 	DeleteIngress(ctx context.Context, info *livekit.IngressInfo) error
 }
 
+//counterfeiter:generate . IOClient
+type IOClient interface {
+	CreateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error)
+	GetEgress(ctx context.Context, req *rpc.GetEgressRequest) (*livekit.EgressInfo, error)
+	ListEgress(ctx context.Context, req *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error)
+}
+
 //counterfeiter:generate . RoomAllocator
 type RoomAllocator interface {
 	CreateRoom(ctx context.Context, req *livekit.CreateRoomRequest) (*livekit.Room, error)

pkg/service/ioservice.go

@@ -28,11 +28,13 @@ import (
 )
 
 type IOInfoService struct {
-	ioServer  rpc.IOInfoServer
+	ioServer rpc.IOInfoServer
+
 	es        EgressStore
 	is        IngressStore
 	telemetry telemetry.TelemetryService
-	shutdown  chan struct{}
+
+	shutdown chan struct{}
 }
 
 func NewIOInfoService(
@@ -73,27 +75,38 @@ func (s *IOInfoService) Start() error {
 	return nil
 }
 
-func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) {
+func (s *IOInfoService) CreateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) {
+	// check if egress already exists to avoid duplicate EgressStarted event
+	if _, err := s.es.LoadEgress(ctx, info.EgressId); err == nil {
+		return &emptypb.Empty{}, nil
+	}
+
+	err := s.es.StoreEgress(ctx, info)
+	if err != nil {
+		logger.Errorw("could not update egress", err)
+		return nil, err
+	}
+
+	s.telemetry.EgressStarted(ctx, info)
+
+	return &emptypb.Empty{}, nil
+}
+
+func (s *IOInfoService) UpdateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) {
 	err := s.es.UpdateEgress(ctx, info)
 
 	switch info.Status {
-	case livekit.EgressStatus_EGRESS_ACTIVE:
+	case livekit.EgressStatus_EGRESS_ACTIVE,
+		livekit.EgressStatus_EGRESS_ENDING:
 		s.telemetry.EgressUpdated(ctx, info)
 
 	case livekit.EgressStatus_EGRESS_COMPLETE,
 		livekit.EgressStatus_EGRESS_FAILED,
 		livekit.EgressStatus_EGRESS_ABORTED,
 		livekit.EgressStatus_EGRESS_LIMIT_REACHED:
-
-		// log results
-		if info.Error != "" {
-			logger.Errorw("egress failed", errors.New(info.Error), "egressID", info.EgressId)
-		} else {
-			logger.Infow("egress ended", "egressID", info.EgressId)
-		}
-
 		s.telemetry.EgressEnded(ctx, info)
 	}
+
 	if err != nil {
 		logger.Errorw("could not update egress", err)
 		return nil, err
@@ -102,6 +115,40 @@ func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.Egre
 	return &emptypb.Empty{}, nil
 }
 
+func (s *IOInfoService) GetEgress(ctx context.Context, req *rpc.GetEgressRequest) (*livekit.EgressInfo, error) {
+	info, err := s.es.LoadEgress(ctx, req.EgressId)
+	if err != nil {
+		logger.Errorw("failed to load egress", err)
+		return nil, err
+	}
+
+	return info, nil
+}
+
+func (s *IOInfoService) ListEgress(ctx context.Context, req *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error) {
+	var items []*livekit.EgressInfo
+	if req.EgressId != "" {
+		info, err := s.es.LoadEgress(ctx, req.EgressId)
+		if err != nil {
+			logger.Errorw("failed to load egress", err)
+			return nil, err
+		}
+
+		if !req.Active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) {
+			items = []*livekit.EgressInfo{info}
+		}
+	} else {
+		var err error
+		items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName), req.Active)
+		if err != nil {
+			logger.Errorw("failed to list egress", err)
+			return nil, err
+		}
+	}
+
+	return &livekit.ListEgressResponse{Items: items}, nil
+}
+
 func (s *IOInfoService) GetIngressInfo(ctx context.Context, req *rpc.GetIngressInfoRequest) (*rpc.GetIngressInfoResponse, error) {
 	info, err := s.loadIngressFromInfoRequest(req)
 	if err != nil {
@@ -128,7 +175,7 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI
 		return nil, err
 	}
 
-	if err := s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil {
+	if err = s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil {
 		logger.Errorw("could not update ingress", err)
 		return nil, err
 	}
@@ -176,3 +223,33 @@ func (s *IOInfoService) Stop() {
 		s.ioServer.Shutdown()
 	}
 }
+
+// deprecated
+func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) {
+	err := s.es.UpdateEgress(ctx, info)
+
+	switch info.Status {
+	case livekit.EgressStatus_EGRESS_ACTIVE:
+		s.telemetry.EgressUpdated(ctx, info)
+
+	case livekit.EgressStatus_EGRESS_COMPLETE,
+		livekit.EgressStatus_EGRESS_FAILED,
+		livekit.EgressStatus_EGRESS_ABORTED,
+		livekit.EgressStatus_EGRESS_LIMIT_REACHED:
+
+		// log results
+		if info.Error != "" {
+			logger.Errorw("egress failed", errors.New(info.Error), "egressID", info.EgressId)
+		} else {
+			logger.Infow("egress ended", "egressID", info.EgressId)
+		}
+
+		s.telemetry.EgressEnded(ctx, info)
+	}
+	if err != nil {
+		logger.Errorw("could not update egress", err)
+		return nil, err
+	}
+
+	return &emptypb.Empty{}, nil
+}

pkg/service/servicefakes/fake_ioclient.go

@@ -0,0 +1,284 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package servicefakes
+
+import (
+	"context"
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/service"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/rpc"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+type FakeIOClient struct {
+	CreateEgressStub        func(context.Context, *livekit.EgressInfo) (*emptypb.Empty, error)
+	createEgressMutex       sync.RWMutex
+	createEgressArgsForCall []struct {
+		arg1 context.Context
+		arg2 *livekit.EgressInfo
+	}
+	createEgressReturns struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
+	createEgressReturnsOnCall map[int]struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
+	GetEgressStub        func(context.Context, *rpc.GetEgressRequest) (*livekit.EgressInfo, error)
+	getEgressMutex       sync.RWMutex
+	getEgressArgsForCall []struct {
+		arg1 context.Context
+		arg2 *rpc.GetEgressRequest
+	}
+	getEgressReturns struct {
+		result1 *livekit.EgressInfo
+		result2 error
+	}
+	getEgressReturnsOnCall map[int]struct {
+		result1 *livekit.EgressInfo
+		result2 error
+	}
+	ListEgressStub        func(context.Context, *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error)
+	listEgressMutex       sync.RWMutex
+	listEgressArgsForCall []struct {
+		arg1 context.Context
+		arg2 *livekit.ListEgressRequest
+	}
+	listEgressReturns struct {
+		result1 *livekit.ListEgressResponse
+		result2 error
+	}
+	listEgressReturnsOnCall map[int]struct {
+		result1 *livekit.ListEgressResponse
+		result2 error
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeIOClient) CreateEgress(arg1 context.Context, arg2 *livekit.EgressInfo) (*emptypb.Empty, error) {
+	fake.createEgressMutex.Lock()
+	ret, specificReturn := fake.createEgressReturnsOnCall[len(fake.createEgressArgsForCall)]
+	fake.createEgressArgsForCall = append(fake.createEgressArgsForCall, struct {
+		arg1 context.Context
+		arg2 *livekit.EgressInfo
+	}{arg1, arg2})
+	stub := fake.CreateEgressStub
+	fakeReturns := fake.createEgressReturns
+	fake.recordInvocation("CreateEgress", []interface{}{arg1, arg2})
+	fake.createEgressMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1, ret.result2
+	}
+	return fakeReturns.result1, fakeReturns.result2
+}
+
+func (fake *FakeIOClient) CreateEgressCallCount() int {
+	fake.createEgressMutex.RLock()
+	defer fake.createEgressMutex.RUnlock()
+	return len(fake.createEgressArgsForCall)
+}
+
+func (fake *FakeIOClient) CreateEgressCalls(stub func(context.Context, *livekit.EgressInfo) (*emptypb.Empty, error)) {
+	fake.createEgressMutex.Lock()
+	defer fake.createEgressMutex.Unlock()
+	fake.CreateEgressStub = stub
+}
+
+func (fake *FakeIOClient) CreateEgressArgsForCall(i int) (context.Context, *livekit.EgressInfo) {
+	fake.createEgressMutex.RLock()
+	defer fake.createEgressMutex.RUnlock()
+	argsForCall := fake.createEgressArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeIOClient) CreateEgressReturns(result1 *emptypb.Empty, result2 error) {
+	fake.createEgressMutex.Lock()
+	defer fake.createEgressMutex.Unlock()
+	fake.CreateEgressStub = nil
+	fake.createEgressReturns = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) CreateEgressReturnsOnCall(i int, result1 *emptypb.Empty, result2 error) {
+	fake.createEgressMutex.Lock()
+	defer fake.createEgressMutex.Unlock()
+	fake.CreateEgressStub = nil
+	if fake.createEgressReturnsOnCall == nil {
+		fake.createEgressReturnsOnCall = make(map[int]struct {
+			result1 *emptypb.Empty
+			result2 error
+		})
+	}
+	fake.createEgressReturnsOnCall[i] = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) GetEgress(arg1 context.Context, arg2 *rpc.GetEgressRequest) (*livekit.EgressInfo, error) {
+	fake.getEgressMutex.Lock()
+	ret, specificReturn := fake.getEgressReturnsOnCall[len(fake.getEgressArgsForCall)]
+	fake.getEgressArgsForCall = append(fake.getEgressArgsForCall, struct {
+		arg1 context.Context
+		arg2 *rpc.GetEgressRequest
+	}{arg1, arg2})
+	stub := fake.GetEgressStub
+	fakeReturns := fake.getEgressReturns
+	fake.recordInvocation("GetEgress", []interface{}{arg1, arg2})
+	fake.getEgressMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1, ret.result2
+	}
+	return fakeReturns.result1, fakeReturns.result2
+}
+
+func (fake *FakeIOClient) GetEgressCallCount() int {
+	fake.getEgressMutex.RLock()
+	defer fake.getEgressMutex.RUnlock()
+	return len(fake.getEgressArgsForCall)
+}
+
+func (fake *FakeIOClient) GetEgressCalls(stub func(context.Context, *rpc.GetEgressRequest) (*livekit.EgressInfo, error)) {
+	fake.getEgressMutex.Lock()
+	defer fake.getEgressMutex.Unlock()
+	fake.GetEgressStub = stub
+}
+
+func (fake *FakeIOClient) GetEgressArgsForCall(i int) (context.Context, *rpc.GetEgressRequest) {
+	fake.getEgressMutex.RLock()
+	defer fake.getEgressMutex.RUnlock()
+	argsForCall := fake.getEgressArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeIOClient) GetEgressReturns(result1 *livekit.EgressInfo, result2 error) {
+	fake.getEgressMutex.Lock()
+	defer fake.getEgressMutex.Unlock()
+	fake.GetEgressStub = nil
+	fake.getEgressReturns = struct {
+		result1 *livekit.EgressInfo
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) GetEgressReturnsOnCall(i int, result1 *livekit.EgressInfo, result2 error) {
+	fake.getEgressMutex.Lock()
+	defer fake.getEgressMutex.Unlock()
+	fake.GetEgressStub = nil
+	if fake.getEgressReturnsOnCall == nil {
+		fake.getEgressReturnsOnCall = make(map[int]struct {
+			result1 *livekit.EgressInfo
+			result2 error
+		})
+	}
+	fake.getEgressReturnsOnCall[i] = struct {
+		result1 *livekit.EgressInfo
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) ListEgress(arg1 context.Context, arg2 *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error) {
+	fake.listEgressMutex.Lock()
+	ret, specificReturn := fake.listEgressReturnsOnCall[len(fake.listEgressArgsForCall)]
+	fake.listEgressArgsForCall = append(fake.listEgressArgsForCall, struct {
+		arg1 context.Context
+		arg2 *livekit.ListEgressRequest
+	}{arg1, arg2})
+	stub := fake.ListEgressStub
+	fakeReturns := fake.listEgressReturns
+	fake.recordInvocation("ListEgress", []interface{}{arg1, arg2})
+	fake.listEgressMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1, ret.result2
+	}
+	return fakeReturns.result1, fakeReturns.result2
+}
+
+func (fake *FakeIOClient) ListEgressCallCount() int {
+	fake.listEgressMutex.RLock()
+	defer fake.listEgressMutex.RUnlock()
+	return len(fake.listEgressArgsForCall)
+}
+
+func (fake *FakeIOClient) ListEgressCalls(stub func(context.Context, *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error)) {
+	fake.listEgressMutex.Lock()
+	defer fake.listEgressMutex.Unlock()
+	fake.ListEgressStub = stub
+}
+
+func (fake *FakeIOClient) ListEgressArgsForCall(i int) (context.Context, *livekit.ListEgressRequest) {
+	fake.listEgressMutex.RLock()
+	defer fake.listEgressMutex.RUnlock()
+	argsForCall := fake.listEgressArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeIOClient) ListEgressReturns(result1 *livekit.ListEgressResponse, result2 error) {
+	fake.listEgressMutex.Lock()
+	defer fake.listEgressMutex.Unlock()
+	fake.ListEgressStub = nil
+	fake.listEgressReturns = struct {
+		result1 *livekit.ListEgressResponse
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) ListEgressReturnsOnCall(i int, result1 *livekit.ListEgressResponse, result2 error) {
+	fake.listEgressMutex.Lock()
+	defer fake.listEgressMutex.Unlock()
+	fake.ListEgressStub = nil
+	if fake.listEgressReturnsOnCall == nil {
+		fake.listEgressReturnsOnCall = make(map[int]struct {
+			result1 *livekit.ListEgressResponse
+			result2 error
+		})
+	}
+	fake.listEgressReturnsOnCall[i] = struct {
+		result1 *livekit.ListEgressResponse
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	fake.createEgressMutex.RLock()
+	defer fake.createEgressMutex.RUnlock()
+	fake.getEgressMutex.RLock()
+	defer fake.getEgressMutex.RUnlock()
+	fake.listEgressMutex.RLock()
+	defer fake.listEgressMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeIOClient) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ service.IOClient = new(FakeIOClient)

pkg/service/wire.go

@@ -58,6 +58,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live
 		telemetry.NewTelemetryService,
 		getMessageBus,
 		NewIOInfoService,
+		wire.Bind(new(IOClient), new(*IOInfoService)),
 		rpc.NewEgressClient,
 		getEgressStore,
 		NewEgressLauncher,

pkg/service/wire_gen.go

@@ -57,6 +57,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live
 		return nil, err
 	}
 	egressStore := getEgressStore(objectStore)
+	ingressStore := getIngressStore(objectStore)
 	keyProvider, err := createKeyProvider(conf)
 	if err != nil {
 		return nil, err
@@ -67,23 +68,22 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live
 	}
 	analyticsService := telemetry.NewAnalyticsService(conf, currentNode)
 	telemetryService := telemetry.NewTelemetryService(queuedNotifier, analyticsService)
-	rtcEgressLauncher := NewEgressLauncher(egressClient, egressStore, telemetryService)
+	ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService)
+	if err != nil {
+		return nil, err
+	}
+	rtcEgressLauncher := NewEgressLauncher(egressClient, ioInfoService)
 	roomService, err := NewRoomService(roomConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher)
 	if err != nil {
 		return nil, err
 	}
-	egressService := NewEgressService(egressClient, objectStore, egressStore, roomService, telemetryService, rtcEgressLauncher)
+	egressService := NewEgressService(egressClient, objectStore, ioInfoService, roomService, rtcEgressLauncher)
 	ingressConfig := getIngressConfig(conf)
 	ingressClient, err := rpc.NewIngressClient(nodeID, messageBus)
 	if err != nil {
 		return nil, err
 	}
-	ingressStore := getIngressStore(objectStore)
 	ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, roomService, telemetryService)
-	ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService)
-	if err != nil {
-		return nil, err
-	}
 	rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, telemetryService)
 	clientConfigurationManager := createClientConfiguration()
 	timedVersionGenerator := utils.NewDefaultTimedVersionGenerator()

pkg/sfu/buffer/rtpstats_sender.go

@@ -327,17 +327,6 @@ func (r *RTPStatsSender) Update(
 			r.extStartSN = extSequenceNumber
 		}
 
-		if extTimestamp < r.extStartTS {
-			r.logger.Infow(
-				"adjusting start timestamp",
-				"snBefore", r.extStartSN,
-				"snAfter", extSequenceNumber,
-				"tsBefore", r.extStartTS,
-				"tsAfter", extTimestamp,
-			)
-			r.extStartTS = extTimestamp
-		}
-
 		if gapSN != 0 {
 			r.packetsOutOfOrder++
 		}
@@ -377,12 +366,27 @@ func (r *RTPStatsSender) Update(
 
 		r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, false)
 
-		if extTimestamp != r.extHighestTS {
-			// update only on first packet as same timestamp could be in multiple packets.
-			// NOTE: this may not be the first packet with this time stamp if there is packet loss.
+		r.extHighestSN = extSequenceNumber
+	}
+
+	if extTimestamp < r.extStartTS {
+		r.logger.Infow(
+			"adjusting start timestamp",
+			"snBefore", r.extStartSN,
+			"snAfter", extSequenceNumber,
+			"tsBefore", r.extStartTS,
+			"tsAfter", extTimestamp,
+		)
+		r.extStartTS = extTimestamp
+	}
+
+	if extTimestamp > r.extHighestTS {
+		// update only on first packet as same timestamp could be in multiple packets.
+		// NOTE: this may not be the first packet with this time stamp if there is packet loss.
+		if payloadSize > 0 {
+			// skip updating on padding only packets as they could re-use an old timestamp
 			r.highestTime = packetTime
 		}
-		r.extHighestSN = extSequenceNumber
 		r.extHighestTS = extTimestamp
 	}
 

pkg/sfu/rtpmunger.go

@@ -54,10 +54,11 @@ type RTPMungerState struct {
 	ExtLastSN       uint64
 	ExtSecondLastSN uint64
 	ExtLastTS       uint64
+	ExtSecondLastTS uint64
 }
 
 func (r RTPMungerState) String() string {
-	return fmt.Sprintf("RTPMungerState{extLastSN: %d, extSecondLastSN: %d, extLastTS: %d)", r.ExtLastSN, r.ExtSecondLastSN, r.ExtLastTS)
+	return fmt.Sprintf("RTPMungerState{extLastSN: %d, extSecondLastSN: %d, extLastTS: %d, extSecondLastTS: %d)", r.ExtLastSN, r.ExtSecondLastSN, r.ExtLastTS, r.ExtSecondLastTS)
 }
 
 // ----------------------------------------------------------------------
@@ -72,8 +73,9 @@ type RTPMunger struct {
 	extSecondLastSN uint64
 	snOffset        uint64
 
-	extLastTS uint64
-	tsOffset  uint64
+	extLastTS       uint64
+	extSecondLastTS uint64
+	tsOffset        uint64
 
 	lastMarker bool
 
@@ -95,6 +97,7 @@ func (r *RTPMunger) DebugInfo() map[string]interface{} {
 		"ExtSecondLastSN":      r.extSecondLastSN,
 		"SNOffset":             r.snOffset,
 		"ExtLastTS":            r.extLastTS,
+		"ExtSecondLastTS":      r.extSecondLastTS,
 		"TSOffset":             r.tsOffset,
 		"LastMarker":           r.lastMarker,
 	}
@@ -105,6 +108,7 @@ func (r *RTPMunger) GetLast() RTPMungerState {
 		ExtLastSN:       r.extLastSN,
 		ExtSecondLastSN: r.extSecondLastSN,
 		ExtLastTS:       r.extLastTS,
+		ExtSecondLastTS: r.extSecondLastTS,
 	}
 }
 
@@ -112,6 +116,7 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) {
 	r.extLastSN = state.ExtLastSN
 	r.extSecondLastSN = state.ExtSecondLastSN
 	r.extLastTS = state.ExtLastTS
+	r.extSecondLastTS = state.ExtSecondLastTS
 }
 
 func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
@@ -123,6 +128,7 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
 	r.updateSnOffset()
 
 	r.extLastTS = extPkt.ExtTimestamp
+	r.extSecondLastTS = extPkt.ExtTimestamp
 }
 
 func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) {
@@ -156,6 +162,8 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) {
 
 	r.extLastSN = r.extSecondLastSN
 	r.updateSnOffset()
+
+	r.extLastTS = r.extSecondLastTS
 }
 
 func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) {
@@ -174,6 +182,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara
 
 		r.extSecondLastSN = r.extLastSN
 		r.extLastSN = extMungedSN
+		r.extSecondLastTS = r.extLastTS
 		r.extLastTS = extMungedTS
 		r.lastMarker = extPkt.Packet.Marker
 
@@ -306,6 +315,11 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
 	r.snRangeMap.DecValue(r.extHighestIncomingSN, uint64(num))
 	r.updateSnOffset()
 
+	if len(vals) == 1 {
+		r.extSecondLastTS = r.extLastTS
+	} else {
+		r.extSecondLastTS = vals[len(vals)-2].extTimestamp
+	}
 	r.tsOffset -= extLastTS - r.extLastTS
 	r.extLastTS = extLastTS