Use rtcscore-go to calculate audio/video score (#689)

* Use rtcscore-go to calculate audio/video score

Signed-off-by: shishir gowda <shishir@livekit.io>

* Get max expected layer and find max actual layer from stream

Signed-off-by: shishir gowda <shishir@livekit.io>

* Cleanup unused methods

Signed-off-by: shishir gowda <shishir@livekit.io>

* Cleanup code - address review comments

Signed-off-by: shishir gowda <shishir@livekit.io>

* get expected layer info instead of just quality

Signed-off-by: shishir gowda <shishir@livekit.io>

* Move SpatialLayerForQuality to utils/helpers

method is required in rtc,sfu and connectionstats pkg
Moved to utils/helpers.go to remove cyclic deps

Signed-off-by: shishir gowda <shishir@livekit.io>

* update tests

Signed-off-by: shishir gowda <shishir@livekit.io>

* Pick stream stats with max layer

Signed-off-by: shishir gowda <shishir@livekit.io>

* Update rtcscore-go pkg to make rtt/jitter optional

when passing 0, rtcscore-go was setting default values

Signed-off-by: shishir gowda <shishir@livekit.io>

* update score to rating

Signed-off-by: shishir gowda <shishir@livekit.io>

* Update rtcscore-go pkg to use simulcast layer info for score

Signed-off-by: shishir gowda <shishir@livekit.io>

* Update score ratings to reflect rtcscore range

Signed-off-by: shishir gowda <shishir@livekit.io>

* update test params for new rtcscore

Signed-off-by: shishir gowda <shishir@livekit.io>

* Delay sending scores to connections only till full data is available

first interval can have partial data leading to lower scores

Signed-off-by: shishir gowda <shishir@livekit.io>

* Check for inf values in quality params

Signed-off-by: shishir gowda <shishir@livekit.io>

* Clean up initial score calculation. Default to 5

Signed-off-by: shishir gowda <shishir@livekit.io>

Co-authored-by: David Zhao <dz@livekit.io>

pkg/rtc/mediatrack.go

@@ -155,7 +155,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra
 			receiver,
 			track,
 			t.PublisherID(),
-			t.params.TrackInfo.Source,
+			t.params.TrackInfo,
 			LoggerWithCodecMime(t.params.Logger, mime),
 			twcc,
 			sfu.WithPliThrottleConfig(t.params.PLIThrottleConfig),

pkg/rtc/mediatrackreceiver.go

@@ -19,6 +19,7 @@ import (
 	"github.com/livekit/livekit-server/pkg/sfu"
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/livekit-server/pkg/telemetry"
+	"github.com/livekit/livekit-server/pkg/utils"
 )
 
 const (
@@ -107,7 +108,7 @@ func (t *MediaTrackReceiver) Restart() {
 	t.lock.Unlock()
 
 	for _, receiver := range receivers {
-		receiver.SetMaxExpectedSpatialLayer(SpatialLayerForQuality(livekit.VideoQuality_HIGH))
+		receiver.SetMaxExpectedSpatialLayer(utils.SpatialLayerForQuality(livekit.VideoQuality_HIGH))
 	}
 
 	t.MediaTrackSubscriptions.Restart()
@@ -592,7 +593,7 @@ func (t *MediaTrackReceiver) OnSubscribedMaxQualityChange(f func(trackID livekit
 		for _, q := range maxSubscribedQualities {
 			receiver := t.Receiver(q.CodecMime)
 			if receiver != nil {
-				receiver.SetMaxExpectedSpatialLayer(SpatialLayerForQuality(q.Quality))
+				receiver.SetMaxExpectedSpatialLayer(utils.SpatialLayerForQuality(q.Quality))
 			}
 		}
 	})
@@ -600,21 +601,6 @@ func (t *MediaTrackReceiver) OnSubscribedMaxQualityChange(f func(trackID livekit
 
 // ---------------------------
 
-func SpatialLayerForQuality(quality livekit.VideoQuality) int32 {
-	switch quality {
-	case livekit.VideoQuality_LOW:
-		return 0
-	case livekit.VideoQuality_MEDIUM:
-		return 1
-	case livekit.VideoQuality_HIGH:
-		return 2
-	case livekit.VideoQuality_OFF:
-		return -1
-	default:
-		return -1
-	}
-}
-
 func QualityForSpatialLayer(layer int32) livekit.VideoQuality {
 	switch layer {
 	case 0:

pkg/rtc/participant.go

@@ -735,7 +735,10 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo
 		if subTrack.IsMuted() || subTrack.MediaTrack().IsMuted() {
 			continue
 		}
-		totalScore += subTrack.DownTrack().GetConnectionScore()
+		score := subTrack.DownTrack().GetConnectionScore()
+
+		totalScore += score
+
 		numTracks++
 	}
 	p.lock.RUnlock()
@@ -1447,7 +1450,9 @@ func (p *ParticipantImpl) getPublisherConnectionQuality() (totalScore float32, n
 		if pt.IsMuted() {
 			continue
 		}
-		totalScore += pt.(types.LocalMediaTrack).GetConnectionScore()
+		score := pt.(types.LocalMediaTrack).GetConnectionScore()
+
+		totalScore += score
 		numTracks++
 	}
 

pkg/rtc/participant_internal_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/protocol/auth"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/utils"
@@ -221,31 +222,28 @@ func TestMuteSetting(t *testing.T) {
 }
 
 func TestConnectionQuality(t *testing.T) {
-	// loss based score is currently a publisher method.
-	videoScore := func(loss float32, numPublishing, numRegistered uint32) float32 {
-		var reducedQuality bool
-		if numRegistered > 0 && numPublishing != numRegistered {
-			reducedQuality = true
-		}
-		return connectionquality.VideoConnectionScore(loss, reducedQuality)
+
+	videoScore := func(totalBytes int64, totalFrames int64, qualityParam *buffer.ConnectionQualityParams,
+		codec string, expectedHeight int32, expectedWidth int32, actualHeight int32, actualWidth int32) float32 {
+		return connectionquality.VideoConnectionScore(1*time.Second, totalBytes, totalFrames, qualityParam, codec,
+			expectedHeight, expectedWidth, actualHeight, actualWidth)
 	}
 
-	testPublishedVideoTrack := func(loss float32, numPublishing, numRegistered uint32) *typesfakes.FakeLocalMediaTrack {
+	testPublishedVideoTrack := func(totalBytes int64, totalFrames int64, qualityParam *buffer.ConnectionQualityParams,
+		codec string, expectedHeight int32, expectedWidth int32, actualHeight int32, actualWidth int32) *typesfakes.FakeLocalMediaTrack {
 		tr := &typesfakes.FakeLocalMediaTrack{}
-		score := videoScore(loss, numPublishing, numRegistered)
+		score := videoScore(totalBytes, totalFrames, qualityParam, codec, expectedHeight, expectedWidth,
+			actualHeight, actualWidth)
 		t.Log("video score: ", score)
 		tr.GetConnectionScoreReturns(score)
 		return tr
 	}
 
-	testPublishedAudioTrack := func(totalPackets, packetsLost uint32) *typesfakes.FakeLocalMediaTrack {
+	testPublishedAudioTrack := func(totalBytes int64, qualityParam *buffer.ConnectionQualityParams,
+		dtxDisabled bool) *typesfakes.FakeLocalMediaTrack {
 		tr := &typesfakes.FakeLocalMediaTrack{}
 
-		pctLoss := float32(0.0)
-		if totalPackets > 0 {
-			pctLoss = (float32(packetsLost) / float32(totalPackets)) * 100.0
-		}
-		score := connectionquality.AudioConnectionScore(pctLoss, 0, 0.0)
+		score := connectionquality.AudioConnectionScore(1*time.Second, totalBytes, qualityParam, dtxDisabled)
 		t.Log("audio score: ", score)
 		tr.GetConnectionScoreReturns(score)
 		return tr
@@ -256,33 +254,78 @@ func TestConnectionQuality(t *testing.T) {
 
 	t.Run("smooth sailing", func(t *testing.T) {
 		p := newParticipantForTest("test")
-		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(2, 3, 3)
-		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, 0)
+
+		// >2Mbps, 30fps,  expected/actual video size = 1280x720
+		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(290000, 30, &buffer.ConnectionQualityParams{},
+			"", 720, 1280, 720, 1280)
+
+		// no packet loss
+		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, &buffer.ConnectionQualityParams{}, false)
 
 		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, p.GetConnectionQuality().GetQuality())
 	})
 
 	t.Run("reduced publishing", func(t *testing.T) {
 		p := newParticipantForTest("test")
-		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(3, 2, 3)
-		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, 100)
+
+		// 1Mbps, 15fps,  expected = 1280x720, actual = 640 x 480
+		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(25000, 15, &buffer.ConnectionQualityParams{},
+			"", 720, 1280, 480, 640)
+
+		// packet loss of 10%
+		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, &buffer.ConnectionQualityParams{LossPercentage: 5}, false)
 
 		require.Equal(t, livekit.ConnectionQuality_GOOD, p.GetConnectionQuality().GetQuality())
 	})
 
 	t.Run("audio smooth publishing", func(t *testing.T) {
 		p := newParticipantForTest("test")
-		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, 10)
+		// no packet loss
+		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, &buffer.ConnectionQualityParams{}, false)
 
 		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, p.GetConnectionQuality().GetQuality())
 	})
 
 	t.Run("audio reduced publishing", func(t *testing.T) {
 		p := newParticipantForTest("test")
-		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, 100)
+		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, &buffer.ConnectionQualityParams{LossPercentage: 5}, false)
 
 		require.Equal(t, livekit.ConnectionQuality_GOOD, p.GetConnectionQuality().GetQuality())
 	})
+	t.Run("audio bad publishing", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		p.UpTrackManager.publishedTracks["audio"] = testPublishedAudioTrack(1000, &buffer.ConnectionQualityParams{LossPercentage: 20}, false)
+
+		require.Equal(t, livekit.ConnectionQuality_POOR, p.GetConnectionQuality().GetQuality())
+	})
+
+	t.Run("video smooth publishing", func(t *testing.T) {
+		p := newParticipantForTest("test")
+
+		// >2Mbps, 30fps,  expected/actual video size = 1280x720
+		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(290000, 30, &buffer.ConnectionQualityParams{},
+			"", 720, 1280, 720, 1280)
+
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, p.GetConnectionQuality().GetQuality())
+	})
+	t.Run("video reduced publishing", func(t *testing.T) {
+		p := newParticipantForTest("test")
+
+		// 1Mbps, 15fps,  expected = 1280x720, actual = 640 x 480
+		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(100000, 15, &buffer.ConnectionQualityParams{},
+			"", 720, 1280, 480, 640)
+
+		require.Equal(t, livekit.ConnectionQuality_GOOD, p.GetConnectionQuality().GetQuality())
+	})
+	t.Run("video poor publishing", func(t *testing.T) {
+		p := newParticipantForTest("test")
+
+		// 20kbps, 8fps,  expected = 1280x720, actual = 640 x 480
+		p.UpTrackManager.publishedTracks["video"] = testPublishedVideoTrack(2500, 8, &buffer.ConnectionQualityParams{},
+			"", 720, 1280, 426, 240)
+
+		require.Equal(t, livekit.ConnectionQuality_POOR, p.GetConnectionQuality().GetQuality())
+	})
 }
 
 func TestSubscriberAsPrimary(t *testing.T) {

pkg/rtc/subscribedtrack.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/livekit/livekit-server/pkg/rtc/types"
 	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/utils"
 )
 
 const (
@@ -53,7 +54,7 @@ func (t *SubscribedTrack) OnBind(f func()) {
 
 func (t *SubscribedTrack) Bound() {
 	if !t.params.AdaptiveStream {
-		t.params.DownTrack.SetMaxSpatialLayer(SpatialLayerForQuality(livekit.VideoQuality_HIGH))
+		t.params.DownTrack.SetMaxSpatialLayer(utils.SpatialLayerForQuality(livekit.VideoQuality_HIGH))
 	}
 	if t.onBind != nil {
 		t.onBind()
@@ -124,7 +125,7 @@ func (t *SubscribedTrack) UpdateVideoLayer() {
 	if settings.Width > 0 {
 		quality = t.MediaTrack().GetQualityForDimension(settings.Width, settings.Height)
 	}
-	t.DownTrack().SetMaxSpatialLayer(SpatialLayerForQuality(quality))
+	t.DownTrack().SetMaxSpatialLayer(utils.SpatialLayerForQuality(quality))
 }
 
 func (t *SubscribedTrack) updateDownTrackMute() {

pkg/sfu/connectionquality/connectionstats.go

@@ -1,6 +1,8 @@
 package connectionquality
 
 import (
+	"math"
+	"sort"
 	"sync"
 	"time"
 
@@ -11,6 +13,7 @@ import (
 	"github.com/livekit/protocol/logger"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/utils"
 )
 
 const (
@@ -20,10 +23,14 @@ const (
 type ConnectionStatsParams struct {
 	UpdateInterval      time.Duration
 	CodecType           webrtc.RTPCodecType
+	CodecName           string
+	DtxDisabled         bool
 	MimeType            string
 	GetDeltaStats       func() map[uint32]*buffer.StreamStatsWithLayers
 	GetQualityParams    func() *buffer.ConnectionQualityParams
 	GetIsReducedQuality func() bool
+	GetLayerDimension   func(int32) (uint32, uint32)
+	GetMaxExpectedLayer func() livekit.VideoLayer
 	Logger              logger.Logger
 }
 
@@ -42,7 +49,7 @@ type ConnectionStats struct {
 func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats {
 	return &ConnectionStats{
 		params: params,
-		score:  4.0,
+		score:  5.0,
 		done:   make(chan struct{}),
 	}
 }
@@ -70,29 +77,67 @@ func (cs *ConnectionStats) GetScore() float32 {
 	return cs.score
 }
 
-func (cs *ConnectionStats) updateScore() float32 {
+func (cs *ConnectionStats) updateScore(streams []*livekit.AnalyticsStream, iteration uint64) float32 {
 	cs.lock.Lock()
 	defer cs.lock.Unlock()
 
-	s := cs.params.GetQualityParams()
-	if s == nil {
+	// Initial interval will have partial data
+	if iteration < 2 {
+		cs.score = 5
 		return cs.score
 	}
 
-	if cs.params.CodecType == webrtc.RTPCodecTypeAudio {
-		cs.score = AudioConnectionScore(s.LossPercentage, s.Rtt, s.Jitter)
+	s := cs.params.GetQualityParams()
+	var qualityParam buffer.ConnectionQualityParams
+	if s == nil {
+		return cs.score
 	} else {
-		isReducedQuality := false
-		if cs.params.GetIsReducedQuality != nil {
-			isReducedQuality = cs.params.GetIsReducedQuality()
+		qualityParam = *s
+		if math.IsInf(float64(qualityParam.Jitter), 0) {
+			qualityParam.Jitter = 0
 		}
-		cs.score = VideoConnectionScore(s.LossPercentage, isReducedQuality)
+		if math.IsInf(float64(qualityParam.LossPercentage), 0) {
+			qualityParam.LossPercentage = 0
+		}
+	}
+
+	interval := cs.params.UpdateInterval
+	if interval == 0 {
+		interval = UpdateInterval
+	}
+	if cs.params.CodecType == webrtc.RTPCodecTypeAudio {
+		totalBytes, _, _ := cs.getBytesFramesFromStreams(streams)
+		cs.score = AudioConnectionScore(interval, int64(totalBytes), s, cs.params.DtxDisabled)
+	} else {
+		// get tracks expected max layer and dimensions
+		expectedLayer := cs.params.GetMaxExpectedLayer()
+		if utils.SpatialLayerForQuality(expectedLayer.Quality) == buffer.InvalidLayerSpatial {
+			return cs.score
+		}
+
+		// get bytes/frames and max later from actual stream stats
+		totalBytes, totalFrames, maxLayer := cs.getBytesFramesFromStreams(streams)
+		var actualHeight uint32
+		var actualWidth uint32
+		// if data present, but maxLayer == -1 no layer info available, set actual to expected, else fetch
+		if maxLayer == buffer.InvalidLayerSpatial && totalBytes > 0 {
+			actualHeight = expectedLayer.Height
+			actualWidth = expectedLayer.Width
+		} else {
+			actualWidth, actualHeight = cs.params.GetLayerDimension(maxLayer)
+		}
+
+		cs.score = VideoConnectionScore(interval, int64(totalBytes), int64(totalFrames), &qualityParam, cs.params.CodecName,
+			int32(expectedLayer.Height), int32(expectedLayer.Width), int32(actualHeight), int32(actualWidth))
+		//logger.Infow("VideoScoreScore", "score", cs.score, "expectedLayer", expectedLayer.Quality, "maxLayer", maxLayer,
+		//	"expectedWidth", expectedLayer.Width, "actualWidth", actualWidth, "expectedHeight", expectedLayer.Height, "actualHeight", actualHeight)
+
 	}
 
 	return cs.score
 }
 
-func (cs *ConnectionStats) getStat() *livekit.AnalyticsStat {
+func (cs *ConnectionStats) getStat(iteration uint64) *livekit.AnalyticsStat {
 	if cs.params.GetDeltaStats == nil {
 		return nil
 	}
@@ -120,7 +165,7 @@ func (cs *ConnectionStats) getStat() *livekit.AnalyticsStat {
 		analyticsStreams = append(analyticsStreams, as)
 	}
 
-	score := cs.updateScore()
+	score := cs.updateScore(analyticsStreams, iteration)
 
 	return &livekit.AnalyticsStat{
 		Score:   score,
@@ -138,13 +183,16 @@ func (cs *ConnectionStats) updateStatsWorker() {
 	tk := time.NewTicker(interval)
 	defer tk.Stop()
 
+	// Delay sending scores until 2nd cycle, as 1st will be partial.
+	counter := uint64(0)
+
 	for {
 		select {
 		case <-cs.done:
 			return
 
 		case <-tk.C:
-			stat := cs.getStat()
+			stat := cs.getStat(counter)
 			if stat == nil {
 				continue
 			}
@@ -152,6 +200,9 @@ func (cs *ConnectionStats) updateStatsWorker() {
 			if cs.onStatsUpdate != nil {
 				cs.onStatsUpdate(cs, stat)
 			}
+
+			counter++
+
 		}
 	}
 }
@@ -183,3 +234,41 @@ func ToAnalyticsVideoLayer(layer int, layerStats *buffer.LayerStats) *livekit.An
 		Frames:  layerStats.Frames,
 	}
 }
+
+func (cs *ConnectionStats) getBytesFramesFromStreams(streams []*livekit.AnalyticsStream) (totalBytes uint64, totalFrames uint32, maxLayer int32) {
+	layerStats := make(map[int32]buffer.LayerStats)
+	hasLayers := false
+	maxLayer = buffer.InvalidLayerSpatial
+	for _, stream := range streams {
+		// get frames/bytes/packets from video layers if available. Store per layer in layerStats map
+		if len(stream.VideoLayers) > 0 {
+			hasLayers = true
+
+			layers := stream.VideoLayers
+			// find max quality 0(LOW), 1(MED), 2(HIGH) . sort on layer.Layer desc
+			sort.Slice(layers, func(i, j int) bool {
+				return layers[i].Layer > layers[j].Layer
+			})
+
+			layerStats[layers[0].Layer] = buffer.LayerStats{
+				Bytes:  layers[0].GetBytes(),
+				Frames: layers[0].GetFrames(),
+			}
+			if layers[0].Layer > maxLayer {
+				maxLayer = layers[0].Layer
+			}
+		} else {
+			totalFrames += stream.GetFrames()
+			totalBytes += stream.GetPrimaryBytes()
+		}
+	}
+	if hasLayers {
+		if stats, ok := layerStats[maxLayer]; ok {
+			return stats.Bytes, stats.Frames, maxLayer
+		} else {
+			return 0, 0, buffer.InvalidLayerSpatial
+		}
+	}
+	return totalBytes, totalFrames, maxLayer
+
+}

pkg/sfu/connectionquality/mos.go

@@ -1,76 +1,78 @@
 package connectionquality
 
 import (
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/protocol/livekit"
-)
-
-// MOS score calculation is based on webrtc-stats
-// available @ https://github.com/oanguenot/webrtc-stats
-
-const (
-	defaultRtt = uint32(70)
+	"github.com/livekit/rtcscore-go/pkg/rtcmos"
 )
 
 func Score2Rating(score float32) livekit.ConnectionQuality {
-	if score > 3.9 {
+	if score > 4.0 {
 		return livekit.ConnectionQuality_EXCELLENT
 	}
 
-	if score > 2.5 {
+	if score > 3.0 {
 		return livekit.ConnectionQuality_GOOD
 	}
 	return livekit.ConnectionQuality_POOR
 }
 
-func mosAudioEModel(pctLoss float32, rtt uint32, jitter float32) float32 {
-	rx := 93.2 - pctLoss
-	ry := 0.18*rx*rx - 27.9*rx + 1126.62
-
-	if rtt == 0 {
-		rtt = defaultRtt
-	}
-	// Jitter is in Milliseconds
-	d := float32(rtt) + jitter
-	h := d - 177.3
-	if h < 0 {
-		h = 0
-	} else {
-		h = 1
-	}
-	id := 0.024*d + 0.11*(d-177.3)*h
-	r := ry - (id)
-	if r < 0 {
-		return 1
-	}
-	if r > 100 {
-		return 4.5
-	}
-	score := 1 + (0.035 * r) + (7.0/1000000)*r*(r-60)*(100-r)
-
-	return score
+func getBitRate(interval float64, totalBytes int64) int32 {
+	return int32(float64(totalBytes*8) / interval)
 }
 
-func loss2Score(pctLoss float32, reducedQuality bool) float32 {
-	// No Loss, excellent
-	if pctLoss == 0.0 && !reducedQuality {
-		return 5.0
-	}
-	// default when loss is minimal, but reducedQuality
-	score := float32(3.5)
-	// loss is bad
-	if pctLoss >= 4.0 {
-		score = 2.0
-	} else if pctLoss <= 2.0 && !reducedQuality {
-		// loss is acceptable and at reduced quality
-		score = 4.5
-	}
-	return score
+func getFrameRate(interval float64, totalFrames int64) int32 {
+	return int32(float64(totalFrames) / interval)
 }
 
-func AudioConnectionScore(pctLoss float32, rtt uint32, jitter float32) float32 {
-	return mosAudioEModel(pctLoss, rtt, jitter)
+func int32Ptr(x int32) *int32 {
+	return &x
 }
 
-func VideoConnectionScore(pctLoss float32, reducedQuality bool) float32 {
-	return loss2Score(pctLoss, reducedQuality)
+func AudioConnectionScore(interval time.Duration, totalBytes int64,
+	qualityParam *buffer.ConnectionQualityParams, dtxDisabled bool) float32 {
+
+	stat := rtcmos.Stat{
+		Bitrate:       getBitRate(interval.Seconds(), totalBytes),
+		PacketLoss:    qualityParam.LossPercentage,
+		RoundTripTime: int32Ptr(int32(qualityParam.Rtt)),
+		BufferDelay:   int32Ptr(int32(qualityParam.Jitter)),
+		AudioConfig:   &rtcmos.AudioConfig{},
+	}
+
+	if dtxDisabled {
+		flag := false
+		stat.AudioConfig.Dtx = &flag
+	}
+
+	scores := rtcmos.Score([]rtcmos.Stat{stat})
+	if len(scores) == 1 {
+		return float32(scores[0].AudioScore)
+	}
+	return 0
+}
+
+func VideoConnectionScore(interval time.Duration, totalBytes int64, totalFrames int64, qualityParam *buffer.ConnectionQualityParams,
+	codec string, expectedHeight int32, expectedWidth int32, actualHeight int32, actualWidth int32) float32 {
+	stat := rtcmos.Stat{
+		Bitrate:       getBitRate(interval.Seconds(), totalBytes),
+		PacketLoss:    qualityParam.LossPercentage,
+		RoundTripTime: int32Ptr(int32(qualityParam.Rtt)),
+		BufferDelay:   int32Ptr(int32(qualityParam.Jitter)),
+		VideoConfig: &rtcmos.VideoConfig{
+			FrameRate:      int32Ptr(getFrameRate(interval.Seconds(), totalFrames)),
+			Codec:          codec,
+			ExpectedHeight: &expectedHeight,
+			ExpectedWidth:  &expectedWidth,
+			Height:         &actualHeight,
+			Width:          &actualWidth,
+		},
+	}
+	scores := rtcmos.Score([]rtcmos.Stat{stat})
+	if len(scores) == 1 {
+		return float32(scores[0].VideoScore)
+	}
+	return 0
 }

pkg/sfu/downtrack.go

@@ -220,7 +220,19 @@ func NewDownTrack(
 		GetIsReducedQuality: func() bool {
 			return d.GetForwardingStatus() != ForwardingStatusOptimal
 		},
-		Logger: d.logger,
+		GetLayerDimension: func(quality int32) (uint32, uint32) {
+			if d.receiver != nil {
+				return d.receiver.GetLayerDimension(quality)
+			}
+			return 0, 0
+		},
+		GetMaxExpectedLayer: func() livekit.VideoLayer {
+			quality := d.forwarder.MaxLayers().Spatial
+			width, height := d.receiver.GetLayerDimension(quality)
+			return livekit.VideoLayer{Quality: livekit.VideoQuality(quality), Width: width, Height: height}
+		},
+		Logger:    d.logger,
+		CodecName: getCodecNameFromMime(codecs[0].MimeType),
 	})
 	d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
 		if d.onStatsUpdate != nil {

pkg/sfu/helpers.go

@@ -55,3 +55,12 @@ func getRttMs(report *rtcp.ReceptionReport) uint32 {
 	ntpDiff := now - report.LastSenderReport - report.Delay
 	return uint32(math.Ceil(float64(ntpDiff) * 1000.0 / 65536.0))
 }
+
+func getCodecNameFromMime(mime string) string {
+	codecName := ""
+	codecParsed := strings.Split(strings.ToLower(mime), "/")
+	if len(codecParsed) > 1 {
+		codecName = codecParsed[1]
+	}
+	return codecName
+}

pkg/sfu/receiver.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pion/webrtc/v3"
 	"go.uber.org/atomic"
 
+	"github.com/livekit/livekit-server/pkg/utils"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 
@@ -51,6 +52,8 @@ type TrackReceiver interface {
 	DeleteDownTrack(peerID livekit.ParticipantID)
 
 	DebugInfo() map[string]interface{}
+
+	GetLayerDimension(quality int32) (uint32, uint32)
 }
 
 // WebRTCReceiver receives a media track
@@ -72,6 +75,7 @@ type WebRTCReceiver struct {
 	closeOnce      sync.Once
 	closed         atomic.Bool
 	useTrackers    bool
+	TrackInfo      *livekit.TrackInfo
 
 	rtcpCh chan []rtcp.Packet
 
@@ -160,7 +164,7 @@ func NewWebRTCReceiver(
 	receiver *webrtc.RTPReceiver,
 	track *webrtc.TrackRemote,
 	pid livekit.ParticipantID,
-	source livekit.TrackSource,
+	trackInfo *livekit.TrackInfo,
 	logger logger.Logger,
 	twcc *twcc.Responder,
 	opts ...ReceiverOpts,
@@ -176,7 +180,8 @@ func NewWebRTCReceiver(
 		// LK-TODO: this should be based on VideoLayers protocol message rather than RID based
 		isSimulcast:          len(track.RID()) > 0,
 		twcc:                 twcc,
-		streamTrackerManager: NewStreamTrackerManager(logger, source),
+		streamTrackerManager: NewStreamTrackerManager(logger, trackInfo.Source),
+		TrackInfo:            trackInfo,
 		isSVC:                IsSvcCodec(track.Codec().MimeType),
 	}
 
@@ -199,7 +204,36 @@ func NewWebRTCReceiver(
 		GetIsReducedQuality: func() bool {
 			return w.streamTrackerManager.IsReducedQuality()
 		},
-		Logger: w.logger,
+		GetLayerDimension: func(quality int32) (uint32, uint32) {
+			return w.GetLayerDimension(quality)
+		},
+		GetMaxExpectedLayer: func() livekit.VideoLayer {
+			var expectedLayer livekit.VideoLayer
+			var maxPublishedLayer livekit.VideoLayer
+			// find min of <expected, published> layer
+			expectedQuality := w.streamTrackerManager.GetMaxExpectedLayer()
+			maxPublishedQuality := InvalidLayerSpatial
+			if w.TrackInfo != nil {
+				for _, layer := range w.TrackInfo.Layers {
+					if layer.Quality == livekit.VideoQuality_OFF {
+						continue
+					}
+					if expectedQuality == utils.SpatialLayerForQuality(layer.Quality) {
+						expectedLayer = *layer
+					}
+					if utils.SpatialLayerForQuality(layer.Quality) > maxPublishedQuality {
+						maxPublishedQuality = int32(layer.Quality)
+						maxPublishedLayer = *layer
+					}
+				}
+			}
+			if expectedQuality < maxPublishedQuality {
+				return expectedLayer
+			}
+			return maxPublishedLayer
+		},
+		Logger:    w.logger,
+		CodecName: getCodecNameFromMime(w.codec.MimeType),
 	})
 	w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
 		if w.onStatsUpdate != nil {
@@ -211,6 +245,19 @@ func NewWebRTCReceiver(
 	return w
 }
 
+func (w *WebRTCReceiver) GetLayerDimension(quality int32) (uint32, uint32) {
+	height := uint32(0)
+	width := uint32(0)
+	for _, layer := range w.TrackInfo.Layers {
+		if layer.Quality == livekit.VideoQuality(quality) {
+			height = layer.Height
+			width = layer.Width
+			break
+		}
+	}
+	return width, height
+}
+
 func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) {
 	w.onStatsUpdate = fn
 }

pkg/sfu/streamtrackermanager.go

@@ -239,6 +239,12 @@ func (s *StreamTrackerManager) IsReducedQuality() bool {
 	return int32(len(s.availableLayers)) < (s.maxExpectedLayer + 1)
 }
 
+func (s *StreamTrackerManager) GetMaxExpectedLayer() int32 {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.maxExpectedLayer
+}
+
 func (s *StreamTrackerManager) GetBitrateTemporalCumulative() Bitrates {
 	s.lock.RLock()
 	defer s.lock.RUnlock()

pkg/utils/helpers.go

@@ -0,0 +1,18 @@
+package utils
+
+import "github.com/livekit/protocol/livekit"
+
+func SpatialLayerForQuality(quality livekit.VideoQuality) int32 {
+	switch quality {
+	case livekit.VideoQuality_LOW:
+		return 0
+	case livekit.VideoQuality_MEDIUM:
+		return 1
+	case livekit.VideoQuality_HIGH:
+		return 2
+	case livekit.VideoQuality_OFF:
+		return -1
+	default:
+		return -1
+	}
+}