diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index d2e4e30a3..98562ae1e 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -252,23 +252,23 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { plw := float64(0.0) switch { case strings.EqualFold(mimeType, webrtc.MimeTypeOpus): - // 2.5%: fall to GOOD, 5%: fall to POOR + // 2.5%: fall to GOOD, 7.5%: fall to POOR plw = 8.0 if isFecEnabled { - // 3.75%: fall to GOOD, 7.5%: fall to POOR + // 3.75%: fall to GOOD, 11.25%: fall to POOR plw /= 1.5 } case strings.EqualFold(mimeType, "audio/red"): - // 6.66%: fall to GOOD, 13.33%: fall to POOR + // 6.66%: fall to GOOD, 20.0%: fall to POOR plw = 3.0 if isFecEnabled { - // 10%: fall to GOOD, 20%: fall to POOR + // 10%: fall to GOOD, 30.0%: fall to POOR plw /= 1.5 } case strings.HasPrefix(strings.ToLower(mimeType), "video/"): - // 2%: fall to GOOD, 4%: fall to POOR + // 2%: fall to GOOD, 6%: fall to POOR plw = 10.0 } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 91667cb39..567ef485e 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -63,11 +63,12 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) - // should stay at POOR quality for one iteration even if the conditions improve - // due to significant loss (12%) in the previous window + // should climb to GOOD quality in one iteration if the conditions improve. + // although significant loss (12%) in the previous window, lowest score is + // bound so that climbing back does not take too long even under excellent conditions. now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: &buffer.StreamStatsWithLayers{ @@ -80,10 +81,10 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) - require.Equal(t, livekit.ConnectionQuality_POOR, quality) + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) - // should climb up to GOOD if conditions continue to be good + // should stay at GOOD if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: &buffer.StreamStatsWithLayers{ @@ -178,7 +179,7 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) now = now.Add(duration) @@ -218,7 +219,7 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) // mute/unmute to bring quality back up @@ -357,7 +358,7 @@ func TestConnectionQuality(t *testing.T) { expectedQualities []expectedQuality }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact - // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 5%: GOOD, >= 5%: POOR + // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR { name: "audio/opus - no fec", mimeType: "audio/opus", @@ -375,13 +376,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.2, - expectedMOS: 3.2, + packetLossPercentage: 9.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 7.5%: GOOD, >= 7.5%: POOR + // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR { name: "audio/opus - fec", mimeType: "audio/opus", @@ -399,13 +400,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 8.2, - expectedMOS: 3.2, + packetLossPercentage: 13.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 13.33%: GOOD, >= 13.33%: POOR + // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 20%: GOOD, >= 20%: POOR { name: "audio/red - no fec", mimeType: "audio/red", @@ -423,13 +424,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 16.0, - expectedMOS: 3.2, + packetLossPercentage: 23.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 20%: GOOD, >= 20%: POOR + // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR { name: "audio/red - fec", mimeType: "audio/red", @@ -447,13 +448,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 22.0, - expectedMOS: 3.2, + packetLossPercentage: 36.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 4%: GOOD, >= 4%: POOR + // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR { name: "video/*", mimeType: "video/vp8", @@ -471,8 +472,8 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.0, - expectedMOS: 3.2, + packetLossPercentage: 7.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, @@ -523,8 +524,8 @@ func TestConnectionQuality(t *testing.T) { }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact // 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT - // ~2.7 <= expectedBits / actualBits < ~7.5 = GOOD - // expectedBits / actualBits >= ~7.5 = POOR + // ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD + // expectedBits / actualBits >= ~20.1 = POOR { name: "excellent", transitions: []transition{ @@ -566,8 +567,8 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 13.0)), - expectedMOS: 3.2, + bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 43.0)), + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } @@ -650,11 +651,11 @@ func TestConnectionQuality(t *testing.T) { distance: 2.0, }, { - distance: 2.0, + distance: 2.2, offset: 1 * time.Second, }, }, - expectedMOS: 3.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 1a2fb6825..5582a7832 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -14,12 +14,13 @@ const ( MaxMOS = float32(4.5) maxScore = float64(100.0) - poorScore = float64(50.0) + poorScore = float64(30.0) + minScore = float64(20.0) increaseFactor = float64(0.4) // slow increase decreaseFactor = float64(0.8) // fast decrease - distanceWeight = float64(25.0) // each spatial layer missed drops a quality level + distanceWeight = float64(35.0) // each spatial layer missed drops a quality level unmuteTimeThreshold = float64(0.5) ) @@ -75,7 +76,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { if w.bytes != 0 { // using the ratio of expectedBitrate / actualBitrate // the quality inflection points are approximately - // GOOD at ~2.7x, POOR at ~7.5x + // GOOD at ~2.7x, POOR at ~20.1x score = maxScore - 20*math.Log(float64(expectedBitrate)/float64(w.bytes*8)) if score > maxScore { score = maxScore @@ -264,12 +265,19 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { reason = "layer" score = layerScore } + + factor := increaseFactor + if score < q.score { + factor = decreaseFactor + } + score = factor*score + (1.0-factor)*q.score } - factor := increaseFactor - if score < q.score { - factor = decreaseFactor + if score < minScore { + // lower bound to prevent score from becoming very small values due to extreme conditions. + // Without a lower bound, it can get so low that it takes a long time to climb back to + // better quality even under excellent conditions. + score = minScore } - score = factor*score + (1.0-factor)*q.score // WARNING NOTE: comparing protobuf enum values directly (livekit.ConnectionQuality) if scoreToConnectionQuality(q.score) > scoreToConnectionQuality(score) { q.params.Logger.Infow( @@ -453,13 +461,19 @@ func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality) // ------------------------------------------ func scoreToConnectionQuality(score float64) livekit.ConnectionQuality { - // R-factor -> livekit.ConnectionQuality scale mapping based on + // R-factor -> livekit.ConnectionQuality scale mapping roughly based on // https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm + // + // As there are only three levels in livekit.ConnectionQuality scale, + // using a larger range for middling quality. Empirical evidence suggests + // that a score of 60 does not correspond to `POOR` quality. Repair + // mechanisms and use of algorithms like de-jittering makes the experience + // better even under harsh conditions. if score > 80.0 { return livekit.ConnectionQuality_EXCELLENT } - if score > 60.0 { + if score > 40.0 { return livekit.ConnectionQuality_GOOD }