dandri/livekit
1
0
Fork 0
mirror of https://github.com/livekit/livekit.git synced 2026-04-26 19:35:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extend range of GOOD scores. (#1536)

Browse Source 
Empirically, the experience is not bad for a larger range.
So, triggering POOR too early causes confusion.
This commit is contained in:
Raja Subramanian
2023-03-22 11:36:30 +05:30
committed by GitHub
parent c76c35474c
commit f782c8956d
3 changed files with 58 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sfu/connectionquality/connectionstats.go
+5 -5
View File
@@ -252,23 +252,23 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 {
plw := float64(0.0)
switch {
case strings.EqualFold(mimeType, webrtc.MimeTypeOpus):
// 2.5%: fall to GOOD, 5%: fall to POOR
// 2.5%: fall to GOOD, 7.5%: fall to POOR
plw = 8.0
if isFecEnabled {
// 3.75%: fall to GOOD, 7.5%: fall to POOR
// 3.75%: fall to GOOD, 11.25%: fall to POOR
plw /= 1.5
}
case strings.EqualFold(mimeType, "audio/red"):
// 6.66%: fall to GOOD, 13.33%: fall to POOR
// 6.66%: fall to GOOD, 20.0%: fall to POOR
plw = 3.0
if isFecEnabled {
// 10%: fall to GOOD, 20%: fall to POOR
// 10%: fall to GOOD, 30.0%: fall to POOR
plw /= 1.5
}
case strings.HasPrefix(strings.ToLower(mimeType), "video/"):
// 2%: fall to GOOD, 4%: fall to POOR
// 2%: fall to GOOD, 6%: fall to POOR
plw = 10.0
}
pkg/sfu/connectionquality/connectionstats_test.go
+30 -29
View File
@@ -63,11 +63,12 @@ func TestConnectionQuality(t *testing.T) {
}
cs.updateScore(streams, now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(3.2), mos)
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// should stay at POOR quality for one iteration even if the conditions improve
// due to significant loss (12%) in the previous window
// should climb to GOOD quality in one iteration if the conditions improve.
// although significant loss (12%) in the previous window, lowest score is
// bound so that climbing back does not take too long even under excellent conditions.
now = now.Add(duration)
streams = map[uint32]*buffer.StreamStatsWithLayers{
1: &buffer.StreamStatsWithLayers{
@@ -80,10 +81,10 @@ func TestConnectionQuality(t *testing.T) {
}
cs.updateScore(streams, now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(3.2), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// should climb up to GOOD if conditions continue to be good
// should stay at GOOD if conditions continue to be good
now = now.Add(duration)
streams = map[uint32]*buffer.StreamStatsWithLayers{
1: &buffer.StreamStatsWithLayers{
@@ -178,7 +179,7 @@ func TestConnectionQuality(t *testing.T) {
}
cs.updateScore(streams, now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(3.2), mos)
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
now = now.Add(duration)
@@ -218,7 +219,7 @@ func TestConnectionQuality(t *testing.T) {
}
cs.updateScore(streams, now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(3.2), mos)
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// mute/unmute to bring quality back up
@@ -357,7 +358,7 @@ func TestConnectionQuality(t *testing.T) {
expectedQualities []expectedQuality
}{
// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
// "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 5%: GOOD, >= 5%: POOR
// "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR
{
name: "audio/opus - no fec",
mimeType: "audio/opus",
@@ -375,13 +376,13 @@ func TestConnectionQuality(t *testing.T) {
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 5.2,
expectedMOS: 3.2,
packetLossPercentage: 9.2,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 7.5%: GOOD, >= 7.5%: POOR
// "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR
{
name: "audio/opus - fec",
mimeType: "audio/opus",
@@ -399,13 +400,13 @@ func TestConnectionQuality(t *testing.T) {
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 8.2,
expectedMOS: 3.2,
packetLossPercentage: 13.2,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 13.33%: GOOD, >= 13.33%: POOR
// "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 20%: GOOD, >= 20%: POOR
{
name: "audio/red - no fec",
mimeType: "audio/red",
@@ -423,13 +424,13 @@ func TestConnectionQuality(t *testing.T) {
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 16.0,
expectedMOS: 3.2,
packetLossPercentage: 23.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 20%: GOOD, >= 20%: POOR
// "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR
{
name: "audio/red - fec",
mimeType: "audio/red",
@@ -447,13 +448,13 @@ func TestConnectionQuality(t *testing.T) {
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 22.0,
expectedMOS: 3.2,
packetLossPercentage: 36.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 4%: GOOD, >= 4%: POOR
// "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR
{
name: "video/*",
mimeType: "video/vp8",
@@ -471,8 +472,8 @@ func TestConnectionQuality(t *testing.T) {
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 5.0,
expectedMOS: 3.2,
packetLossPercentage: 7.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
@@ -523,8 +524,8 @@ func TestConnectionQuality(t *testing.T) {
}{
// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
// 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT
// ~2.7 <= expectedBits / actualBits < ~7.5 = GOOD
// expectedBits / actualBits >= ~7.5 = POOR
// ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD
// expectedBits / actualBits >= ~20.1 = POOR
{
name: "excellent",
transitions: []transition{
@@ -566,8 +567,8 @@ func TestConnectionQuality(t *testing.T) {
offset: 3 * time.Second,
},
},
bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 13.0)),
expectedMOS: 3.2,
bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 43.0)),
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
}
@@ -650,11 +651,11 @@ func TestConnectionQuality(t *testing.T) {
distance: 2.0,
},
{
distance: 2.0,
distance: 2.2,
offset: 1 * time.Second,
},
},
expectedMOS: 3.2,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
}
pkg/sfu/connectionquality/scorer.go
+23 -9
View File
@@ -14,12 +14,13 @@ const (
MaxMOS = float32(4.5)
maxScore = float64(100.0)
poorScore = float64(50.0)
poorScore = float64(30.0)
minScore = float64(20.0)
increaseFactor = float64(0.4) // slow increase
decreaseFactor = float64(0.8) // fast decrease
distanceWeight = float64(25.0) // each spatial layer missed drops a quality level
distanceWeight = float64(35.0) // each spatial layer missed drops a quality level
unmuteTimeThreshold = float64(0.5)
)
@@ -75,7 +76,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 {
if w.bytes != 0 {
// using the ratio of expectedBitrate / actualBitrate
// the quality inflection points are approximately
// GOOD at ~2.7x, POOR at ~7.5x
// GOOD at ~2.7x, POOR at ~20.1x
score = maxScore - 20*math.Log(float64(expectedBitrate)/float64(w.bytes*8))
if score > maxScore {
score = maxScore
@@ -264,12 +265,19 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) {
reason = "layer"
score = layerScore
}
factor := increaseFactor
if score < q.score {
factor = decreaseFactor
}
score = factor*score + (1.0-factor)*q.score
}
factor := increaseFactor
if score < q.score {
factor = decreaseFactor
if score < minScore {
// lower bound to prevent score from becoming very small values due to extreme conditions.
// Without a lower bound, it can get so low that it takes a long time to climb back to
// better quality even under excellent conditions.
score = minScore
}
score = factor*score + (1.0-factor)*q.score
// WARNING NOTE: comparing protobuf enum values directly (livekit.ConnectionQuality)
if scoreToConnectionQuality(q.score) > scoreToConnectionQuality(score) {
q.params.Logger.Infow(
@@ -453,13 +461,19 @@ func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality)
// ------------------------------------------
func scoreToConnectionQuality(score float64) livekit.ConnectionQuality {
// R-factor -> livekit.ConnectionQuality scale mapping based on
// R-factor -> livekit.ConnectionQuality scale mapping roughly based on
// https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm
//
// As there are only three levels in livekit.ConnectionQuality scale,
// using a larger range for middling quality. Empirical evidence suggests
// that a score of 60 does not correspond to `POOR` quality. Repair
// mechanisms and use of algorithms like de-jittering makes the experience
// better even under harsh conditions.
if score > 80.0 {
return livekit.ConnectionQuality_EXCELLENT
}
if score > 60.0 {
if score > 40.0 {
return livekit.ConnectionQuality_GOOD
}