mirror of
https://github.com/livekit/livekit.git
synced 2026-08-28 00:44:12 +00:00
Add FlexFEC benchmark tooling
This commit is contained in:
David Chen
@@ -171,6 +171,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
|
||||
func (b *Buffer) Write(pkt []byte) (int, error) {
|
||||
if b.impair != nil {
|
||||
if b.impair.dropInbound() {
|
||||
b.logImpairedDrop(pkt)
|
||||
// Packet "lost" on the simulated 5G uplink; report success so pion's read
|
||||
// loop is unaffected.
|
||||
return len(pkt), nil
|
||||
@@ -188,6 +189,7 @@ func (b *Buffer) writeNow(pkt []byte) (n int, err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.logImpairedMarker(&rtpPacket)
|
||||
|
||||
b.Lock()
|
||||
if b.BufferBase.IsClosed() {
|
||||
|
||||
@@ -22,6 +22,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/webrtc/v4"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/packettrailer"
|
||||
"github.com/livekit/mediatransportutil/pkg/codec"
|
||||
"github.com/livekit/protocol/codecs/mime"
|
||||
)
|
||||
|
||||
// Debug-only simulated publisher ("robot") uplink impairment for the local FlexFEC test
|
||||
@@ -46,12 +52,18 @@ import (
|
||||
//
|
||||
// LK_PUB_LOSS fractional loss on the publisher->SFU path, e.g. 0.03 (3%)
|
||||
// LK_PUB_DELAY_MS one-way publisher<->SFU delay in ms (inbound media + outbound NACK)
|
||||
// LK_PUB_BURST_MS optional burst-on window; loss only applies inside this window
|
||||
// LK_PUB_GAP_MS optional burst-off window after LK_PUB_BURST_MS
|
||||
// LK_PUB_DROP_LOG set to 1 to log dropped RTP packets and received marker frame IDs
|
||||
//
|
||||
// (These are the robot-link knobs. The operator/subscriber-side knobs live in the sfu
|
||||
// package: LK_DOWNLINK_LOSS, LK_DOWNLINK_DELAY_MS, LK_UPLINK_DELAY_MS.)
|
||||
const (
|
||||
envPubLoss = "LK_PUB_LOSS"
|
||||
envPubDelayMs = "LK_PUB_DELAY_MS"
|
||||
envPubBurstMs = "LK_PUB_BURST_MS"
|
||||
envPubGapMs = "LK_PUB_GAP_MS"
|
||||
envPubDropLog = "LK_PUB_DROP_LOG"
|
||||
|
||||
impairQueueDepth = 8192
|
||||
)
|
||||
@@ -69,8 +81,12 @@ type delayedFeedback struct {
|
||||
}
|
||||
|
||||
type uplinkImpair struct {
|
||||
loss float64
|
||||
delay time.Duration
|
||||
loss float64
|
||||
delay time.Duration
|
||||
burst time.Duration
|
||||
gap time.Duration
|
||||
started time.Time
|
||||
logDrops bool
|
||||
|
||||
rngMu sync.Mutex
|
||||
rng *rand.Rand
|
||||
@@ -91,13 +107,20 @@ func getUplinkImpair() *uplinkImpair {
|
||||
globalImpairOnce.Do(func() {
|
||||
loss := parseEnvFloat(envPubLoss)
|
||||
delay := time.Duration(parseEnvInt(envPubDelayMs)) * time.Millisecond
|
||||
if loss <= 0 && delay <= 0 {
|
||||
burst := time.Duration(parseEnvInt(envPubBurstMs)) * time.Millisecond
|
||||
gap := time.Duration(parseEnvInt(envPubGapMs)) * time.Millisecond
|
||||
logDrops := parseEnvBool(envPubDropLog)
|
||||
if loss <= 0 && delay <= 0 && !logDrops {
|
||||
return
|
||||
}
|
||||
imp := &uplinkImpair{
|
||||
loss: loss,
|
||||
delay: delay,
|
||||
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
loss: loss,
|
||||
delay: delay,
|
||||
burst: burst,
|
||||
gap: gap,
|
||||
started: time.Now(),
|
||||
logDrops: logDrops,
|
||||
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
}
|
||||
if delay > 0 {
|
||||
imp.inQueue = make(chan delayedInbound, impairQueueDepth)
|
||||
@@ -115,7 +138,14 @@ func getUplinkImpair() *uplinkImpair {
|
||||
func (b *Buffer) initUplinkImpair() {
|
||||
b.impair = getUplinkImpair()
|
||||
if b.impair != nil && b.logger != nil {
|
||||
b.logger.Infow("uplink impairment enabled (debug)", "lossFraction", b.impair.loss, "delay", b.impair.delay)
|
||||
b.logger.Infow(
|
||||
"uplink impairment enabled (debug)",
|
||||
"lossFraction", b.impair.loss,
|
||||
"delay", b.impair.delay,
|
||||
"burst", b.impair.burst,
|
||||
"gap", b.impair.gap,
|
||||
"logDrops", b.impair.logDrops,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,12 +153,29 @@ func (u *uplinkImpair) dropInbound() bool {
|
||||
if u.loss <= 0 {
|
||||
return false
|
||||
}
|
||||
if !u.inLossWindow(time.Now()) {
|
||||
return false
|
||||
}
|
||||
u.rngMu.Lock()
|
||||
r := u.rng.Float64()
|
||||
u.rngMu.Unlock()
|
||||
return r < u.loss
|
||||
}
|
||||
|
||||
func (u *uplinkImpair) inLossWindow(now time.Time) bool {
|
||||
if u.burst <= 0 {
|
||||
return true
|
||||
}
|
||||
if u.gap <= 0 {
|
||||
return true
|
||||
}
|
||||
cycle := u.burst + u.gap
|
||||
if cycle <= 0 {
|
||||
return true
|
||||
}
|
||||
return now.Sub(u.started)%cycle < u.burst
|
||||
}
|
||||
|
||||
// delayInbound returns true when the packet was deferred onto the shared delay line (caller
|
||||
// should return immediately); false when no delay is configured and the caller should
|
||||
// proceed inline.
|
||||
@@ -202,3 +249,117 @@ func parseEnvInt(key string) int {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func parseEnvBool(key string) bool {
|
||||
v := os.Getenv(key)
|
||||
return v == "1" || v == "true" || v == "TRUE" || v == "yes" || v == "YES"
|
||||
}
|
||||
|
||||
func (u *uplinkImpair) shouldLogDrops() bool {
|
||||
return u != nil && u.logDrops
|
||||
}
|
||||
|
||||
func (b *Buffer) logImpairedDrop(pkt []byte) {
|
||||
if b.impair == nil || !b.impair.shouldLogDrops() || b.logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var rtpPacket rtp.Packet
|
||||
if err := rtpPacket.Unmarshal(pkt); err != nil {
|
||||
b.logger.Infow("uplink impairment dropped invalid RTP packet", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
fields := []any{
|
||||
"stream", b.impairedStreamKind(),
|
||||
"ssrc", rtpPacket.SSRC,
|
||||
"sequenceNumber", rtpPacket.SequenceNumber,
|
||||
"rtpTimestamp", rtpPacket.Timestamp,
|
||||
"payloadType", rtpPacket.PayloadType,
|
||||
"marker", rtpPacket.Marker,
|
||||
"frameTypeHint", b.frameTypeHint(&rtpPacket),
|
||||
}
|
||||
if metadata, ok := packettrailer.ParseTrailer(rtpPacket.Payload, rtpPacket.Marker); ok {
|
||||
fields = append(fields,
|
||||
"hasFrameID", metadata.HasFrameID,
|
||||
"frameID", metadata.FrameID,
|
||||
"hasUserTimestamp", metadata.HasTimestampUs,
|
||||
"userTimestampUs", metadata.TimestampUs,
|
||||
)
|
||||
}
|
||||
|
||||
b.logger.Infow("uplink impairment dropped RTP packet", fields...)
|
||||
}
|
||||
|
||||
func (b *Buffer) logImpairedMarker(pkt *rtp.Packet) {
|
||||
if b.impair == nil || !b.impair.shouldLogDrops() || b.logger == nil || !pkt.Marker {
|
||||
return
|
||||
}
|
||||
if b.primaryBufferForRTX != nil || b.primaryBufferForFEC != nil {
|
||||
return
|
||||
}
|
||||
|
||||
metadata, ok := packettrailer.ParseTrailer(pkt.Payload, pkt.Marker)
|
||||
if !ok || !metadata.HasFrameID {
|
||||
return
|
||||
}
|
||||
|
||||
b.logger.Infow(
|
||||
"uplink impairment received frame marker",
|
||||
"ssrc", pkt.SSRC,
|
||||
"sequenceNumber", pkt.SequenceNumber,
|
||||
"rtpTimestamp", pkt.Timestamp,
|
||||
"frameID", metadata.FrameID,
|
||||
"hasUserTimestamp", metadata.HasTimestampUs,
|
||||
"userTimestampUs", metadata.TimestampUs,
|
||||
"frameTypeHint", b.frameTypeHint(pkt),
|
||||
)
|
||||
}
|
||||
|
||||
func (b *Buffer) impairedStreamKind() string {
|
||||
switch {
|
||||
case b.primaryBufferForRTX != nil:
|
||||
return "rtx"
|
||||
case b.primaryBufferForFEC != nil:
|
||||
return "fec"
|
||||
default:
|
||||
return "media"
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) frameTypeHint(pkt *rtp.Packet) string {
|
||||
if b.codecType != webrtc.RTPCodecTypeVideo || b.impairedStreamKind() != "media" {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
switch b.mime {
|
||||
case mime.MimeTypeH264:
|
||||
if codec.IsH264KeyFrame(pkt.Payload) {
|
||||
return "I"
|
||||
}
|
||||
return "P"
|
||||
case mime.MimeTypeH265:
|
||||
if codec.IsH265KeyFrame(pkt.Payload) {
|
||||
return "I"
|
||||
}
|
||||
return "P"
|
||||
case mime.MimeTypeAV1:
|
||||
if codec.IsAV1KeyFrame(pkt.Payload) {
|
||||
return "I"
|
||||
}
|
||||
return "P"
|
||||
case mime.MimeTypeVP8:
|
||||
var vp8Packet codec.VP8
|
||||
if err := vp8Packet.Unmarshal(pkt.Payload); err == nil && vp8Packet.IsKeyFrame {
|
||||
return "I"
|
||||
}
|
||||
return "P"
|
||||
case mime.MimeTypeVP9:
|
||||
if codec.IsVP9KeyFrame(nil, pkt.Payload) {
|
||||
return "I"
|
||||
}
|
||||
return "P"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUplinkImpairLossWindow(t *testing.T) {
|
||||
started := time.Unix(100, 0)
|
||||
impair := &uplinkImpair{
|
||||
burst: 100 * time.Millisecond,
|
||||
gap: 200 * time.Millisecond,
|
||||
started: started,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
at time.Duration
|
||||
want bool
|
||||
}{
|
||||
{name: "start in burst", at: 0, want: true},
|
||||
{name: "inside burst", at: 99 * time.Millisecond, want: true},
|
||||
{name: "gap begins", at: 100 * time.Millisecond, want: false},
|
||||
{name: "inside gap", at: 250 * time.Millisecond, want: false},
|
||||
{name: "next burst", at: 300 * time.Millisecond, want: true},
|
||||
{name: "next gap", at: 450 * time.Millisecond, want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := impair.inLossWindow(started.Add(tt.at)); got != tt.want {
|
||||
t.Fatalf("inLossWindow() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUplinkImpairLossWindowWithoutBurst(t *testing.T) {
|
||||
impair := &uplinkImpair{started: time.Unix(100, 0)}
|
||||
|
||||
if !impair.inLossWindow(time.Unix(200, 0)) {
|
||||
t.Fatal("inLossWindow() = false, want true without burst config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUplinkImpairLossWindowWithoutGap(t *testing.T) {
|
||||
started := time.Unix(100, 0)
|
||||
impair := &uplinkImpair{burst: 100 * time.Millisecond, started: started}
|
||||
|
||||
if !impair.inLossWindow(started.Add(time.Hour)) {
|
||||
t.Fatal("inLossWindow() = false, want true when burst is set without gap")
|
||||
}
|
||||
}
|
||||
@@ -14,14 +14,26 @@
|
||||
|
||||
package packettrailer
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
var Magic = [4]byte{'L', 'K', 'T', 'S'}
|
||||
|
||||
const (
|
||||
xorByte = 0xFF
|
||||
|
||||
envelopeSize = 5 // 1B trailer_len + 4B magic
|
||||
|
||||
tagTimestampUs = 0x01
|
||||
tagFrameID = 0x02
|
||||
)
|
||||
|
||||
type Metadata struct {
|
||||
TimestampUs uint64
|
||||
HasTimestampUs bool
|
||||
FrameID uint32
|
||||
HasFrameID bool
|
||||
}
|
||||
|
||||
// StripTrailer returns the number of bytes to strip from the end of an RTP
|
||||
// payload if it contains an LKTS trailer. The trailer is located by checking
|
||||
// for the "LKTS" magic suffix and then reading the XORed trailer_len byte
|
||||
@@ -44,3 +56,48 @@ func StripTrailer(payload []byte, marker bool) int {
|
||||
|
||||
return trailerLen
|
||||
}
|
||||
|
||||
func ParseTrailer(payload []byte, marker bool) (Metadata, bool) {
|
||||
strip := StripTrailer(payload, marker)
|
||||
if strip == 0 {
|
||||
return Metadata{}, false
|
||||
}
|
||||
|
||||
var metadata Metadata
|
||||
tlvEnd := len(payload) - envelopeSize
|
||||
for i := len(payload) - strip; i < tlvEnd; {
|
||||
if i+2 > tlvEnd {
|
||||
return metadata, true
|
||||
}
|
||||
tag := payload[i] ^ xorByte
|
||||
length := int(payload[i+1] ^ xorByte)
|
||||
i += 2
|
||||
if length < 0 || i+length > tlvEnd {
|
||||
return metadata, true
|
||||
}
|
||||
|
||||
switch tag {
|
||||
case tagTimestampUs:
|
||||
if length == 8 {
|
||||
var buf [8]byte
|
||||
for j := range buf {
|
||||
buf[j] = payload[i+j] ^ xorByte
|
||||
}
|
||||
metadata.TimestampUs = binary.BigEndian.Uint64(buf[:])
|
||||
metadata.HasTimestampUs = true
|
||||
}
|
||||
case tagFrameID:
|
||||
if length == 4 {
|
||||
var buf [4]byte
|
||||
for j := range buf {
|
||||
buf[j] = payload[i+j] ^ xorByte
|
||||
}
|
||||
metadata.FrameID = binary.BigEndian.Uint32(buf[:])
|
||||
metadata.HasFrameID = true
|
||||
}
|
||||
}
|
||||
i += length
|
||||
}
|
||||
|
||||
return metadata, true
|
||||
}
|
||||
|
||||
@@ -19,11 +19,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
tagTimestampUs = 0x01
|
||||
tagFrameID = 0x02
|
||||
)
|
||||
|
||||
// appendTLV appends a single XORed TLV element to dst.
|
||||
func appendTLV(dst []byte, tag byte, value []byte) []byte {
|
||||
dst = append(dst, tag^xorByte, byte(len(value))^xorByte)
|
||||
@@ -78,7 +73,7 @@ func makeTimestampOnlyTrailer(timestampUs int64) []byte {
|
||||
}
|
||||
|
||||
func TestStripTrailer(t *testing.T) {
|
||||
fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5
|
||||
fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5
|
||||
tsOnlyTrailerSize := 15 // (1+1+8) + 5
|
||||
|
||||
tests := []struct {
|
||||
@@ -180,3 +175,33 @@ func TestStripTrailer(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTrailer(t *testing.T) {
|
||||
payload := makePayloadWithTrailer(20, 1700000000000000, 42)
|
||||
|
||||
metadata, ok := ParseTrailer(payload, true)
|
||||
if !ok {
|
||||
t.Fatal("ParseTrailer() did not find trailer")
|
||||
}
|
||||
if !metadata.HasTimestampUs || metadata.TimestampUs != 1700000000000000 {
|
||||
t.Fatalf("timestamp = (%v, %d), want (true, 1700000000000000)", metadata.HasTimestampUs, metadata.TimestampUs)
|
||||
}
|
||||
if !metadata.HasFrameID || metadata.FrameID != 42 {
|
||||
t.Fatalf("frame id = (%v, %d), want (true, 42)", metadata.HasFrameID, metadata.FrameID)
|
||||
}
|
||||
|
||||
metadata, ok = ParseTrailer(makeTimestampOnlyTrailer(99), true)
|
||||
if !ok {
|
||||
t.Fatal("ParseTrailer() did not find timestamp-only trailer")
|
||||
}
|
||||
if !metadata.HasTimestampUs || metadata.TimestampUs != 99 {
|
||||
t.Fatalf("timestamp = (%v, %d), want (true, 99)", metadata.HasTimestampUs, metadata.TimestampUs)
|
||||
}
|
||||
if metadata.HasFrameID {
|
||||
t.Fatalf("frame id present = %v, want false", metadata.HasFrameID)
|
||||
}
|
||||
|
||||
if _, ok := ParseTrailer(payload, false); ok {
|
||||
t.Fatal("ParseTrailer() found trailer without RTP marker")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,10 +30,14 @@ to UDP `7882` (see `flexfec-local.yaml`) so shaping can target it by port.
|
||||
| File | Purpose |
|
||||
| -------------------- | ---------------------------------------------------- |
|
||||
| `flexfec-local.yaml` | SFU config: flexfec on (pub+sub), fixed loopback port |
|
||||
| `flexfec-linux.yaml` | SFU config for Linux netns/veth uplink tests |
|
||||
| `run-sfu.sh` | Build + run the SFU (`--dev`, devkey/secret) |
|
||||
| `netem.sh` | Start/stop/inspect packet-loss shaping on udp/7882 |
|
||||
| `netns.sh` | Linux robot netns + veth shaping for uplink tests |
|
||||
| `run-leg-linux.sh` | Run one Linux benchmark leg and print log summaries |
|
||||
| `run-publisher.sh` | Run the publisher with `--flex-fec` |
|
||||
| `run-subscriber.sh` | Run the subscriber, surface `Video FEC ...` logs |
|
||||
| `sweep-linux.sh` | Linux multi-case FlexFEC-vs-RTX benchmark sweep |
|
||||
|
||||
## Workflow
|
||||
|
||||
@@ -127,6 +131,117 @@ With FEC ON you expect fewer `freezes`/`frames_dropped`, lower net `packets_lost
|
||||
meaningful `fec_recv`; with FEC OFF the same loss leans entirely on `rtx_recovered` and
|
||||
(because of the delay) typically shows more freezes and dropped frames.
|
||||
|
||||
## Full Linux benchmark sweep
|
||||
|
||||
For a full comparison on a Linux box, use:
|
||||
|
||||
```bash
|
||||
scripts/flexfec/sweep-linux.sh
|
||||
```
|
||||
|
||||
The sweep builds this SFU plus the `local_video` publisher/subscriber, generates one SFU
|
||||
config per case, runs every case for `DURATION=45` seconds by default, and writes raw logs
|
||||
plus a summary CSV:
|
||||
|
||||
```text
|
||||
scripts/flexfec/results/<timestamp>/
|
||||
summary.csv
|
||||
<leg>_<profile>_<fec-config>/
|
||||
sfu.yaml
|
||||
sfu.log
|
||||
publisher.log
|
||||
subscriber.log
|
||||
netem.log
|
||||
```
|
||||
|
||||
Default legs:
|
||||
|
||||
| Leg | Topology | Impairment | Compared signal |
|
||||
| -------- | --------------------------------------- | ----------------------------------- | --------------- |
|
||||
| Uplink | publisher in `robot` netns -> SFU | `netns.sh` burst loss on robot veth | SFU `packetsRecovered` |
|
||||
| Downlink | SFU -> subscriber on loopback/root netns | `netem.sh burst down` on UDP/7882 | subscriber FEC accepted payload + quality/latency lines |
|
||||
|
||||
Default FEC matrix:
|
||||
|
||||
| Label | Meaning |
|
||||
| --------- | -------------------------- |
|
||||
| `rtx` | FEC disabled, RTX baseline |
|
||||
| `fec5x1` | 5 media packets + 1 repair |
|
||||
| `fec8x1` | 8 media packets + 1 repair |
|
||||
| `fec10x2` | 10 media packets + 2 repairs |
|
||||
|
||||
Default burst profiles:
|
||||
|
||||
| Label | Loss burst |
|
||||
| ------------ | ---------------------------------------------------- |
|
||||
| `fade-short` | 35% loss for 250ms every 1750ms, 40ms one-way delay |
|
||||
| `fade-long` | 70% loss for 750ms every 5000ms, 60ms one-way delay |
|
||||
|
||||
Useful overrides:
|
||||
|
||||
```bash
|
||||
RUST_DIR=~/workspace/rust-sdks5 \
|
||||
OUT_DIR=/tmp/flexfec-results \
|
||||
DURATION=60 \
|
||||
SCENARIOS="uplink downlink" \
|
||||
FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec10x2:on:10:2" \
|
||||
LOSS_PROFILES="fade:35:250:1750:40:10:70 handoff:80:750:5000:60:20:85" \
|
||||
scripts/flexfec/sweep-linux.sh
|
||||
```
|
||||
|
||||
`LOSS_PROFILES` format:
|
||||
|
||||
```text
|
||||
label:loss_pct:burst_ms:gap_ms:one_way_delay_ms:jitter_ms:loss_correlation_pct
|
||||
```
|
||||
|
||||
For downlink loopback shaping, only `loss_pct`, `burst_ms`, `gap_ms`, and
|
||||
`one_way_delay_ms` are used. For uplink netns shaping, `jitter_ms` and
|
||||
`loss_correlation_pct` are also passed to `tc netem` during each burst.
|
||||
|
||||
## Focused remote-SFU publish-leg sweep
|
||||
|
||||
For tele-op tests where this computer runs the publisher/subscriber and `DC-Linux`
|
||||
runs only the SFU, use:
|
||||
|
||||
```bash
|
||||
scripts/flexfec/sweep-remote-publish.sh
|
||||
```
|
||||
|
||||
This keeps the real network RTT between this machine and `DC-Linux`; the SFU only
|
||||
injects publisher-side packet loss and does not add latency. Subscriber/downlink FEC is
|
||||
disabled in the generated SFU config, but subscriber quality and frame-latency logs are
|
||||
still captured for every case.
|
||||
|
||||
Default matrix:
|
||||
|
||||
| Axis | Values |
|
||||
| ---- | ------ |
|
||||
| Loss percent | `0.1 0.5 1 2 5` |
|
||||
| Burst duration | `100 200 500` ms |
|
||||
| Burst gap | `2000` ms |
|
||||
| FEC configs | `rtx`, `fec6x1`, `fec8x1`, `fec5x1` |
|
||||
|
||||
Useful overrides:
|
||||
|
||||
```bash
|
||||
REMOTE=dc@DC-Linux \
|
||||
SFU_HOST=DC-Linux \
|
||||
DURATION=90 \
|
||||
LOSS_PCTS="0.1 0.5 1 2 5" \
|
||||
BURST_MS="100 200 500" \
|
||||
BURST_GAP_MS=2000 \
|
||||
FEC_CONFIGS="rtx:off:0:0 fec6x1:on:6:1 fec8x1:on:8:1" \
|
||||
scripts/flexfec/sweep-remote-publish.sh
|
||||
```
|
||||
|
||||
Publisher packet-trailer metadata is always enabled with `--attach-timestamp` and
|
||||
`--attach-frame-id`. The SFU can log publish-side dropped RTP packets and received marker
|
||||
packets when `DROP_LOG=1` (the default for this sweep), including RTP timestamp, frame ID
|
||||
when present on marker packets, and a best-effort `frameTypeHint` (`I`/`P`/`unknown`).
|
||||
Each case also writes `dropped-packets.log` and `frame-markers.log` extracted from the SFU
|
||||
log for easier frame/timestamp correlation.
|
||||
|
||||
## Notes & caveats
|
||||
|
||||
- **Loopback shaping (macOS):** `netem.sh` loads its dummynet rules into a dedicated
|
||||
|
||||
Regular → Executable
@@ -0,0 +1,3 @@
|
||||
*
|
||||
!.gitignore
|
||||
!*.md
|
||||
@@ -0,0 +1,186 @@
|
||||
# FlexFEC Benchmark Report - 2026-06-06
|
||||
|
||||
## Summary
|
||||
|
||||
FlexFEC is functionally active on both tested legs:
|
||||
|
||||
- Publisher to SFU: the SFU enabled its FlexFEC decoder and recovered media packets from publisher-side repair packets.
|
||||
- SFU to subscriber: the subscriber received FlexFEC repair packets and accepted recovered payloads.
|
||||
|
||||
The clearest quality win in this run was on the SFU-to-subscriber leg. Under the long fade profile, RTX-only delivery averaged 1694.3 ms capture-to-decode latency, while FlexFEC reduced that to 295.0 ms with `fec5x1` and 298.1 ms with `fec10x2`.
|
||||
|
||||
The publisher-to-SFU leg also proved correction, but the single-run quality result was more configuration-sensitive. `fec8x1` was the best uplink long-fade result in this run: 146 SFU-recovered packets, 21 subscriber-reported lost packets vs. 195 for RTX-only, and 141.2 ms capture-to-decode average vs. 154.1 ms for RTX-only.
|
||||
|
||||
## Environment
|
||||
|
||||
- Host: `DC-Linux`
|
||||
- OS: Ubuntu 24.04.2 LTS
|
||||
- Kernel: `Linux dc-linux 6.11.0-25-generic`
|
||||
- Go: `/home/dc/go1.26/bin/go`, `go1.26.4 linux/amd64`
|
||||
- SFU source: synced from local branch `dc/exp/flexfec` at `ec606596` plus local benchmark script changes
|
||||
- Remote SFU workspace: `/home/dc/workspace/livekit-flexfec-bench`
|
||||
- Local video binaries: `/home/dc/workspace/rust-sdks5/target/debug/publisher` and `subscriber`
|
||||
|
||||
## Validation
|
||||
|
||||
Passed locally:
|
||||
|
||||
```bash
|
||||
go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc
|
||||
bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh scripts/flexfec/run-publisher.sh scripts/flexfec/run-sfu.sh scripts/flexfec/run-subscriber.sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Passed on `DC-Linux` after syncing:
|
||||
|
||||
```bash
|
||||
/home/dc/go1.26/bin/go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc
|
||||
bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh
|
||||
```
|
||||
|
||||
## Methodology
|
||||
|
||||
The sweep ran each case for 45 seconds after publisher and subscriber startup.
|
||||
|
||||
FEC matrix:
|
||||
|
||||
| Label | Meaning |
|
||||
|---|---|
|
||||
| `rtx` | FEC disabled, RTX baseline |
|
||||
| `fec5x1` | 5 media packets, 1 repair packet |
|
||||
| `fec8x1` | 8 media packets, 1 repair packet |
|
||||
| `fec10x2` | 10 media packets, 2 repair packets |
|
||||
|
||||
Loss profiles:
|
||||
|
||||
| Profile | Loss | Burst | Gap | One-way delay | Jitter | Correlation |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| `fade-short` | 35% | 250 ms | 1750 ms | 40 ms | 10 ms | 70% |
|
||||
| `fade-long` | 70% | 750 ms | 5000 ms | 60 ms | 20 ms | 85% |
|
||||
|
||||
Legs:
|
||||
|
||||
| Leg | Topology | Impairment | Primary correction signal |
|
||||
|---|---|---|---|
|
||||
| Publisher to SFU | publisher in `robot` netns -> SFU | `tc netem` on robot veth | SFU `packetsRecovered` |
|
||||
| SFU to subscriber | SFU -> subscriber on loopback/root netns | `tc netem` on SFU UDP/7882 downlink | Subscriber accepted FEC recovery payloads |
|
||||
|
||||
Raw artifacts:
|
||||
|
||||
- Downlink/full sweep: `scripts/flexfec/results/flexfec-full-20260606-120329/`
|
||||
- Corrected uplink sweep: `scripts/flexfec/results/flexfec-uplink-rerun-20260606-124334/`
|
||||
- Superseded uplink sweep with launcher failure: `scripts/flexfec/results/flexfec-uplink-20260606-122520/`
|
||||
|
||||
The first uplink sweep was not used for conclusions because publisher namespace startup failed in FEC cases. The sweep scripts were fixed to invoke helper scripts via `bash`, then the uplink-only sweep was rerun successfully.
|
||||
|
||||
## Publisher to SFU Results
|
||||
|
||||
Subscriber FEC counters are expected to stay zero in this leg because the downlink is clean and subscriber-side FlexFEC is disabled. The relevant correction signal is SFU `packetsRecovered`.
|
||||
|
||||
### Uplink `fade-short`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 1515 | 8 | 40 | 0 | 98.4 ms | 0.0 ms | 99.7 ms |
|
||||
| `fec5x1` | 212 | 1134 | 12 | 507 | 25 | 102.9 ms | +4.5 ms | 104.9 ms |
|
||||
| `fec8x1` | 118 | 1361 | 8 | 123 | 4 | 105.1 ms | +6.7 ms | 231.0 ms |
|
||||
| `fec10x2` | 50 | 1485 | 10 | 88 | 0 | 103.3 ms | +4.9 ms | 209.5 ms |
|
||||
|
||||
Interpretation:
|
||||
|
||||
- All FEC configurations except RTX produced SFU packet recovery.
|
||||
- RTX-only had the best short-fade quality metrics in this single run.
|
||||
- `fec8x1` was the least disruptive FEC option in this profile: same freeze count as RTX and only 4 lost packets, while still proving 118 recovered packets at the SFU.
|
||||
|
||||
### Uplink `fade-long`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 1224 | 9 | 239 | 195 | 154.1 ms | 0.0 ms | 156.2 ms |
|
||||
| `fec5x1` | 32 | 1148 | 9 | 261 | 105 | 395.7 ms | +241.6 ms | 4141.8 ms |
|
||||
| `fec8x1` | 146 | 1295 | 9 | 213 | 21 | 141.2 ms | -12.9 ms | 142.7 ms |
|
||||
| `fec10x2` | 0 | 991 | 13 | 262 | 31 | 386.0 ms | +231.9 ms | 4139.4 ms |
|
||||
|
||||
Interpretation:
|
||||
|
||||
- `fec8x1` was the best uplink long-fade configuration in this run.
|
||||
- `fec8x1` recovered 146 packets at the SFU, decoded more frames than RTX, reduced reported lost packets from 195 to 21, and reduced average capture-to-decode latency by 12.9 ms.
|
||||
- `fec5x1` and `fec10x2` showed large latency spikes in this run, despite lower reported lost packets than RTX. That makes them poor uplink choices until repeated runs show otherwise.
|
||||
- `fec10x2` enabled the decoder, but its last SFU recovery counter was zero in this case. That needs repeat testing; burst alignment can dominate a single 45-second run.
|
||||
|
||||
## SFU to Subscriber Results
|
||||
|
||||
Here the correction signal is subscriber-side accepted recovery payloads. The `fec_recv` column comes from the last video quality log line and can be slightly lower than accepted total when the accepted counter logged later.
|
||||
|
||||
### Downlink `fade-short`
|
||||
|
||||
| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 0 | 1425 | 14 | 82 | 57 | 57 | 131.1 ms | 0.0 ms | 270.0 ms |
|
||||
| `fec5x1` | 325 | 315 | 1392 | 15 | 81 | 63 | 50 | 113.0 ms | -18.1 ms | 254.9 ms |
|
||||
| `fec8x1` | 203 | 203 | 1450 | 15 | 80 | 60 | 52 | 108.6 ms | -22.5 ms | 255.4 ms |
|
||||
| `fec10x2` | 327 | 327 | 1452 | 13 | 90 | 69 | 59 | 118.5 ms | -12.6 ms | 270.5 ms |
|
||||
|
||||
Interpretation:
|
||||
|
||||
- All FEC configurations generated accepted recovery payloads at the subscriber.
|
||||
- Average capture-to-decode latency improved by 12.6 to 22.5 ms vs. RTX-only.
|
||||
- Freeze counts were broadly similar in this short profile, so latency and accepted recovery are the stronger signals.
|
||||
- `fec8x1` had the lowest average latency; `fec10x2` decoded the most frames and had the fewest freezes.
|
||||
|
||||
### Downlink `fade-long`
|
||||
|
||||
| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 0 | 839 | 10 | 284 | 123 | 105 | 1694.3 ms | 0.0 ms | 3262.7 ms |
|
||||
| `fec5x1` | 307 | 295 | 632 | 12 | 309 | 118 | 101 | 295.0 ms | -1399.3 ms | 3260.3 ms |
|
||||
| `fec8x1` | 198 | 187 | 843 | 16 | 313 | 125 | 105 | 527.0 ms | -1167.3 ms | 3754.8 ms |
|
||||
| `fec10x2` | 312 | 300 | 747 | 11 | 307 | 130 | 112 | 298.1 ms | -1396.2 ms | 3259.6 ms |
|
||||
|
||||
Interpretation:
|
||||
|
||||
- Downlink FlexFEC strongly reduced average decode latency under long fades.
|
||||
- `fec5x1` and `fec10x2` performed similarly on latency: 295.0 ms and 298.1 ms vs. 1694.3 ms for RTX-only.
|
||||
- `fec8x1` decoded the most frames, but had worse latency and more freezes than the other FEC profiles.
|
||||
- The high max latency values show there are still isolated stalls; the average latency improvement is still large.
|
||||
|
||||
## Conclusions
|
||||
|
||||
1. FlexFEC correction is proven in both directions.
|
||||
- Publisher to SFU: SFU logs show the FlexFEC decoder enabled and `packetsRecovered` increasing.
|
||||
- SFU to subscriber: subscriber logs show FEC received and recovery payloads accepted.
|
||||
|
||||
2. SFU-to-subscriber FEC is clearly useful in this burst-loss setup.
|
||||
- It substantially reduced average capture-to-decode latency, especially for long fades.
|
||||
- `fec5x1` looks like the best initial downlink default because it matched `fec10x2` latency with lower nominal repair overhead.
|
||||
|
||||
3. Publisher-to-SFU FEC is useful but needs more repetitions before choosing a default.
|
||||
- `fec8x1` was the best uplink long-fade configuration in this run.
|
||||
- `fec5x1` recovered the most short-fade packets but hurt subscriber quality metrics in that case.
|
||||
- `fec10x2` was inconsistent on uplink and should not be chosen from this single run.
|
||||
|
||||
4. Freeze count alone is not a reliable success metric here.
|
||||
- Some FEC cases recovered packets and reduced latency without reducing freezes.
|
||||
- Report `packetsRecovered`, accepted recovery payloads, latency, lost packets, NACK/PLI, and decoded frames together.
|
||||
|
||||
## Recommended Next Sweep
|
||||
|
||||
Run 3 to 5 repetitions per case and compare medians/percentiles. The current single-run results are enough to validate functionality, but burst timing makes per-config ranking noisy.
|
||||
|
||||
Recommended next matrix:
|
||||
|
||||
```bash
|
||||
DURATION=120 \
|
||||
SCENARIOS="uplink downlink" \
|
||||
FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec6x1:on:6:1 fec8x1:on:8:1 fec10x1:on:10:1 fec10x2:on:10:2" \
|
||||
LOSS_PROFILES="fade-short:35:250:1750:40:10:70 fade-long:70:750:5000:60:20:85 handoff:85:1200:7000:80:30:90" \
|
||||
scripts/flexfec/sweep-linux.sh
|
||||
```
|
||||
|
||||
Add report fields for:
|
||||
|
||||
- FEC bytes/packets sent to compute overhead.
|
||||
- P50/P95/P99 capture-to-decode latency from all timing windows, not just the last one.
|
||||
- SFU ingress loss before and after recovery.
|
||||
- Subscriber RTP jitter and jitter-buffer delay.
|
||||
- CPU usage for SFU and publisher/subscriber.
|
||||
@@ -0,0 +1,180 @@
|
||||
# FlexFEC Tele-Op Benchmark Report - 2026-06-06
|
||||
|
||||
## Summary
|
||||
|
||||
Ran a tele-op-oriented benchmark suite on `DC-Linux`:
|
||||
|
||||
- Main path: publisher/robot uplink over cellular-like loss into the SFU.
|
||||
- Control path: SFU-to-subscriber downlink over clean/mild landline-like loss.
|
||||
|
||||
The result matches the tele-op premise:
|
||||
|
||||
- Uplink FEC is useful under moderate to severe cellular fades, but the right block size depends on the fade shape.
|
||||
- Downlink FEC showed no practical quality benefit under clean/mild landline conditions; it only adds repair traffic.
|
||||
|
||||
Best uplink candidates from this single run:
|
||||
|
||||
- `fec6x1` for moderate `cell-edge` loss.
|
||||
- `fec5x1` for severe long handoff where latency matters most.
|
||||
- `fec8x1` as a conservative general candidate: it behaved well in `cell-edge`, `handoff-long`, and `bursty-uplink`, but was not always the lowest-latency choice.
|
||||
|
||||
Avoid `fec10x2` as a default for tele-op uplink. It recovered packets, but it caused clear latency/quality regressions in `bursty-uplink`.
|
||||
|
||||
## Validation
|
||||
|
||||
Passed locally:
|
||||
|
||||
```bash
|
||||
go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc
|
||||
bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh scripts/flexfec/run-publisher.sh scripts/flexfec/run-sfu.sh scripts/flexfec/run-subscriber.sh
|
||||
```
|
||||
|
||||
Passed on `DC-Linux` after syncing:
|
||||
|
||||
```bash
|
||||
/home/dc/go1.26/bin/go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc
|
||||
bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh
|
||||
```
|
||||
|
||||
No launcher failures were found in the copied log artifacts.
|
||||
|
||||
## Raw Artifacts
|
||||
|
||||
- Uplink cellular sweep: `scripts/flexfec/results/flexfec-teleop-uplink-20260606-135721/`
|
||||
- Downlink landline control: `scripts/flexfec/results/flexfec-teleop-downlink-control-20260606-151840/`
|
||||
|
||||
## Uplink Sweep
|
||||
|
||||
Command shape:
|
||||
|
||||
```bash
|
||||
DURATION=120 \
|
||||
SCENARIOS=uplink \
|
||||
FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec6x1:on:6:1 fec8x1:on:8:1 fec10x1:on:10:1 fec10x2:on:10:2" \
|
||||
LOSS_PROFILES="clean-control:0:100:2000:20:2:0 urban-5g:3:100:2000:35:8:50 cell-edge:20:250:3000:60:20:75 handoff-short:60:500:8000:80:30:85 handoff-long:85:1200:12000:100:40:90 bursty-uplink:35:300:2500:70:25:85" \
|
||||
scripts/flexfec/sweep-linux.sh
|
||||
```
|
||||
|
||||
Primary correction signal: SFU `packetsRecovered`.
|
||||
|
||||
### `clean-control`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3645 | 2 | 2 | 0 | 0 | 72.8 ms | 0.0 ms | 73.6 ms |
|
||||
| `fec5x1` | 0 | 3706 | 2 | 3 | 0 | 0 | 71.0 ms | -1.8 ms | 72.0 ms |
|
||||
| `fec6x1` | 0 | 3736 | 2 | 2 | 0 | 0 | 71.9 ms | -0.9 ms | 73.0 ms |
|
||||
| `fec8x1` | 0 | 3676 | 2 | 2 | 0 | 0 | 72.4 ms | -0.4 ms | 73.5 ms |
|
||||
| `fec10x1` | 498 | 3585 | 3 | 818 | 37 | 1 | 71.9 ms | -0.9 ms | 72.9 ms |
|
||||
| `fec10x2` | 0 | 3676 | 2 | 5 | 0 | 0 | 71.9 ms | -0.9 ms | 72.7 ms |
|
||||
|
||||
Note: `fec10x1` is anomalous here: a no-loss case produced SFU recovery, high NACK, and lost packets. Treat that row as suspect and rerun before drawing conclusions from it.
|
||||
|
||||
### `urban-5g`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3736 | 2 | 14 | 0 | 0 | 86.9 ms | 0.0 ms | 87.8 ms |
|
||||
| `fec5x1` | 0 | 3647 | 2 | 10 | 0 | 0 | 87.8 ms | +0.9 ms | 88.5 ms |
|
||||
| `fec6x1` | 0 | 3676 | 2 | 11 | 0 | 0 | 88.6 ms | +1.7 ms | 89.5 ms |
|
||||
| `fec8x1` | 20 | 3646 | 2 | 29 | 0 | 0 | 87.8 ms | +0.9 ms | 94.5 ms |
|
||||
| `fec10x1` | 101 | 3705 | 2 | 34 | 0 | 0 | 87.8 ms | +0.9 ms | 92.8 ms |
|
||||
| `fec10x2` | 101 | 3706 | 2 | 47 | 0 | 0 | 88.8 ms | +1.9 ms | 90.0 ms |
|
||||
|
||||
Interpretation: RTX/no-FEC is already fine under mild cellular loss. FEC adds little benefit here.
|
||||
|
||||
### `cell-edge`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3608 | 5 | 446 | 0 | 0 | 137.6 ms | 0.0 ms | 138.8 ms |
|
||||
| `fec5x1` | 145 | 3674 | 5 | 506 | 0 | 0 | 137.1 ms | -0.5 ms | 159.7 ms |
|
||||
| `fec6x1` | 77 | 3692 | 4 | 443 | 0 | 0 | 134.1 ms | -3.5 ms | 138.8 ms |
|
||||
| `fec8x1` | 77 | 3734 | 4 | 474 | 0 | 0 | 137.8 ms | +0.2 ms | 147.9 ms |
|
||||
| `fec10x1` | 0 | 3674 | 4 | 456 | 0 | 0 | 135.0 ms | -2.6 ms | 143.1 ms |
|
||||
| `fec10x2` | 189 | 3674 | 4 | 568 | 0 | 0 | 137.2 ms | -0.4 ms | 161.5 ms |
|
||||
|
||||
Interpretation: `fec6x1` was the best-balanced result: lowest latency, fewer freezes than RTX, and similar NACK load.
|
||||
|
||||
### `handoff-short`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3465 | 15 | 889 | 7 | 2 | 222.5 ms | 0.0 ms | 789.2 ms |
|
||||
| `fec5x1` | 185 | 3551 | 16 | 880 | 8 | 1 | 201.4 ms | -21.1 ms | 205.6 ms |
|
||||
| `fec6x1` | 125 | 3500 | 18 | 828 | 2 | 1 | 192.4 ms | -30.1 ms | 927.9 ms |
|
||||
| `fec8x1` | 206 | 3655 | 16 | 832 | 0 | 0 | 231.6 ms | +9.1 ms | 403.5 ms |
|
||||
| `fec10x1` | 25 | 3734 | 17 | 866 | 0 | 0 | 242.4 ms | +19.9 ms | 630.5 ms |
|
||||
| `fec10x2` | 181 | 3617 | 16 | 856 | 2 | 1 | 227.3 ms | +4.8 ms | 571.6 ms |
|
||||
|
||||
Interpretation: `fec6x1` had the best average latency, but its max latency and freeze count were worse. `fec5x1` is safer for short handoffs if max-latency stability matters more than the lowest average.
|
||||
|
||||
### `handoff-long`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3373 | 12 | 915 | 3 | 2 | 245.3 ms | 0.0 ms | 261.1 ms |
|
||||
| `fec5x1` | 454 | 3268 | 14 | 847 | 0 | 0 | 171.2 ms | -74.1 ms | 214.0 ms |
|
||||
| `fec6x1` | 503 | 3426 | 15 | 954 | 1 | 1 | 219.6 ms | -25.7 ms | 238.5 ms |
|
||||
| `fec8x1` | 446 | 3549 | 12 | 1105 | 35 | 1 | 202.2 ms | -43.1 ms | 231.6 ms |
|
||||
| `fec10x1` | 500 | 2963 | 21 | 823 | 1 | 1 | 1120.8 ms | +875.5 ms | 1197.1 ms |
|
||||
| `fec10x2` | 491 | 3207 | 16 | 922 | 8 | 1 | 223.8 ms | -21.5 ms | 234.6 ms |
|
||||
|
||||
Interpretation: `fec5x1` clearly lowered latency under severe long fades. `fec8x1` decoded the most frames and matched RTX freeze count, but reported more lost packets. `fec10x1` is a bad fit here.
|
||||
|
||||
### `bursty-uplink`
|
||||
|
||||
| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 3726 | 20 | 661 | 0 | 0 | 153.5 ms | 0.0 ms | 157.5 ms |
|
||||
| `fec5x1` | 20 | 3704 | 21 | 650 | 0 | 0 | 154.1 ms | +0.6 ms | 164.3 ms |
|
||||
| `fec6x1` | 122 | 3668 | 16 | 732 | 8 | 1 | 160.0 ms | +6.5 ms | 169.7 ms |
|
||||
| `fec8x1` | 143 | 3668 | 17 | 775 | 6 | 1 | 154.0 ms | +0.5 ms | 159.3 ms |
|
||||
| `fec10x1` | 120 | 3608 | 20 | 736 | 3 | 1 | 157.4 ms | +3.9 ms | 158.9 ms |
|
||||
| `fec10x2` | 290 | 3199 | 18 | 1091 | 27 | 5 | 254.6 ms | +101.1 ms | 664.0 ms |
|
||||
|
||||
Interpretation: `fec8x1` was the best FEC tradeoff here: nearly RTX latency with fewer freezes, but still more NACK and some lost packets. `fec10x2` is too heavy/unstable for this profile.
|
||||
|
||||
## Downlink Landline Control
|
||||
|
||||
Command shape:
|
||||
|
||||
```bash
|
||||
DURATION=60 \
|
||||
SCENARIOS=downlink \
|
||||
FEC_CONFIGS="rtx:off:0:0 fec8x1:on:8:1 fec10x2:on:10:2" \
|
||||
LOSS_PROFILES="landline-clean:0:100:2000:5:1:0 landline-mild:1:50:2000:10:2:0" \
|
||||
scripts/flexfec/sweep-linux.sh
|
||||
```
|
||||
|
||||
Primary correction signal: subscriber accepted recovery payloads. Under these landline profiles, accepted payloads prove FEC is active, not that it is needed.
|
||||
|
||||
### `landline-clean`
|
||||
|
||||
| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 0 | 1906 | 2 | 0 | 0 | 0 | 56.8 ms | 0.0 ms | 57.6 ms |
|
||||
| `fec8x1` | 267 | 267 | 1907 | 2 | 0 | 0 | 0 | 56.8 ms | 0.0 ms | 57.7 ms |
|
||||
| `fec10x2` | 426 | 408 | 1816 | 2 | 0 | 0 | 0 | 55.9 ms | -0.9 ms | 56.8 ms |
|
||||
|
||||
### `landline-mild`
|
||||
|
||||
| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Max |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| `rtx` | 0 | 0 | 1906 | 2 | 0 | 0 | 0 | 62.8 ms | 0.0 ms | 63.7 ms |
|
||||
| `fec8x1` | 267 | 259 | 1847 | 2 | 0 | 0 | 0 | 62.9 ms | +0.1 ms | 63.6 ms |
|
||||
| `fec10x2` | 426 | 408 | 1817 | 2 | 0 | 0 | 0 | 62.9 ms | +0.1 ms | 63.7 ms |
|
||||
|
||||
Interpretation: downlink FEC provided no visible quality benefit in clean or mild landline conditions. Keep subscriber-side/downlink FEC disabled for the tele-op default unless the operator path is known to be lossy.
|
||||
|
||||
## Recommendation
|
||||
|
||||
For a tele-op default where only the robot uplink is cellular:
|
||||
|
||||
1. Enable FEC on publisher-to-SFU only.
|
||||
2. Leave SFU-to-subscriber FEC disabled by default.
|
||||
3. Start field testing with `fec6x1` and `fec8x1`.
|
||||
4. Add `fec5x1` as an aggressive mode for known handoff/fade zones.
|
||||
5. Do not default to `fec10x2`; it is too costly under correlated burst loss in this run.
|
||||
|
||||
The next benchmark improvement should be repeated runs with median/P95/P99 latency extraction from all timing windows. This run is enough to choose candidates, but not enough to lock a production default from a single sample per case.
|
||||
Regular → Executable
+4
-4
@@ -67,9 +67,9 @@ if [[ "$USE_NETNS" == "1" ]]; then
|
||||
NETEM_LOSS="${NETEM_LOSS:-0}"
|
||||
NETEM_OWD="${NETEM_OWD:-0}"
|
||||
if [[ "$NETEM_LOSS" != "0" || "$NETEM_OWD" != "0" ]]; then
|
||||
"$LK_DIR/scripts/flexfec/netns.sh" shape "$NETEM_LOSS" "$NETEM_OWD" "${NETEM_JITTER:-0}" "${NETEM_CORR:-0}"
|
||||
bash "$LK_DIR/scripts/flexfec/netns.sh" shape "$NETEM_LOSS" "$NETEM_OWD" "${NETEM_JITTER:-0}" "${NETEM_CORR:-0}"
|
||||
else
|
||||
"$LK_DIR/scripts/flexfec/netns.sh" clear
|
||||
bash "$LK_DIR/scripts/flexfec/netns.sh" clear
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -94,13 +94,13 @@ PUB="$RUST_DIR/target/debug/publisher"
|
||||
SUB="$RUST_DIR/target/debug/subscriber"
|
||||
|
||||
# --- Publisher (robot) ---
|
||||
PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity flexfec-pub --test-pattern --animate-test-pattern --attach-timestamp --attach-frame-id)
|
||||
PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity flexfec-pub --test-pattern 1 --attach-timestamp --attach-frame-id)
|
||||
# shellcheck disable=SC2206
|
||||
[[ -n "$PUB_EXTRA" ]] && PUB_ARGS+=($PUB_EXTRA)
|
||||
|
||||
if [[ "$USE_NETNS" == "1" ]]; then
|
||||
# netns.sh sudoes ip/tc internally; do NOT wrap it in sudo (the script isn't allowlisted).
|
||||
"$LK_DIR/scripts/flexfec/netns.sh" exec env \
|
||||
bash "$LK_DIR/scripts/flexfec/netns.sh" exec env \
|
||||
LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$PUB" "${PUB_ARGS[@]}" > "$PUB_LOG" 2>&1 &
|
||||
else
|
||||
|
||||
@@ -42,7 +42,8 @@ PUB_IDENTITY="${PUB_IDENTITY:-flexfec-pub}"
|
||||
|
||||
PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity "$PUB_IDENTITY")
|
||||
if [[ "${TEST_PATTERN:-1}" == "1" ]]; then
|
||||
PUB_ARGS+=(--test-pattern)
|
||||
# TEST_PATTERN_ANIMATE=1 -> animated (scrolling) pattern for a realistic motion bitrate.
|
||||
PUB_ARGS+=(--test-pattern "${TEST_PATTERN_ANIMATE:-0}")
|
||||
fi
|
||||
if [[ "${ATTACH_META:-1}" == "1" ]]; then
|
||||
PUB_ARGS+=(--attach-timestamp --attach-frame-id)
|
||||
|
||||
Executable
+456
@@ -0,0 +1,456 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run a Linux FlexFEC-vs-RTX benchmark sweep.
|
||||
#
|
||||
# The sweep runs two independent legs:
|
||||
# - uplink: publisher/robot -> SFU, shaped through the robot netns veth.
|
||||
# - downlink: SFU -> subscriber/operator, shaped on loopback by SFU UDP source port.
|
||||
#
|
||||
# Each case writes raw logs and appends one row to summary.csv. Defaults are intentionally
|
||||
# small enough to run overnight-ish while still covering RTX-only, low-latency FEC, and
|
||||
# stronger FEC under short fade and longer handoff-style burst loss.
|
||||
#
|
||||
# Run from this repo on Linux:
|
||||
# scripts/flexfec/sweep-linux.sh
|
||||
#
|
||||
# Useful overrides:
|
||||
# RUST_DIR=~/workspace/rust-sdks5
|
||||
# GO_BIN=~/go1.26/bin/go
|
||||
# OUT_DIR=/tmp/flexfec-results
|
||||
# DURATION=60
|
||||
# SCENARIOS="uplink downlink"
|
||||
# FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec10x2:on:10:2"
|
||||
# LOSS_PROFILES="fade:35:250:1750:40:10:70 handoff:80:750:5000:60:20:85"
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
LK_DIR="${LK_DIR:-$REPO_ROOT}"
|
||||
RUST_DIR="${RUST_DIR:-$REPO_ROOT/../rust-sdks5}"
|
||||
SFU_BIN="${SFU_BIN:-/tmp/flexfec-livekit-server}"
|
||||
OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results/$(date +%Y%m%d-%H%M%S)}"
|
||||
|
||||
DURATION="${DURATION:-45}"
|
||||
SFU_BOOT="${SFU_BOOT:-3}"
|
||||
PUB_BOOT="${PUB_BOOT:-4}"
|
||||
SUB_BOOT="${SUB_BOOT:-5}"
|
||||
|
||||
SCENARIOS="${SCENARIOS:-uplink downlink}"
|
||||
FEC_CONFIGS="${FEC_CONFIGS:-rtx:off:0:0 fec5x1:on:5:1 fec8x1:on:8:1 fec10x2:on:10:2}"
|
||||
LOSS_PROFILES="${LOSS_PROFILES:-fade-short:35:250:1750:40:10:70 fade-long:70:750:5000:60:20:85}"
|
||||
|
||||
ROOM_PREFIX="${ROOM_PREFIX:-flexfec-bench}"
|
||||
RUST_LOG="${RUST_LOG:-info}"
|
||||
PAYLOAD_TYPE="${PAYLOAD_TYPE:-49}"
|
||||
GO_BIN="${GO_BIN:-go}"
|
||||
REBUILD_RUST="${REBUILD_RUST:-0}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
PUB_BIN="$RUST_DIR/target/debug/publisher"
|
||||
SUB_BIN="$RUST_DIR/target/debug/subscriber"
|
||||
SUMMARY_CSV="$OUT_DIR/summary.csv"
|
||||
|
||||
declare -a CASE_PIDS=()
|
||||
IMPAIR_PID=""
|
||||
NETNS_ACTIVE=0
|
||||
CURRENT_LEG=""
|
||||
|
||||
if [[ "$(uname -s)" != "Linux" ]]; then
|
||||
echo "error: sweep-linux.sh must run on Linux" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
SUDO=()
|
||||
elif [[ -n "${SUDO_ASKPASS:-}" ]]; then
|
||||
SUDO=(sudo -A)
|
||||
else
|
||||
SUDO=(sudo)
|
||||
fi
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
safe_name() {
|
||||
printf '%s' "$1" | tr -c 'A-Za-z0-9_.=-' '_'
|
||||
}
|
||||
|
||||
ms_to_seconds() {
|
||||
awk "BEGIN { printf \"%.3f\", $1 / 1000 }"
|
||||
}
|
||||
|
||||
csv_quote() {
|
||||
local s="${1:-}"
|
||||
s="${s//\"/\"\"}"
|
||||
printf '"%s"' "$s"
|
||||
}
|
||||
|
||||
run_netns() {
|
||||
"${SUDO[@]}" bash "$SCRIPT_DIR/netns.sh" "$@"
|
||||
}
|
||||
|
||||
run_netem() {
|
||||
"${SUDO[@]}" bash "$SCRIPT_DIR/netem.sh" "$@"
|
||||
}
|
||||
|
||||
case_contains_scenario() {
|
||||
local want="$1"
|
||||
for scenario in $SCENARIOS; do
|
||||
[[ "$scenario" == "$want" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
prepare_sudo() {
|
||||
if [[ "${#SUDO[@]}" -gt 0 ]]; then
|
||||
log "checking sudo access for tc/ip netem setup"
|
||||
"${SUDO[@]}" -v
|
||||
fi
|
||||
}
|
||||
|
||||
build_artifacts() {
|
||||
[[ -d "$RUST_DIR" ]] || die "rust-sdks checkout not found at $RUST_DIR (set RUST_DIR)"
|
||||
|
||||
log "building SFU: $SFU_BIN"
|
||||
(cd "$LK_DIR" && "$GO_BIN" build -o "$SFU_BIN" ./cmd/server)
|
||||
|
||||
if [[ "$REBUILD_RUST" == "1" || ! -x "$PUB_BIN" || ! -x "$SUB_BIN" ]]; then
|
||||
log "building local_video publisher/subscriber in $RUST_DIR"
|
||||
(cd "$RUST_DIR" && cargo build -p local_video --features desktop --bin publisher --bin subscriber)
|
||||
else
|
||||
log "using existing local_video binaries in $RUST_DIR/target/debug"
|
||||
fi
|
||||
}
|
||||
|
||||
write_summary_header() {
|
||||
mkdir -p "$OUT_DIR"
|
||||
cat >"$SUMMARY_CSV" <<'CSV'
|
||||
timestamp,leg,profile,fec_label,fec_enabled,num_media_packets,num_fec_packets,loss_pct,burst_ms,gap_ms,one_way_delay_ms,jitter_ms,loss_correlation_pct,sfu_packets_recovered,subscriber_fec_accepted_total,video_quality_last,frame_latency_last,case_dir
|
||||
CSV
|
||||
}
|
||||
|
||||
write_config() {
|
||||
local leg="$1"
|
||||
local fec_enabled="$2"
|
||||
local num_media="$3"
|
||||
local num_fec="$4"
|
||||
local config="$5"
|
||||
|
||||
local publisher=false
|
||||
local subscriber=false
|
||||
local bind_addresses=""
|
||||
local includes=""
|
||||
|
||||
if [[ "$leg" == "uplink" ]]; then
|
||||
publisher="$fec_enabled"
|
||||
bind_addresses=$'bind_addresses:\n - 10.200.0.1\n - 127.0.0.1'
|
||||
includes=$' - 10.200.0.0/24\n - 127.0.0.0/8'
|
||||
else
|
||||
subscriber="$fec_enabled"
|
||||
bind_addresses=$'bind_addresses:\n - 127.0.0.1'
|
||||
includes=$' - 127.0.0.0/8'
|
||||
fi
|
||||
|
||||
if [[ "$fec_enabled" == "false" ]]; then
|
||||
num_media=5
|
||||
num_fec=1
|
||||
fi
|
||||
|
||||
cat >"$config" <<EOF
|
||||
port: 7880
|
||||
|
||||
$bind_addresses
|
||||
|
||||
rtc:
|
||||
udp_port: 7882
|
||||
tcp_port: 7881
|
||||
use_external_ip: false
|
||||
enable_loopback_candidate: true
|
||||
ips:
|
||||
includes:
|
||||
$includes
|
||||
|
||||
flexfec:
|
||||
subscriber: $subscriber
|
||||
publisher: $publisher
|
||||
payload_type: $PAYLOAD_TYPE
|
||||
num_media_packets: $num_media
|
||||
num_fec_packets: $num_fec
|
||||
|
||||
logging:
|
||||
level: debug
|
||||
json: false
|
||||
EOF
|
||||
}
|
||||
|
||||
stop_impairment() {
|
||||
set +e
|
||||
if [[ -n "${IMPAIR_PID:-}" ]]; then
|
||||
kill "$IMPAIR_PID" 2>/dev/null
|
||||
for _ in {1..20}; do
|
||||
kill -0 "$IMPAIR_PID" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
kill -9 "$IMPAIR_PID" 2>/dev/null
|
||||
wait "$IMPAIR_PID" 2>/dev/null
|
||||
IMPAIR_PID=""
|
||||
fi
|
||||
|
||||
case "$CURRENT_LEG" in
|
||||
uplink) run_netns clear >/dev/null 2>&1 ;;
|
||||
downlink) run_netem stop >/dev/null 2>&1 ;;
|
||||
esac
|
||||
set -e
|
||||
}
|
||||
|
||||
stop_case_processes() {
|
||||
set +e
|
||||
if [[ "${#CASE_PIDS[@]}" -gt 0 ]]; then
|
||||
kill "${CASE_PIDS[@]}" 2>/dev/null
|
||||
sleep 1
|
||||
kill -9 "${CASE_PIDS[@]}" 2>/dev/null
|
||||
wait "${CASE_PIDS[@]}" 2>/dev/null
|
||||
fi
|
||||
CASE_PIDS=()
|
||||
|
||||
pkill -f "$SFU_BIN" 2>/dev/null
|
||||
pkill -f "$PUB_BIN" 2>/dev/null
|
||||
pkill -f "$SUB_BIN" 2>/dev/null
|
||||
set -e
|
||||
}
|
||||
|
||||
cleanup_all() {
|
||||
set +e
|
||||
stop_impairment
|
||||
set +e
|
||||
stop_case_processes
|
||||
set +e
|
||||
if [[ "$NETNS_ACTIVE" == "1" ]]; then
|
||||
run_netns down >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
start_uplink_burst() {
|
||||
local case_dir="$1"
|
||||
local loss="$2"
|
||||
local burst_ms="$3"
|
||||
local gap_ms="$4"
|
||||
local owd_ms="$5"
|
||||
local jitter_ms="$6"
|
||||
local corr_pct="$7"
|
||||
local burst_s gap_s
|
||||
burst_s="$(ms_to_seconds "$burst_ms")"
|
||||
gap_s="$(ms_to_seconds "$gap_ms")"
|
||||
|
||||
(
|
||||
trap 'exit 0' INT TERM
|
||||
while true; do
|
||||
run_netns shape "$loss" "$owd_ms" "$jitter_ms" "$corr_pct"
|
||||
sleep "$burst_s"
|
||||
run_netns shape 0 "$owd_ms" "$jitter_ms" 0
|
||||
sleep "$gap_s"
|
||||
done
|
||||
) >"$case_dir/netem.log" 2>&1 &
|
||||
IMPAIR_PID="$!"
|
||||
}
|
||||
|
||||
start_downlink_burst() {
|
||||
local case_dir="$1"
|
||||
local loss="$2"
|
||||
local burst_ms="$3"
|
||||
local gap_ms="$4"
|
||||
local owd_ms="$5"
|
||||
|
||||
run_netem burst "$loss" "$burst_ms" "$gap_ms" down "$owd_ms" >"$case_dir/netem.log" 2>&1 &
|
||||
IMPAIR_PID="$!"
|
||||
}
|
||||
|
||||
start_impairment() {
|
||||
local leg="$1"
|
||||
local case_dir="$2"
|
||||
local loss="$3"
|
||||
local burst_ms="$4"
|
||||
local gap_ms="$5"
|
||||
local owd_ms="$6"
|
||||
local jitter_ms="$7"
|
||||
local corr_pct="$8"
|
||||
|
||||
CURRENT_LEG="$leg"
|
||||
if [[ "$leg" == "uplink" ]]; then
|
||||
start_uplink_burst "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct"
|
||||
else
|
||||
start_downlink_burst "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms"
|
||||
fi
|
||||
}
|
||||
|
||||
last_line() {
|
||||
local pattern="$1"
|
||||
local file="$2"
|
||||
grep -iE "$pattern" "$file" 2>/dev/null | tail -1 || true
|
||||
}
|
||||
|
||||
extract_key_number() {
|
||||
local key="$1"
|
||||
local line="$2"
|
||||
printf '%s' "$line" | sed -nE "s/.*${key}\"?[:= ]+([0-9]+).*/\1/p"
|
||||
}
|
||||
|
||||
summarize_case() {
|
||||
local leg="$1"
|
||||
local profile="$2"
|
||||
local fec_label="$3"
|
||||
local fec_enabled="$4"
|
||||
local num_media="$5"
|
||||
local num_fec="$6"
|
||||
local loss="$7"
|
||||
local burst_ms="$8"
|
||||
local gap_ms="$9"
|
||||
local owd_ms="${10}"
|
||||
local jitter_ms="${11}"
|
||||
local corr_pct="${12}"
|
||||
local case_dir="${13}"
|
||||
|
||||
local sfu_log="$case_dir/sfu.log"
|
||||
local sub_log="$case_dir/subscriber.log"
|
||||
local sfu_recovery_line video_quality frame_latency accepted_line recovered accepted
|
||||
|
||||
sfu_recovery_line="$(last_line 'flexfec recovery stats' "$sfu_log")"
|
||||
video_quality="$(last_line 'Video quality' "$sub_log")"
|
||||
frame_latency="$(last_line 'frame latency' "$sub_log")"
|
||||
accepted_line="$(last_line 'accepted_recovery_payload_total|accepted_total|Video FEC recovery payload accepted' "$sub_log")"
|
||||
|
||||
recovered="$(extract_key_number 'packetsRecovered' "$sfu_recovery_line")"
|
||||
accepted="$(extract_key_number 'accepted_recovery_payload_total' "$accepted_line")"
|
||||
if [[ -z "$accepted" ]]; then
|
||||
accepted="$(extract_key_number 'accepted_total' "$accepted_line")"
|
||||
fi
|
||||
|
||||
{
|
||||
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,' \
|
||||
"$(date -Iseconds)" "$leg" "$profile" "$fec_label" "$fec_enabled" \
|
||||
"$num_media" "$num_fec" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" \
|
||||
"${recovered:-0}" "${accepted:-0}"
|
||||
csv_quote "$video_quality"
|
||||
printf ','
|
||||
csv_quote "$frame_latency"
|
||||
printf ','
|
||||
csv_quote "$case_dir"
|
||||
printf '\n'
|
||||
} >>"$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
run_case() {
|
||||
local leg="$1"
|
||||
local config_spec="$2"
|
||||
local profile_spec="$3"
|
||||
local fec_label fec_mode num_media num_fec
|
||||
local profile loss burst_ms gap_ms owd_ms jitter_ms corr_pct
|
||||
|
||||
IFS=: read -r fec_label fec_mode num_media num_fec <<<"$config_spec"
|
||||
IFS=: read -r profile loss burst_ms gap_ms owd_ms jitter_ms corr_pct <<<"$profile_spec"
|
||||
|
||||
local fec_enabled=false
|
||||
[[ "$fec_mode" == "on" ]] && fec_enabled=true
|
||||
|
||||
local case_name case_dir config sfu_host room
|
||||
case_name="$(safe_name "${leg}_${profile}_${fec_label}")"
|
||||
case_dir="$OUT_DIR/$case_name"
|
||||
config="$case_dir/sfu.yaml"
|
||||
room="${ROOM_PREFIX}-${case_name}"
|
||||
mkdir -p "$case_dir"
|
||||
write_config "$leg" "$fec_enabled" "$num_media" "$num_fec" "$config"
|
||||
|
||||
if [[ "$leg" == "uplink" ]]; then
|
||||
sfu_host="10.200.0.1"
|
||||
run_netns clear >/dev/null 2>&1 || true
|
||||
else
|
||||
sfu_host="127.0.0.1"
|
||||
run_netem stop >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
log "case: leg=$leg profile=$profile fec=$fec_label loss=${loss}% burst=${burst_ms}ms gap=${gap_ms}ms owd=${owd_ms}ms"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
stop_case_processes
|
||||
CASE_PIDS=()
|
||||
IMPAIR_PID=""
|
||||
CURRENT_LEG=""
|
||||
|
||||
(
|
||||
cd "$LK_DIR"
|
||||
"$SFU_BIN" --dev --config "$config"
|
||||
) >"$case_dir/sfu.log" 2>&1 &
|
||||
CASE_PIDS+=("$!")
|
||||
sleep "$SFU_BOOT"
|
||||
|
||||
local url="ws://$sfu_host:7880"
|
||||
local pub_args=(--flex-fec --room-name "$room" --identity "${case_name}-pub" --test-pattern 1 --attach-timestamp --attach-frame-id)
|
||||
if [[ "$leg" == "uplink" ]]; then
|
||||
run_netns exec env \
|
||||
LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$PUB_BIN" "${pub_args[@]}" >"$case_dir/publisher.log" 2>&1 &
|
||||
else
|
||||
LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$PUB_BIN" "${pub_args[@]}" >"$case_dir/publisher.log" 2>&1 &
|
||||
fi
|
||||
CASE_PIDS+=("$!")
|
||||
sleep "$PUB_BOOT"
|
||||
|
||||
local sub_display sub_xauth
|
||||
sub_display="${DISPLAY:-${SUB_DISPLAY:-:1}}"
|
||||
sub_xauth="${XAUTHORITY:-${SUB_XAUTHORITY:-/run/user/$(id -u)/gdm/Xauthority}}"
|
||||
DISPLAY="$sub_display" XAUTHORITY="$sub_xauth" \
|
||||
LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$SUB_BIN" --room-name "$room" --identity "${case_name}-sub" >"$case_dir/subscriber.log" 2>&1 &
|
||||
CASE_PIDS+=("$!")
|
||||
sleep "$SUB_BOOT"
|
||||
|
||||
start_impairment "$leg" "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct"
|
||||
sleep "$DURATION"
|
||||
|
||||
stop_impairment
|
||||
stop_case_processes
|
||||
summarize_case "$leg" "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" \
|
||||
"$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" "$case_dir"
|
||||
}
|
||||
|
||||
main() {
|
||||
prepare_sudo
|
||||
build_artifacts
|
||||
write_summary_header
|
||||
|
||||
if case_contains_scenario uplink; then
|
||||
log "setting up robot netns for uplink benchmarks"
|
||||
run_netns up
|
||||
NETNS_ACTIVE=1
|
||||
fi
|
||||
|
||||
log "results: $OUT_DIR"
|
||||
for scenario in $SCENARIOS; do
|
||||
case "$scenario" in
|
||||
uplink | downlink) ;;
|
||||
*) die "unknown scenario '$scenario' (use uplink/downlink)" ;;
|
||||
esac
|
||||
for profile_spec in $LOSS_PROFILES; do
|
||||
for config_spec in $FEC_CONFIGS; do
|
||||
run_case "$scenario" "$config_spec" "$profile_spec"
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
cleanup_all
|
||||
log "summary: $SUMMARY_CSV"
|
||||
}
|
||||
|
||||
trap cleanup_all EXIT INT TERM
|
||||
main "$@"
|
||||
Executable
+339
@@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Focused tele-op publish-leg benchmark:
|
||||
# - this computer runs local_video publisher + subscriber
|
||||
# - DC-Linux runs the SFU
|
||||
# - the SFU applies publish-side packet loss only, with no added delay
|
||||
#
|
||||
# This models a robot publishing over an already-real public/LAN path where the link RTT is
|
||||
# real and should not be inflated by the test harness. Subscriber/downlink metrics are still
|
||||
# collected, but FlexFEC generation toward the subscriber is disabled by default.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
REMOTE="${REMOTE:-dc@DC-Linux}"
|
||||
REMOTE_LK_DIR="${REMOTE_LK_DIR:-/home/dc/workspace/livekit-flexfec-bench}"
|
||||
REMOTE_GO_BIN="${REMOTE_GO_BIN:-/home/dc/go1.26/bin/go}"
|
||||
REMOTE_SFU_BIN="${REMOTE_SFU_BIN:-/tmp/flexfec-livekit-server}"
|
||||
REMOTE_WORK_DIR="${REMOTE_WORK_DIR:-/tmp/flexfec-remote-publish}"
|
||||
REMOTE_BIND_ADDRESSES="${REMOTE_BIND_ADDRESSES:-10.104.0.88 100.69.171.8 127.0.0.1}"
|
||||
REMOTE_ICE_INCLUDES="${REMOTE_ICE_INCLUDES:-10.104.0.0/16 100.64.0.0/10 127.0.0.0/8}"
|
||||
SSH_OPTS="${SSH_OPTS:--o ControlPath=/tmp/dc-linux-ctrl}"
|
||||
|
||||
RUST_DIR="${RUST_DIR:-$REPO_ROOT/../rust-sdks5}"
|
||||
PUB_BIN="${PUB_BIN:-$RUST_DIR/target/debug/publisher}"
|
||||
SUB_BIN="${SUB_BIN:-$RUST_DIR/target/debug/subscriber}"
|
||||
SFU_HOST="${SFU_HOST:-DC-Linux}"
|
||||
LIVEKIT_URL="${LIVEKIT_URL:-ws://$SFU_HOST:7880}"
|
||||
|
||||
OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results/remote-publish-$(date +%Y%m%d-%H%M%S)}"
|
||||
SUMMARY_CSV="$OUT_DIR/summary.csv"
|
||||
|
||||
DURATION="${DURATION:-60}"
|
||||
SFU_BOOT="${SFU_BOOT:-3}"
|
||||
SUB_BOOT="${SUB_BOOT:-4}"
|
||||
PUB_BOOT="${PUB_BOOT:-4}"
|
||||
ROOM_PREFIX="${ROOM_PREFIX:-teleop-publish}"
|
||||
RUST_LOG="${RUST_LOG:-info}"
|
||||
PAYLOAD_TYPE="${PAYLOAD_TYPE:-49}"
|
||||
DROP_LOG="${DROP_LOG:-1}"
|
||||
REBUILD_RUST="${REBUILD_RUST:-0}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
FEC_CONFIGS="${FEC_CONFIGS:-rtx:off:0:0 fec6x1:on:6:1 fec8x1:on:8:1 fec5x1:on:5:1}"
|
||||
LOSS_PCTS="${LOSS_PCTS:-0.1 0.5 1 2 5}"
|
||||
BURST_MS="${BURST_MS:-100 200 500}"
|
||||
BURST_GAP_MS="${BURST_GAP_MS:-2000}"
|
||||
|
||||
PUB_EXTRA="${PUB_EXTRA:---codec vp8 --encoder software}"
|
||||
SUB_EXTRA="${SUB_EXTRA:-}"
|
||||
|
||||
declare -a LOCAL_PIDS=()
|
||||
REMOTE_SFU_PID=""
|
||||
REMOTE_CASE_DIR=""
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ssh_remote() {
|
||||
# shellcheck disable=SC2086
|
||||
ssh $SSH_OPTS "$REMOTE" "$@"
|
||||
}
|
||||
|
||||
rsync_from_remote() {
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
# shellcheck disable=SC2086
|
||||
rsync -az -e "ssh $SSH_OPTS" "$REMOTE:$src" "$dst"
|
||||
}
|
||||
|
||||
safe_name() {
|
||||
printf '%s' "$1" | tr -c 'A-Za-z0-9_.=-' '_'
|
||||
}
|
||||
|
||||
csv_quote() {
|
||||
local s="${1:-}"
|
||||
s="${s//\"/\"\"}"
|
||||
printf '"%s"' "$s"
|
||||
}
|
||||
|
||||
shell_quote() {
|
||||
printf '%q' "$1"
|
||||
}
|
||||
|
||||
loss_fraction() {
|
||||
awk "BEGIN { printf \"%.6f\", $1 / 100 }"
|
||||
}
|
||||
|
||||
write_summary_header() {
|
||||
mkdir -p "$OUT_DIR"
|
||||
cat >"$SUMMARY_CSV" <<'CSV'
|
||||
timestamp,profile,fec_label,fec_enabled,num_media_packets,num_fec_packets,loss_pct,burst_ms,gap_ms,sfu_packets_recovered,sfu_drop_count,sfu_marker_count,subscriber_fec_accepted_total,video_quality_last,frame_latency_last,case_dir
|
||||
CSV
|
||||
}
|
||||
|
||||
write_remote_config() {
|
||||
local fec_enabled="$1"
|
||||
local num_media="$2"
|
||||
local num_fec="$3"
|
||||
local remote_config="$4"
|
||||
local bind_addresses_yaml=""
|
||||
local ice_includes_yaml=""
|
||||
|
||||
if [[ "$fec_enabled" == "false" ]]; then
|
||||
num_media=5
|
||||
num_fec=1
|
||||
fi
|
||||
|
||||
local bind_address
|
||||
for bind_address in $REMOTE_BIND_ADDRESSES; do
|
||||
bind_addresses_yaml+=" - $bind_address"$'\n'
|
||||
done
|
||||
|
||||
local ice_include
|
||||
for ice_include in $REMOTE_ICE_INCLUDES; do
|
||||
ice_includes_yaml+=" - $ice_include"$'\n'
|
||||
done
|
||||
|
||||
ssh_remote "cat > '$remote_config' <<'EOF'
|
||||
port: 7880
|
||||
|
||||
bind_addresses:
|
||||
$bind_addresses_yaml
|
||||
|
||||
rtc:
|
||||
udp_port: 7882
|
||||
tcp_port: 7881
|
||||
use_external_ip: false
|
||||
enable_loopback_candidate: true
|
||||
ips:
|
||||
includes:
|
||||
$ice_includes_yaml
|
||||
|
||||
flexfec:
|
||||
subscriber: false
|
||||
publisher: $fec_enabled
|
||||
payload_type: $PAYLOAD_TYPE
|
||||
num_media_packets: $num_media
|
||||
num_fec_packets: $num_fec
|
||||
|
||||
logging:
|
||||
level: debug
|
||||
json: false
|
||||
EOF"
|
||||
}
|
||||
|
||||
stop_local_processes() {
|
||||
set +e
|
||||
if [[ "${#LOCAL_PIDS[@]}" -gt 0 ]]; then
|
||||
kill "${LOCAL_PIDS[@]}" 2>/dev/null
|
||||
sleep 1
|
||||
kill -9 "${LOCAL_PIDS[@]}" 2>/dev/null
|
||||
wait "${LOCAL_PIDS[@]}" 2>/dev/null
|
||||
fi
|
||||
LOCAL_PIDS=()
|
||||
set -e
|
||||
}
|
||||
|
||||
stop_remote_sfu() {
|
||||
set +e
|
||||
if [[ -n "${REMOTE_SFU_PID:-}" ]]; then
|
||||
ssh_remote "kill '$REMOTE_SFU_PID' 2>/dev/null || true; sleep 1; kill -9 '$REMOTE_SFU_PID' 2>/dev/null || true; wait '$REMOTE_SFU_PID' 2>/dev/null || true" >/dev/null 2>&1
|
||||
REMOTE_SFU_PID=""
|
||||
fi
|
||||
ssh_remote "pkill -f '$REMOTE_SFU_BIN' 2>/dev/null || true" >/dev/null 2>&1
|
||||
set -e
|
||||
}
|
||||
|
||||
cleanup_all() {
|
||||
set +e
|
||||
stop_local_processes
|
||||
stop_remote_sfu
|
||||
}
|
||||
|
||||
last_line() {
|
||||
local pattern="$1"
|
||||
local file="$2"
|
||||
grep -iE "$pattern" "$file" 2>/dev/null | tail -1 || true
|
||||
}
|
||||
|
||||
extract_key_number() {
|
||||
local key="$1"
|
||||
local line="$2"
|
||||
printf '%s' "$line" | sed -nE "s/.*${key}\"?[:= ]+([0-9]+).*/\1/p"
|
||||
}
|
||||
|
||||
summarize_case() {
|
||||
local profile="$1"
|
||||
local fec_label="$2"
|
||||
local fec_enabled="$3"
|
||||
local num_media="$4"
|
||||
local num_fec="$5"
|
||||
local loss="$6"
|
||||
local burst_ms="$7"
|
||||
local gap_ms="$8"
|
||||
local case_dir="$9"
|
||||
|
||||
local sfu_log="$case_dir/sfu.log"
|
||||
local sub_log="$case_dir/subscriber.log"
|
||||
local sfu_recovery_line video_quality frame_latency accepted_line recovered accepted drops markers
|
||||
|
||||
sfu_recovery_line="$(last_line 'flexfec recovery stats' "$sfu_log")"
|
||||
video_quality="$(last_line 'Video quality' "$sub_log")"
|
||||
frame_latency="$(last_line 'frame latency' "$sub_log")"
|
||||
accepted_line="$(last_line 'accepted_recovery_payload_total|accepted_total|Video FEC recovery payload accepted' "$sub_log")"
|
||||
|
||||
recovered="$(extract_key_number 'packetsRecovered' "$sfu_recovery_line")"
|
||||
accepted="$(extract_key_number 'accepted_recovery_payload_total' "$accepted_line")"
|
||||
if [[ -z "$accepted" ]]; then
|
||||
accepted="$(extract_key_number 'accepted_total' "$accepted_line")"
|
||||
fi
|
||||
drops="$(grep -c 'uplink impairment dropped RTP packet' "$sfu_log" 2>/dev/null || true)"
|
||||
markers="$(grep -c 'uplink impairment received frame marker' "$sfu_log" 2>/dev/null || true)"
|
||||
|
||||
{
|
||||
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,' \
|
||||
"$(date -Iseconds)" "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" \
|
||||
"$loss" "$burst_ms" "$gap_ms" "${recovered:-0}" "${drops:-0}" "${markers:-0}" "${accepted:-0}"
|
||||
csv_quote "$video_quality"
|
||||
printf ','
|
||||
csv_quote "$frame_latency"
|
||||
printf ','
|
||||
csv_quote "$case_dir"
|
||||
printf '\n'
|
||||
} >>"$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
build_artifacts() {
|
||||
[[ -d "$RUST_DIR" ]] || die "rust-sdks checkout not found at $RUST_DIR (set RUST_DIR)"
|
||||
|
||||
log "building remote SFU: $REMOTE:$REMOTE_SFU_BIN"
|
||||
ssh_remote "cd '$REMOTE_LK_DIR' && '$REMOTE_GO_BIN' build -o '$REMOTE_SFU_BIN' ./cmd/server"
|
||||
|
||||
if [[ "$REBUILD_RUST" == "1" || ! -x "$PUB_BIN" || ! -x "$SUB_BIN" ]]; then
|
||||
log "building local_video publisher/subscriber in $RUST_DIR"
|
||||
(cd "$RUST_DIR" && cargo build -p local_video --features desktop --bin publisher --bin subscriber)
|
||||
else
|
||||
log "using existing local_video binaries in $RUST_DIR/target/debug"
|
||||
fi
|
||||
}
|
||||
|
||||
make_profiles() {
|
||||
for loss in $LOSS_PCTS; do
|
||||
for burst in $BURST_MS; do
|
||||
printf 'loss%s_b%s:%s:%s:%s\n' \
|
||||
"$(printf '%s' "$loss" | tr '.' 'p')" "$burst" "$loss" "$burst" "$BURST_GAP_MS"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
run_case() {
|
||||
local config_spec="$1"
|
||||
local profile_spec="$2"
|
||||
local fec_label fec_mode num_media num_fec profile loss burst_ms gap_ms
|
||||
IFS=: read -r fec_label fec_mode num_media num_fec <<<"$config_spec"
|
||||
IFS=: read -r profile loss burst_ms gap_ms <<<"$profile_spec"
|
||||
|
||||
local fec_enabled=false
|
||||
[[ "$fec_mode" == "on" ]] && fec_enabled=true
|
||||
|
||||
local case_name case_dir remote_case config room pub_args
|
||||
case_name="$(safe_name "${profile}_${fec_label}")"
|
||||
case_dir="$OUT_DIR/$case_name"
|
||||
remote_case="$REMOTE_WORK_DIR/$case_name"
|
||||
config="$remote_case/sfu.yaml"
|
||||
room="${ROOM_PREFIX}-${case_name}"
|
||||
mkdir -p "$case_dir"
|
||||
|
||||
log "case: profile=$profile fec=$fec_label loss=${loss}% burst=${burst_ms}ms gap=${gap_ms}ms url=$LIVEKIT_URL"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
stop_local_processes
|
||||
stop_remote_sfu
|
||||
LOCAL_PIDS=()
|
||||
|
||||
ssh_remote "mkdir -p '$remote_case'"
|
||||
write_remote_config "$fec_enabled" "$num_media" "$num_fec" "$config"
|
||||
|
||||
local loss_frac
|
||||
loss_frac="$(loss_fraction "$loss")"
|
||||
local remote_start_cmd
|
||||
remote_start_cmd="cd $(shell_quote "$REMOTE_LK_DIR") || exit; env LK_PUB_LOSS=$(shell_quote "$loss_frac") LK_PUB_DELAY_MS=0 LK_PUB_BURST_MS=$(shell_quote "$burst_ms") LK_PUB_GAP_MS=$(shell_quote "$gap_ms") LK_PUB_DROP_LOG=$(shell_quote "$DROP_LOG") $(shell_quote "$REMOTE_SFU_BIN") --dev --config $(shell_quote "$config") > $(shell_quote "$remote_case/sfu.log") 2>&1 < /dev/null & pid=\$!; disown \"\$pid\"; echo \"\$pid\""
|
||||
REMOTE_SFU_PID="$(
|
||||
ssh_remote "bash -lc $(shell_quote "$remote_start_cmd")"
|
||||
)"
|
||||
sleep "$SFU_BOOT"
|
||||
|
||||
LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$SUB_BIN" --room-name "$room" --identity "${case_name}-sub" $SUB_EXTRA >"$case_dir/subscriber.log" 2>&1 &
|
||||
LOCAL_PIDS+=("$!")
|
||||
sleep "$SUB_BOOT"
|
||||
|
||||
pub_args=(--room-name "$room" --identity "${case_name}-pub" --test-pattern 1 --attach-timestamp --attach-frame-id)
|
||||
if [[ "$fec_enabled" == "true" ]]; then
|
||||
pub_args=(--flex-fec "${pub_args[@]}")
|
||||
fi
|
||||
LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \
|
||||
"$PUB_BIN" "${pub_args[@]}" $PUB_EXTRA >"$case_dir/publisher.log" 2>&1 &
|
||||
LOCAL_PIDS+=("$!")
|
||||
sleep "$PUB_BOOT"
|
||||
|
||||
sleep "$DURATION"
|
||||
|
||||
stop_local_processes
|
||||
stop_remote_sfu
|
||||
rsync_from_remote "$remote_case/" "$case_dir/"
|
||||
grep 'uplink impairment dropped RTP packet' "$case_dir/sfu.log" >"$case_dir/dropped-packets.log" 2>/dev/null || true
|
||||
grep 'uplink impairment received frame marker' "$case_dir/sfu.log" >"$case_dir/frame-markers.log" 2>/dev/null || true
|
||||
|
||||
summarize_case "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" "$loss" "$burst_ms" "$gap_ms" "$case_dir"
|
||||
}
|
||||
|
||||
main() {
|
||||
build_artifacts
|
||||
write_summary_header
|
||||
|
||||
log "results: $OUT_DIR"
|
||||
for profile_spec in $(make_profiles); do
|
||||
for config_spec in $FEC_CONFIGS; do
|
||||
run_case "$config_spec" "$profile_spec"
|
||||
done
|
||||
done
|
||||
|
||||
cleanup_all
|
||||
log "summary: $SUMMARY_CSV"
|
||||
}
|
||||
|
||||
trap cleanup_all EXIT INT TERM
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user