From 8ef4cb89e50b065f88ee6fcf075bc5e918b87141 Mon Sep 17 00:00:00 2001 From: David Chen Date: Sun, 7 Jun 2026 19:54:49 -0700 Subject: [PATCH] Add FlexFEC benchmark tooling --- pkg/sfu/buffer/buffer.go | 2 + pkg/sfu/buffer/buffer_impair.go | 175 ++++++- pkg/sfu/buffer/buffer_impair_test.go | 53 ++ pkg/sfu/packettrailer/packet_trailer.go | 57 +++ pkg/sfu/packettrailer/packet_trailer_test.go | 37 +- scripts/flexfec/README.md | 115 +++++ scripts/flexfec/netns.sh | 0 scripts/flexfec/results/.gitignore | 3 + .../flexfec-benchmark-report-20260606.md | 186 +++++++ ...lexfec-teleop-benchmark-report-20260606.md | 180 +++++++ scripts/flexfec/run-leg-linux.sh | 8 +- scripts/flexfec/run-publisher.sh | 3 +- scripts/flexfec/sweep-linux.sh | 456 ++++++++++++++++++ scripts/flexfec/sweep-remote-publish.sh | 339 +++++++++++++ 14 files changed, 1596 insertions(+), 18 deletions(-) create mode 100644 pkg/sfu/buffer/buffer_impair_test.go mode change 100644 => 100755 scripts/flexfec/netns.sh create mode 100644 scripts/flexfec/results/.gitignore create mode 100644 scripts/flexfec/results/flexfec-benchmark-report-20260606.md create mode 100644 scripts/flexfec/results/flexfec-teleop-benchmark-report-20260606.md mode change 100644 => 100755 scripts/flexfec/run-leg-linux.sh create mode 100755 scripts/flexfec/sweep-linux.sh create mode 100755 scripts/flexfec/sweep-remote-publish.sh diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index c91b5dd7e..5e07d5f94 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -171,6 +171,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili func (b *Buffer) Write(pkt []byte) (int, error) { if b.impair != nil { if b.impair.dropInbound() { + b.logImpairedDrop(pkt) // Packet "lost" on the simulated 5G uplink; report success so pion's read // loop is unaffected. return len(pkt), nil @@ -188,6 +189,7 @@ func (b *Buffer) writeNow(pkt []byte) (n int, err error) { if err != nil { return } + b.logImpairedMarker(&rtpPacket) b.Lock() if b.BufferBase.IsClosed() { diff --git a/pkg/sfu/buffer/buffer_impair.go b/pkg/sfu/buffer/buffer_impair.go index 05abc483f..d67eddc66 100644 --- a/pkg/sfu/buffer/buffer_impair.go +++ b/pkg/sfu/buffer/buffer_impair.go @@ -22,6 +22,12 @@ import ( "time" "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/webrtc/v4" + + "github.com/livekit/livekit-server/pkg/sfu/packettrailer" + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/protocol/codecs/mime" ) // Debug-only simulated publisher ("robot") uplink impairment for the local FlexFEC test @@ -46,12 +52,18 @@ import ( // // LK_PUB_LOSS fractional loss on the publisher->SFU path, e.g. 0.03 (3%) // LK_PUB_DELAY_MS one-way publisher<->SFU delay in ms (inbound media + outbound NACK) +// LK_PUB_BURST_MS optional burst-on window; loss only applies inside this window +// LK_PUB_GAP_MS optional burst-off window after LK_PUB_BURST_MS +// LK_PUB_DROP_LOG set to 1 to log dropped RTP packets and received marker frame IDs // // (These are the robot-link knobs. The operator/subscriber-side knobs live in the sfu // package: LK_DOWNLINK_LOSS, LK_DOWNLINK_DELAY_MS, LK_UPLINK_DELAY_MS.) const ( envPubLoss = "LK_PUB_LOSS" envPubDelayMs = "LK_PUB_DELAY_MS" + envPubBurstMs = "LK_PUB_BURST_MS" + envPubGapMs = "LK_PUB_GAP_MS" + envPubDropLog = "LK_PUB_DROP_LOG" impairQueueDepth = 8192 ) @@ -69,8 +81,12 @@ type delayedFeedback struct { } type uplinkImpair struct { - loss float64 - delay time.Duration + loss float64 + delay time.Duration + burst time.Duration + gap time.Duration + started time.Time + logDrops bool rngMu sync.Mutex rng *rand.Rand @@ -91,13 +107,20 @@ func getUplinkImpair() *uplinkImpair { globalImpairOnce.Do(func() { loss := parseEnvFloat(envPubLoss) delay := time.Duration(parseEnvInt(envPubDelayMs)) * time.Millisecond - if loss <= 0 && delay <= 0 { + burst := time.Duration(parseEnvInt(envPubBurstMs)) * time.Millisecond + gap := time.Duration(parseEnvInt(envPubGapMs)) * time.Millisecond + logDrops := parseEnvBool(envPubDropLog) + if loss <= 0 && delay <= 0 && !logDrops { return } imp := &uplinkImpair{ - loss: loss, - delay: delay, - rng: rand.New(rand.NewSource(time.Now().UnixNano())), + loss: loss, + delay: delay, + burst: burst, + gap: gap, + started: time.Now(), + logDrops: logDrops, + rng: rand.New(rand.NewSource(time.Now().UnixNano())), } if delay > 0 { imp.inQueue = make(chan delayedInbound, impairQueueDepth) @@ -115,7 +138,14 @@ func getUplinkImpair() *uplinkImpair { func (b *Buffer) initUplinkImpair() { b.impair = getUplinkImpair() if b.impair != nil && b.logger != nil { - b.logger.Infow("uplink impairment enabled (debug)", "lossFraction", b.impair.loss, "delay", b.impair.delay) + b.logger.Infow( + "uplink impairment enabled (debug)", + "lossFraction", b.impair.loss, + "delay", b.impair.delay, + "burst", b.impair.burst, + "gap", b.impair.gap, + "logDrops", b.impair.logDrops, + ) } } @@ -123,12 +153,29 @@ func (u *uplinkImpair) dropInbound() bool { if u.loss <= 0 { return false } + if !u.inLossWindow(time.Now()) { + return false + } u.rngMu.Lock() r := u.rng.Float64() u.rngMu.Unlock() return r < u.loss } +func (u *uplinkImpair) inLossWindow(now time.Time) bool { + if u.burst <= 0 { + return true + } + if u.gap <= 0 { + return true + } + cycle := u.burst + u.gap + if cycle <= 0 { + return true + } + return now.Sub(u.started)%cycle < u.burst +} + // delayInbound returns true when the packet was deferred onto the shared delay line (caller // should return immediately); false when no delay is configured and the caller should // proceed inline. @@ -202,3 +249,117 @@ func parseEnvInt(key string) int { } return v } + +func parseEnvBool(key string) bool { + v := os.Getenv(key) + return v == "1" || v == "true" || v == "TRUE" || v == "yes" || v == "YES" +} + +func (u *uplinkImpair) shouldLogDrops() bool { + return u != nil && u.logDrops +} + +func (b *Buffer) logImpairedDrop(pkt []byte) { + if b.impair == nil || !b.impair.shouldLogDrops() || b.logger == nil { + return + } + + var rtpPacket rtp.Packet + if err := rtpPacket.Unmarshal(pkt); err != nil { + b.logger.Infow("uplink impairment dropped invalid RTP packet", "err", err) + return + } + + fields := []any{ + "stream", b.impairedStreamKind(), + "ssrc", rtpPacket.SSRC, + "sequenceNumber", rtpPacket.SequenceNumber, + "rtpTimestamp", rtpPacket.Timestamp, + "payloadType", rtpPacket.PayloadType, + "marker", rtpPacket.Marker, + "frameTypeHint", b.frameTypeHint(&rtpPacket), + } + if metadata, ok := packettrailer.ParseTrailer(rtpPacket.Payload, rtpPacket.Marker); ok { + fields = append(fields, + "hasFrameID", metadata.HasFrameID, + "frameID", metadata.FrameID, + "hasUserTimestamp", metadata.HasTimestampUs, + "userTimestampUs", metadata.TimestampUs, + ) + } + + b.logger.Infow("uplink impairment dropped RTP packet", fields...) +} + +func (b *Buffer) logImpairedMarker(pkt *rtp.Packet) { + if b.impair == nil || !b.impair.shouldLogDrops() || b.logger == nil || !pkt.Marker { + return + } + if b.primaryBufferForRTX != nil || b.primaryBufferForFEC != nil { + return + } + + metadata, ok := packettrailer.ParseTrailer(pkt.Payload, pkt.Marker) + if !ok || !metadata.HasFrameID { + return + } + + b.logger.Infow( + "uplink impairment received frame marker", + "ssrc", pkt.SSRC, + "sequenceNumber", pkt.SequenceNumber, + "rtpTimestamp", pkt.Timestamp, + "frameID", metadata.FrameID, + "hasUserTimestamp", metadata.HasTimestampUs, + "userTimestampUs", metadata.TimestampUs, + "frameTypeHint", b.frameTypeHint(pkt), + ) +} + +func (b *Buffer) impairedStreamKind() string { + switch { + case b.primaryBufferForRTX != nil: + return "rtx" + case b.primaryBufferForFEC != nil: + return "fec" + default: + return "media" + } +} + +func (b *Buffer) frameTypeHint(pkt *rtp.Packet) string { + if b.codecType != webrtc.RTPCodecTypeVideo || b.impairedStreamKind() != "media" { + return "unknown" + } + + switch b.mime { + case mime.MimeTypeH264: + if codec.IsH264KeyFrame(pkt.Payload) { + return "I" + } + return "P" + case mime.MimeTypeH265: + if codec.IsH265KeyFrame(pkt.Payload) { + return "I" + } + return "P" + case mime.MimeTypeAV1: + if codec.IsAV1KeyFrame(pkt.Payload) { + return "I" + } + return "P" + case mime.MimeTypeVP8: + var vp8Packet codec.VP8 + if err := vp8Packet.Unmarshal(pkt.Payload); err == nil && vp8Packet.IsKeyFrame { + return "I" + } + return "P" + case mime.MimeTypeVP9: + if codec.IsVP9KeyFrame(nil, pkt.Payload) { + return "I" + } + return "P" + default: + return "unknown" + } +} diff --git a/pkg/sfu/buffer/buffer_impair_test.go b/pkg/sfu/buffer/buffer_impair_test.go new file mode 100644 index 000000000..b6655ba06 --- /dev/null +++ b/pkg/sfu/buffer/buffer_impair_test.go @@ -0,0 +1,53 @@ +package buffer + +import ( + "testing" + "time" +) + +func TestUplinkImpairLossWindow(t *testing.T) { + started := time.Unix(100, 0) + impair := &uplinkImpair{ + burst: 100 * time.Millisecond, + gap: 200 * time.Millisecond, + started: started, + } + + tests := []struct { + name string + at time.Duration + want bool + }{ + {name: "start in burst", at: 0, want: true}, + {name: "inside burst", at: 99 * time.Millisecond, want: true}, + {name: "gap begins", at: 100 * time.Millisecond, want: false}, + {name: "inside gap", at: 250 * time.Millisecond, want: false}, + {name: "next burst", at: 300 * time.Millisecond, want: true}, + {name: "next gap", at: 450 * time.Millisecond, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := impair.inLossWindow(started.Add(tt.at)); got != tt.want { + t.Fatalf("inLossWindow() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestUplinkImpairLossWindowWithoutBurst(t *testing.T) { + impair := &uplinkImpair{started: time.Unix(100, 0)} + + if !impair.inLossWindow(time.Unix(200, 0)) { + t.Fatal("inLossWindow() = false, want true without burst config") + } +} + +func TestUplinkImpairLossWindowWithoutGap(t *testing.T) { + started := time.Unix(100, 0) + impair := &uplinkImpair{burst: 100 * time.Millisecond, started: started} + + if !impair.inLossWindow(started.Add(time.Hour)) { + t.Fatal("inLossWindow() = false, want true when burst is set without gap") + } +} diff --git a/pkg/sfu/packettrailer/packet_trailer.go b/pkg/sfu/packettrailer/packet_trailer.go index 3b58f6681..7bcd89bb4 100644 --- a/pkg/sfu/packettrailer/packet_trailer.go +++ b/pkg/sfu/packettrailer/packet_trailer.go @@ -14,14 +14,26 @@ package packettrailer +import "encoding/binary" + var Magic = [4]byte{'L', 'K', 'T', 'S'} const ( xorByte = 0xFF envelopeSize = 5 // 1B trailer_len + 4B magic + + tagTimestampUs = 0x01 + tagFrameID = 0x02 ) +type Metadata struct { + TimestampUs uint64 + HasTimestampUs bool + FrameID uint32 + HasFrameID bool +} + // StripTrailer returns the number of bytes to strip from the end of an RTP // payload if it contains an LKTS trailer. The trailer is located by checking // for the "LKTS" magic suffix and then reading the XORed trailer_len byte @@ -44,3 +56,48 @@ func StripTrailer(payload []byte, marker bool) int { return trailerLen } + +func ParseTrailer(payload []byte, marker bool) (Metadata, bool) { + strip := StripTrailer(payload, marker) + if strip == 0 { + return Metadata{}, false + } + + var metadata Metadata + tlvEnd := len(payload) - envelopeSize + for i := len(payload) - strip; i < tlvEnd; { + if i+2 > tlvEnd { + return metadata, true + } + tag := payload[i] ^ xorByte + length := int(payload[i+1] ^ xorByte) + i += 2 + if length < 0 || i+length > tlvEnd { + return metadata, true + } + + switch tag { + case tagTimestampUs: + if length == 8 { + var buf [8]byte + for j := range buf { + buf[j] = payload[i+j] ^ xorByte + } + metadata.TimestampUs = binary.BigEndian.Uint64(buf[:]) + metadata.HasTimestampUs = true + } + case tagFrameID: + if length == 4 { + var buf [4]byte + for j := range buf { + buf[j] = payload[i+j] ^ xorByte + } + metadata.FrameID = binary.BigEndian.Uint32(buf[:]) + metadata.HasFrameID = true + } + } + i += length + } + + return metadata, true +} diff --git a/pkg/sfu/packettrailer/packet_trailer_test.go b/pkg/sfu/packettrailer/packet_trailer_test.go index 653b71fa2..81b83497e 100644 --- a/pkg/sfu/packettrailer/packet_trailer_test.go +++ b/pkg/sfu/packettrailer/packet_trailer_test.go @@ -19,11 +19,6 @@ import ( "testing" ) -const ( - tagTimestampUs = 0x01 - tagFrameID = 0x02 -) - // appendTLV appends a single XORed TLV element to dst. func appendTLV(dst []byte, tag byte, value []byte) []byte { dst = append(dst, tag^xorByte, byte(len(value))^xorByte) @@ -78,7 +73,7 @@ func makeTimestampOnlyTrailer(timestampUs int64) []byte { } func TestStripTrailer(t *testing.T) { - fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5 + fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5 tsOnlyTrailerSize := 15 // (1+1+8) + 5 tests := []struct { @@ -180,3 +175,33 @@ func TestStripTrailer(t *testing.T) { }) } } + +func TestParseTrailer(t *testing.T) { + payload := makePayloadWithTrailer(20, 1700000000000000, 42) + + metadata, ok := ParseTrailer(payload, true) + if !ok { + t.Fatal("ParseTrailer() did not find trailer") + } + if !metadata.HasTimestampUs || metadata.TimestampUs != 1700000000000000 { + t.Fatalf("timestamp = (%v, %d), want (true, 1700000000000000)", metadata.HasTimestampUs, metadata.TimestampUs) + } + if !metadata.HasFrameID || metadata.FrameID != 42 { + t.Fatalf("frame id = (%v, %d), want (true, 42)", metadata.HasFrameID, metadata.FrameID) + } + + metadata, ok = ParseTrailer(makeTimestampOnlyTrailer(99), true) + if !ok { + t.Fatal("ParseTrailer() did not find timestamp-only trailer") + } + if !metadata.HasTimestampUs || metadata.TimestampUs != 99 { + t.Fatalf("timestamp = (%v, %d), want (true, 99)", metadata.HasTimestampUs, metadata.TimestampUs) + } + if metadata.HasFrameID { + t.Fatalf("frame id present = %v, want false", metadata.HasFrameID) + } + + if _, ok := ParseTrailer(payload, false); ok { + t.Fatal("ParseTrailer() found trailer without RTP marker") + } +} diff --git a/scripts/flexfec/README.md b/scripts/flexfec/README.md index 63c7a36d5..fcf2a0334 100644 --- a/scripts/flexfec/README.md +++ b/scripts/flexfec/README.md @@ -30,10 +30,14 @@ to UDP `7882` (see `flexfec-local.yaml`) so shaping can target it by port. | File | Purpose | | -------------------- | ---------------------------------------------------- | | `flexfec-local.yaml` | SFU config: flexfec on (pub+sub), fixed loopback port | +| `flexfec-linux.yaml` | SFU config for Linux netns/veth uplink tests | | `run-sfu.sh` | Build + run the SFU (`--dev`, devkey/secret) | | `netem.sh` | Start/stop/inspect packet-loss shaping on udp/7882 | +| `netns.sh` | Linux robot netns + veth shaping for uplink tests | +| `run-leg-linux.sh` | Run one Linux benchmark leg and print log summaries | | `run-publisher.sh` | Run the publisher with `--flex-fec` | | `run-subscriber.sh` | Run the subscriber, surface `Video FEC ...` logs | +| `sweep-linux.sh` | Linux multi-case FlexFEC-vs-RTX benchmark sweep | ## Workflow @@ -127,6 +131,117 @@ With FEC ON you expect fewer `freezes`/`frames_dropped`, lower net `packets_lost meaningful `fec_recv`; with FEC OFF the same loss leans entirely on `rtx_recovered` and (because of the delay) typically shows more freezes and dropped frames. +## Full Linux benchmark sweep + +For a full comparison on a Linux box, use: + +```bash +scripts/flexfec/sweep-linux.sh +``` + +The sweep builds this SFU plus the `local_video` publisher/subscriber, generates one SFU +config per case, runs every case for `DURATION=45` seconds by default, and writes raw logs +plus a summary CSV: + +```text +scripts/flexfec/results// + summary.csv + __/ + sfu.yaml + sfu.log + publisher.log + subscriber.log + netem.log +``` + +Default legs: + +| Leg | Topology | Impairment | Compared signal | +| -------- | --------------------------------------- | ----------------------------------- | --------------- | +| Uplink | publisher in `robot` netns -> SFU | `netns.sh` burst loss on robot veth | SFU `packetsRecovered` | +| Downlink | SFU -> subscriber on loopback/root netns | `netem.sh burst down` on UDP/7882 | subscriber FEC accepted payload + quality/latency lines | + +Default FEC matrix: + +| Label | Meaning | +| --------- | -------------------------- | +| `rtx` | FEC disabled, RTX baseline | +| `fec5x1` | 5 media packets + 1 repair | +| `fec8x1` | 8 media packets + 1 repair | +| `fec10x2` | 10 media packets + 2 repairs | + +Default burst profiles: + +| Label | Loss burst | +| ------------ | ---------------------------------------------------- | +| `fade-short` | 35% loss for 250ms every 1750ms, 40ms one-way delay | +| `fade-long` | 70% loss for 750ms every 5000ms, 60ms one-way delay | + +Useful overrides: + +```bash +RUST_DIR=~/workspace/rust-sdks5 \ +OUT_DIR=/tmp/flexfec-results \ +DURATION=60 \ +SCENARIOS="uplink downlink" \ +FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec10x2:on:10:2" \ +LOSS_PROFILES="fade:35:250:1750:40:10:70 handoff:80:750:5000:60:20:85" \ +scripts/flexfec/sweep-linux.sh +``` + +`LOSS_PROFILES` format: + +```text +label:loss_pct:burst_ms:gap_ms:one_way_delay_ms:jitter_ms:loss_correlation_pct +``` + +For downlink loopback shaping, only `loss_pct`, `burst_ms`, `gap_ms`, and +`one_way_delay_ms` are used. For uplink netns shaping, `jitter_ms` and +`loss_correlation_pct` are also passed to `tc netem` during each burst. + +## Focused remote-SFU publish-leg sweep + +For tele-op tests where this computer runs the publisher/subscriber and `DC-Linux` +runs only the SFU, use: + +```bash +scripts/flexfec/sweep-remote-publish.sh +``` + +This keeps the real network RTT between this machine and `DC-Linux`; the SFU only +injects publisher-side packet loss and does not add latency. Subscriber/downlink FEC is +disabled in the generated SFU config, but subscriber quality and frame-latency logs are +still captured for every case. + +Default matrix: + +| Axis | Values | +| ---- | ------ | +| Loss percent | `0.1 0.5 1 2 5` | +| Burst duration | `100 200 500` ms | +| Burst gap | `2000` ms | +| FEC configs | `rtx`, `fec6x1`, `fec8x1`, `fec5x1` | + +Useful overrides: + +```bash +REMOTE=dc@DC-Linux \ +SFU_HOST=DC-Linux \ +DURATION=90 \ +LOSS_PCTS="0.1 0.5 1 2 5" \ +BURST_MS="100 200 500" \ +BURST_GAP_MS=2000 \ +FEC_CONFIGS="rtx:off:0:0 fec6x1:on:6:1 fec8x1:on:8:1" \ +scripts/flexfec/sweep-remote-publish.sh +``` + +Publisher packet-trailer metadata is always enabled with `--attach-timestamp` and +`--attach-frame-id`. The SFU can log publish-side dropped RTP packets and received marker +packets when `DROP_LOG=1` (the default for this sweep), including RTP timestamp, frame ID +when present on marker packets, and a best-effort `frameTypeHint` (`I`/`P`/`unknown`). +Each case also writes `dropped-packets.log` and `frame-markers.log` extracted from the SFU +log for easier frame/timestamp correlation. + ## Notes & caveats - **Loopback shaping (macOS):** `netem.sh` loads its dummynet rules into a dedicated diff --git a/scripts/flexfec/netns.sh b/scripts/flexfec/netns.sh old mode 100644 new mode 100755 diff --git a/scripts/flexfec/results/.gitignore b/scripts/flexfec/results/.gitignore new file mode 100644 index 000000000..91dee7495 --- /dev/null +++ b/scripts/flexfec/results/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!*.md diff --git a/scripts/flexfec/results/flexfec-benchmark-report-20260606.md b/scripts/flexfec/results/flexfec-benchmark-report-20260606.md new file mode 100644 index 000000000..2850b9fdf --- /dev/null +++ b/scripts/flexfec/results/flexfec-benchmark-report-20260606.md @@ -0,0 +1,186 @@ +# FlexFEC Benchmark Report - 2026-06-06 + +## Summary + +FlexFEC is functionally active on both tested legs: + +- Publisher to SFU: the SFU enabled its FlexFEC decoder and recovered media packets from publisher-side repair packets. +- SFU to subscriber: the subscriber received FlexFEC repair packets and accepted recovered payloads. + +The clearest quality win in this run was on the SFU-to-subscriber leg. Under the long fade profile, RTX-only delivery averaged 1694.3 ms capture-to-decode latency, while FlexFEC reduced that to 295.0 ms with `fec5x1` and 298.1 ms with `fec10x2`. + +The publisher-to-SFU leg also proved correction, but the single-run quality result was more configuration-sensitive. `fec8x1` was the best uplink long-fade result in this run: 146 SFU-recovered packets, 21 subscriber-reported lost packets vs. 195 for RTX-only, and 141.2 ms capture-to-decode average vs. 154.1 ms for RTX-only. + +## Environment + +- Host: `DC-Linux` +- OS: Ubuntu 24.04.2 LTS +- Kernel: `Linux dc-linux 6.11.0-25-generic` +- Go: `/home/dc/go1.26/bin/go`, `go1.26.4 linux/amd64` +- SFU source: synced from local branch `dc/exp/flexfec` at `ec606596` plus local benchmark script changes +- Remote SFU workspace: `/home/dc/workspace/livekit-flexfec-bench` +- Local video binaries: `/home/dc/workspace/rust-sdks5/target/debug/publisher` and `subscriber` + +## Validation + +Passed locally: + +```bash +go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc +bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh scripts/flexfec/run-publisher.sh scripts/flexfec/run-sfu.sh scripts/flexfec/run-subscriber.sh +git diff --check +``` + +Passed on `DC-Linux` after syncing: + +```bash +/home/dc/go1.26/bin/go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc +bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh +``` + +## Methodology + +The sweep ran each case for 45 seconds after publisher and subscriber startup. + +FEC matrix: + +| Label | Meaning | +|---|---| +| `rtx` | FEC disabled, RTX baseline | +| `fec5x1` | 5 media packets, 1 repair packet | +| `fec8x1` | 8 media packets, 1 repair packet | +| `fec10x2` | 10 media packets, 2 repair packets | + +Loss profiles: + +| Profile | Loss | Burst | Gap | One-way delay | Jitter | Correlation | +|---|---:|---:|---:|---:|---:|---:| +| `fade-short` | 35% | 250 ms | 1750 ms | 40 ms | 10 ms | 70% | +| `fade-long` | 70% | 750 ms | 5000 ms | 60 ms | 20 ms | 85% | + +Legs: + +| Leg | Topology | Impairment | Primary correction signal | +|---|---|---|---| +| Publisher to SFU | publisher in `robot` netns -> SFU | `tc netem` on robot veth | SFU `packetsRecovered` | +| SFU to subscriber | SFU -> subscriber on loopback/root netns | `tc netem` on SFU UDP/7882 downlink | Subscriber accepted FEC recovery payloads | + +Raw artifacts: + +- Downlink/full sweep: `scripts/flexfec/results/flexfec-full-20260606-120329/` +- Corrected uplink sweep: `scripts/flexfec/results/flexfec-uplink-rerun-20260606-124334/` +- Superseded uplink sweep with launcher failure: `scripts/flexfec/results/flexfec-uplink-20260606-122520/` + +The first uplink sweep was not used for conclusions because publisher namespace startup failed in FEC cases. The sweep scripts were fixed to invoke helper scripts via `bash`, then the uplink-only sweep was rerun successfully. + +## Publisher to SFU Results + +Subscriber FEC counters are expected to stay zero in this leg because the downlink is clean and subscriber-side FlexFEC is disabled. The relevant correction signal is SFU `packetsRecovered`. + +### Uplink `fade-short` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 1515 | 8 | 40 | 0 | 98.4 ms | 0.0 ms | 99.7 ms | +| `fec5x1` | 212 | 1134 | 12 | 507 | 25 | 102.9 ms | +4.5 ms | 104.9 ms | +| `fec8x1` | 118 | 1361 | 8 | 123 | 4 | 105.1 ms | +6.7 ms | 231.0 ms | +| `fec10x2` | 50 | 1485 | 10 | 88 | 0 | 103.3 ms | +4.9 ms | 209.5 ms | + +Interpretation: + +- All FEC configurations except RTX produced SFU packet recovery. +- RTX-only had the best short-fade quality metrics in this single run. +- `fec8x1` was the least disruptive FEC option in this profile: same freeze count as RTX and only 4 lost packets, while still proving 118 recovered packets at the SFU. + +### Uplink `fade-long` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 1224 | 9 | 239 | 195 | 154.1 ms | 0.0 ms | 156.2 ms | +| `fec5x1` | 32 | 1148 | 9 | 261 | 105 | 395.7 ms | +241.6 ms | 4141.8 ms | +| `fec8x1` | 146 | 1295 | 9 | 213 | 21 | 141.2 ms | -12.9 ms | 142.7 ms | +| `fec10x2` | 0 | 991 | 13 | 262 | 31 | 386.0 ms | +231.9 ms | 4139.4 ms | + +Interpretation: + +- `fec8x1` was the best uplink long-fade configuration in this run. +- `fec8x1` recovered 146 packets at the SFU, decoded more frames than RTX, reduced reported lost packets from 195 to 21, and reduced average capture-to-decode latency by 12.9 ms. +- `fec5x1` and `fec10x2` showed large latency spikes in this run, despite lower reported lost packets than RTX. That makes them poor uplink choices until repeated runs show otherwise. +- `fec10x2` enabled the decoder, but its last SFU recovery counter was zero in this case. That needs repeat testing; burst alignment can dominate a single 45-second run. + +## SFU to Subscriber Results + +Here the correction signal is subscriber-side accepted recovery payloads. The `fec_recv` column comes from the last video quality log line and can be slightly lower than accepted total when the accepted counter logged later. + +### Downlink `fade-short` + +| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 0 | 1425 | 14 | 82 | 57 | 57 | 131.1 ms | 0.0 ms | 270.0 ms | +| `fec5x1` | 325 | 315 | 1392 | 15 | 81 | 63 | 50 | 113.0 ms | -18.1 ms | 254.9 ms | +| `fec8x1` | 203 | 203 | 1450 | 15 | 80 | 60 | 52 | 108.6 ms | -22.5 ms | 255.4 ms | +| `fec10x2` | 327 | 327 | 1452 | 13 | 90 | 69 | 59 | 118.5 ms | -12.6 ms | 270.5 ms | + +Interpretation: + +- All FEC configurations generated accepted recovery payloads at the subscriber. +- Average capture-to-decode latency improved by 12.6 to 22.5 ms vs. RTX-only. +- Freeze counts were broadly similar in this short profile, so latency and accepted recovery are the stronger signals. +- `fec8x1` had the lowest average latency; `fec10x2` decoded the most frames and had the fewest freezes. + +### Downlink `fade-long` + +| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Capture-to-decode max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 0 | 839 | 10 | 284 | 123 | 105 | 1694.3 ms | 0.0 ms | 3262.7 ms | +| `fec5x1` | 307 | 295 | 632 | 12 | 309 | 118 | 101 | 295.0 ms | -1399.3 ms | 3260.3 ms | +| `fec8x1` | 198 | 187 | 843 | 16 | 313 | 125 | 105 | 527.0 ms | -1167.3 ms | 3754.8 ms | +| `fec10x2` | 312 | 300 | 747 | 11 | 307 | 130 | 112 | 298.1 ms | -1396.2 ms | 3259.6 ms | + +Interpretation: + +- Downlink FlexFEC strongly reduced average decode latency under long fades. +- `fec5x1` and `fec10x2` performed similarly on latency: 295.0 ms and 298.1 ms vs. 1694.3 ms for RTX-only. +- `fec8x1` decoded the most frames, but had worse latency and more freezes than the other FEC profiles. +- The high max latency values show there are still isolated stalls; the average latency improvement is still large. + +## Conclusions + +1. FlexFEC correction is proven in both directions. + - Publisher to SFU: SFU logs show the FlexFEC decoder enabled and `packetsRecovered` increasing. + - SFU to subscriber: subscriber logs show FEC received and recovery payloads accepted. + +2. SFU-to-subscriber FEC is clearly useful in this burst-loss setup. + - It substantially reduced average capture-to-decode latency, especially for long fades. + - `fec5x1` looks like the best initial downlink default because it matched `fec10x2` latency with lower nominal repair overhead. + +3. Publisher-to-SFU FEC is useful but needs more repetitions before choosing a default. + - `fec8x1` was the best uplink long-fade configuration in this run. + - `fec5x1` recovered the most short-fade packets but hurt subscriber quality metrics in that case. + - `fec10x2` was inconsistent on uplink and should not be chosen from this single run. + +4. Freeze count alone is not a reliable success metric here. + - Some FEC cases recovered packets and reduced latency without reducing freezes. + - Report `packetsRecovered`, accepted recovery payloads, latency, lost packets, NACK/PLI, and decoded frames together. + +## Recommended Next Sweep + +Run 3 to 5 repetitions per case and compare medians/percentiles. The current single-run results are enough to validate functionality, but burst timing makes per-config ranking noisy. + +Recommended next matrix: + +```bash +DURATION=120 \ +SCENARIOS="uplink downlink" \ +FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec6x1:on:6:1 fec8x1:on:8:1 fec10x1:on:10:1 fec10x2:on:10:2" \ +LOSS_PROFILES="fade-short:35:250:1750:40:10:70 fade-long:70:750:5000:60:20:85 handoff:85:1200:7000:80:30:90" \ +scripts/flexfec/sweep-linux.sh +``` + +Add report fields for: + +- FEC bytes/packets sent to compute overhead. +- P50/P95/P99 capture-to-decode latency from all timing windows, not just the last one. +- SFU ingress loss before and after recovery. +- Subscriber RTP jitter and jitter-buffer delay. +- CPU usage for SFU and publisher/subscriber. diff --git a/scripts/flexfec/results/flexfec-teleop-benchmark-report-20260606.md b/scripts/flexfec/results/flexfec-teleop-benchmark-report-20260606.md new file mode 100644 index 000000000..89a78105f --- /dev/null +++ b/scripts/flexfec/results/flexfec-teleop-benchmark-report-20260606.md @@ -0,0 +1,180 @@ +# FlexFEC Tele-Op Benchmark Report - 2026-06-06 + +## Summary + +Ran a tele-op-oriented benchmark suite on `DC-Linux`: + +- Main path: publisher/robot uplink over cellular-like loss into the SFU. +- Control path: SFU-to-subscriber downlink over clean/mild landline-like loss. + +The result matches the tele-op premise: + +- Uplink FEC is useful under moderate to severe cellular fades, but the right block size depends on the fade shape. +- Downlink FEC showed no practical quality benefit under clean/mild landline conditions; it only adds repair traffic. + +Best uplink candidates from this single run: + +- `fec6x1` for moderate `cell-edge` loss. +- `fec5x1` for severe long handoff where latency matters most. +- `fec8x1` as a conservative general candidate: it behaved well in `cell-edge`, `handoff-long`, and `bursty-uplink`, but was not always the lowest-latency choice. + +Avoid `fec10x2` as a default for tele-op uplink. It recovered packets, but it caused clear latency/quality regressions in `bursty-uplink`. + +## Validation + +Passed locally: + +```bash +go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc +bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh scripts/flexfec/run-publisher.sh scripts/flexfec/run-sfu.sh scripts/flexfec/run-subscriber.sh +``` + +Passed on `DC-Linux` after syncing: + +```bash +/home/dc/go1.26/bin/go test ./pkg/config ./pkg/sfu/flexfec ./pkg/sfu/buffer ./pkg/rtc +bash -n scripts/flexfec/sweep-linux.sh scripts/flexfec/netns.sh scripts/flexfec/netem.sh scripts/flexfec/run-leg-linux.sh +``` + +No launcher failures were found in the copied log artifacts. + +## Raw Artifacts + +- Uplink cellular sweep: `scripts/flexfec/results/flexfec-teleop-uplink-20260606-135721/` +- Downlink landline control: `scripts/flexfec/results/flexfec-teleop-downlink-control-20260606-151840/` + +## Uplink Sweep + +Command shape: + +```bash +DURATION=120 \ +SCENARIOS=uplink \ +FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec6x1:on:6:1 fec8x1:on:8:1 fec10x1:on:10:1 fec10x2:on:10:2" \ +LOSS_PROFILES="clean-control:0:100:2000:20:2:0 urban-5g:3:100:2000:35:8:50 cell-edge:20:250:3000:60:20:75 handoff-short:60:500:8000:80:30:85 handoff-long:85:1200:12000:100:40:90 bursty-uplink:35:300:2500:70:25:85" \ +scripts/flexfec/sweep-linux.sh +``` + +Primary correction signal: SFU `packetsRecovered`. + +### `clean-control` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3645 | 2 | 2 | 0 | 0 | 72.8 ms | 0.0 ms | 73.6 ms | +| `fec5x1` | 0 | 3706 | 2 | 3 | 0 | 0 | 71.0 ms | -1.8 ms | 72.0 ms | +| `fec6x1` | 0 | 3736 | 2 | 2 | 0 | 0 | 71.9 ms | -0.9 ms | 73.0 ms | +| `fec8x1` | 0 | 3676 | 2 | 2 | 0 | 0 | 72.4 ms | -0.4 ms | 73.5 ms | +| `fec10x1` | 498 | 3585 | 3 | 818 | 37 | 1 | 71.9 ms | -0.9 ms | 72.9 ms | +| `fec10x2` | 0 | 3676 | 2 | 5 | 0 | 0 | 71.9 ms | -0.9 ms | 72.7 ms | + +Note: `fec10x1` is anomalous here: a no-loss case produced SFU recovery, high NACK, and lost packets. Treat that row as suspect and rerun before drawing conclusions from it. + +### `urban-5g` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3736 | 2 | 14 | 0 | 0 | 86.9 ms | 0.0 ms | 87.8 ms | +| `fec5x1` | 0 | 3647 | 2 | 10 | 0 | 0 | 87.8 ms | +0.9 ms | 88.5 ms | +| `fec6x1` | 0 | 3676 | 2 | 11 | 0 | 0 | 88.6 ms | +1.7 ms | 89.5 ms | +| `fec8x1` | 20 | 3646 | 2 | 29 | 0 | 0 | 87.8 ms | +0.9 ms | 94.5 ms | +| `fec10x1` | 101 | 3705 | 2 | 34 | 0 | 0 | 87.8 ms | +0.9 ms | 92.8 ms | +| `fec10x2` | 101 | 3706 | 2 | 47 | 0 | 0 | 88.8 ms | +1.9 ms | 90.0 ms | + +Interpretation: RTX/no-FEC is already fine under mild cellular loss. FEC adds little benefit here. + +### `cell-edge` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3608 | 5 | 446 | 0 | 0 | 137.6 ms | 0.0 ms | 138.8 ms | +| `fec5x1` | 145 | 3674 | 5 | 506 | 0 | 0 | 137.1 ms | -0.5 ms | 159.7 ms | +| `fec6x1` | 77 | 3692 | 4 | 443 | 0 | 0 | 134.1 ms | -3.5 ms | 138.8 ms | +| `fec8x1` | 77 | 3734 | 4 | 474 | 0 | 0 | 137.8 ms | +0.2 ms | 147.9 ms | +| `fec10x1` | 0 | 3674 | 4 | 456 | 0 | 0 | 135.0 ms | -2.6 ms | 143.1 ms | +| `fec10x2` | 189 | 3674 | 4 | 568 | 0 | 0 | 137.2 ms | -0.4 ms | 161.5 ms | + +Interpretation: `fec6x1` was the best-balanced result: lowest latency, fewer freezes than RTX, and similar NACK load. + +### `handoff-short` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3465 | 15 | 889 | 7 | 2 | 222.5 ms | 0.0 ms | 789.2 ms | +| `fec5x1` | 185 | 3551 | 16 | 880 | 8 | 1 | 201.4 ms | -21.1 ms | 205.6 ms | +| `fec6x1` | 125 | 3500 | 18 | 828 | 2 | 1 | 192.4 ms | -30.1 ms | 927.9 ms | +| `fec8x1` | 206 | 3655 | 16 | 832 | 0 | 0 | 231.6 ms | +9.1 ms | 403.5 ms | +| `fec10x1` | 25 | 3734 | 17 | 866 | 0 | 0 | 242.4 ms | +19.9 ms | 630.5 ms | +| `fec10x2` | 181 | 3617 | 16 | 856 | 2 | 1 | 227.3 ms | +4.8 ms | 571.6 ms | + +Interpretation: `fec6x1` had the best average latency, but its max latency and freeze count were worse. `fec5x1` is safer for short handoffs if max-latency stability matters more than the lowest average. + +### `handoff-long` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3373 | 12 | 915 | 3 | 2 | 245.3 ms | 0.0 ms | 261.1 ms | +| `fec5x1` | 454 | 3268 | 14 | 847 | 0 | 0 | 171.2 ms | -74.1 ms | 214.0 ms | +| `fec6x1` | 503 | 3426 | 15 | 954 | 1 | 1 | 219.6 ms | -25.7 ms | 238.5 ms | +| `fec8x1` | 446 | 3549 | 12 | 1105 | 35 | 1 | 202.2 ms | -43.1 ms | 231.6 ms | +| `fec10x1` | 500 | 2963 | 21 | 823 | 1 | 1 | 1120.8 ms | +875.5 ms | 1197.1 ms | +| `fec10x2` | 491 | 3207 | 16 | 922 | 8 | 1 | 223.8 ms | -21.5 ms | 234.6 ms | + +Interpretation: `fec5x1` clearly lowered latency under severe long fades. `fec8x1` decoded the most frames and matched RTX freeze count, but reported more lost packets. `fec10x1` is a bad fit here. + +### `bursty-uplink` + +| Config | SFU recovered | Decoded | Freezes | NACK | Lost | PLI | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 3726 | 20 | 661 | 0 | 0 | 153.5 ms | 0.0 ms | 157.5 ms | +| `fec5x1` | 20 | 3704 | 21 | 650 | 0 | 0 | 154.1 ms | +0.6 ms | 164.3 ms | +| `fec6x1` | 122 | 3668 | 16 | 732 | 8 | 1 | 160.0 ms | +6.5 ms | 169.7 ms | +| `fec8x1` | 143 | 3668 | 17 | 775 | 6 | 1 | 154.0 ms | +0.5 ms | 159.3 ms | +| `fec10x1` | 120 | 3608 | 20 | 736 | 3 | 1 | 157.4 ms | +3.9 ms | 158.9 ms | +| `fec10x2` | 290 | 3199 | 18 | 1091 | 27 | 5 | 254.6 ms | +101.1 ms | 664.0 ms | + +Interpretation: `fec8x1` was the best FEC tradeoff here: nearly RTX latency with fewer freezes, but still more NACK and some lost packets. `fec10x2` is too heavy/unstable for this profile. + +## Downlink Landline Control + +Command shape: + +```bash +DURATION=60 \ +SCENARIOS=downlink \ +FEC_CONFIGS="rtx:off:0:0 fec8x1:on:8:1 fec10x2:on:10:2" \ +LOSS_PROFILES="landline-clean:0:100:2000:5:1:0 landline-mild:1:50:2000:10:2:0" \ +scripts/flexfec/sweep-linux.sh +``` + +Primary correction signal: subscriber accepted recovery payloads. Under these landline profiles, accepted payloads prove FEC is active, not that it is needed. + +### `landline-clean` + +| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 0 | 1906 | 2 | 0 | 0 | 0 | 56.8 ms | 0.0 ms | 57.6 ms | +| `fec8x1` | 267 | 267 | 1907 | 2 | 0 | 0 | 0 | 56.8 ms | 0.0 ms | 57.7 ms | +| `fec10x2` | 426 | 408 | 1816 | 2 | 0 | 0 | 0 | 55.9 ms | -0.9 ms | 56.8 ms | + +### `landline-mild` + +| Config | Subscriber FEC accepted | FEC recv | Decoded | Freezes | NACK | Lost | RTX recovered | Capture-to-decode avg | Delta vs RTX | Max | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `rtx` | 0 | 0 | 1906 | 2 | 0 | 0 | 0 | 62.8 ms | 0.0 ms | 63.7 ms | +| `fec8x1` | 267 | 259 | 1847 | 2 | 0 | 0 | 0 | 62.9 ms | +0.1 ms | 63.6 ms | +| `fec10x2` | 426 | 408 | 1817 | 2 | 0 | 0 | 0 | 62.9 ms | +0.1 ms | 63.7 ms | + +Interpretation: downlink FEC provided no visible quality benefit in clean or mild landline conditions. Keep subscriber-side/downlink FEC disabled for the tele-op default unless the operator path is known to be lossy. + +## Recommendation + +For a tele-op default where only the robot uplink is cellular: + +1. Enable FEC on publisher-to-SFU only. +2. Leave SFU-to-subscriber FEC disabled by default. +3. Start field testing with `fec6x1` and `fec8x1`. +4. Add `fec5x1` as an aggressive mode for known handoff/fade zones. +5. Do not default to `fec10x2`; it is too costly under correlated burst loss in this run. + +The next benchmark improvement should be repeated runs with median/P95/P99 latency extraction from all timing windows. This run is enough to choose candidates, but not enough to lock a production default from a single sample per case. diff --git a/scripts/flexfec/run-leg-linux.sh b/scripts/flexfec/run-leg-linux.sh old mode 100644 new mode 100755 index 168560105..73d0cd6a8 --- a/scripts/flexfec/run-leg-linux.sh +++ b/scripts/flexfec/run-leg-linux.sh @@ -67,9 +67,9 @@ if [[ "$USE_NETNS" == "1" ]]; then NETEM_LOSS="${NETEM_LOSS:-0}" NETEM_OWD="${NETEM_OWD:-0}" if [[ "$NETEM_LOSS" != "0" || "$NETEM_OWD" != "0" ]]; then - "$LK_DIR/scripts/flexfec/netns.sh" shape "$NETEM_LOSS" "$NETEM_OWD" "${NETEM_JITTER:-0}" "${NETEM_CORR:-0}" + bash "$LK_DIR/scripts/flexfec/netns.sh" shape "$NETEM_LOSS" "$NETEM_OWD" "${NETEM_JITTER:-0}" "${NETEM_CORR:-0}" else - "$LK_DIR/scripts/flexfec/netns.sh" clear + bash "$LK_DIR/scripts/flexfec/netns.sh" clear fi fi @@ -94,13 +94,13 @@ PUB="$RUST_DIR/target/debug/publisher" SUB="$RUST_DIR/target/debug/subscriber" # --- Publisher (robot) --- -PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity flexfec-pub --test-pattern --animate-test-pattern --attach-timestamp --attach-frame-id) +PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity flexfec-pub --test-pattern 1 --attach-timestamp --attach-frame-id) # shellcheck disable=SC2206 [[ -n "$PUB_EXTRA" ]] && PUB_ARGS+=($PUB_EXTRA) if [[ "$USE_NETNS" == "1" ]]; then # netns.sh sudoes ip/tc internally; do NOT wrap it in sudo (the script isn't allowlisted). - "$LK_DIR/scripts/flexfec/netns.sh" exec env \ + bash "$LK_DIR/scripts/flexfec/netns.sh" exec env \ LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ "$PUB" "${PUB_ARGS[@]}" > "$PUB_LOG" 2>&1 & else diff --git a/scripts/flexfec/run-publisher.sh b/scripts/flexfec/run-publisher.sh index 4ce98ce3f..7b6dd700b 100755 --- a/scripts/flexfec/run-publisher.sh +++ b/scripts/flexfec/run-publisher.sh @@ -42,7 +42,8 @@ PUB_IDENTITY="${PUB_IDENTITY:-flexfec-pub}" PUB_ARGS=(--flex-fec --room-name "$ROOM" --identity "$PUB_IDENTITY") if [[ "${TEST_PATTERN:-1}" == "1" ]]; then - PUB_ARGS+=(--test-pattern) + # TEST_PATTERN_ANIMATE=1 -> animated (scrolling) pattern for a realistic motion bitrate. + PUB_ARGS+=(--test-pattern "${TEST_PATTERN_ANIMATE:-0}") fi if [[ "${ATTACH_META:-1}" == "1" ]]; then PUB_ARGS+=(--attach-timestamp --attach-frame-id) diff --git a/scripts/flexfec/sweep-linux.sh b/scripts/flexfec/sweep-linux.sh new file mode 100755 index 000000000..d32f92656 --- /dev/null +++ b/scripts/flexfec/sweep-linux.sh @@ -0,0 +1,456 @@ +#!/usr/bin/env bash +# +# Run a Linux FlexFEC-vs-RTX benchmark sweep. +# +# The sweep runs two independent legs: +# - uplink: publisher/robot -> SFU, shaped through the robot netns veth. +# - downlink: SFU -> subscriber/operator, shaped on loopback by SFU UDP source port. +# +# Each case writes raw logs and appends one row to summary.csv. Defaults are intentionally +# small enough to run overnight-ish while still covering RTX-only, low-latency FEC, and +# stronger FEC under short fade and longer handoff-style burst loss. +# +# Run from this repo on Linux: +# scripts/flexfec/sweep-linux.sh +# +# Useful overrides: +# RUST_DIR=~/workspace/rust-sdks5 +# GO_BIN=~/go1.26/bin/go +# OUT_DIR=/tmp/flexfec-results +# DURATION=60 +# SCENARIOS="uplink downlink" +# FEC_CONFIGS="rtx:off:0:0 fec5x1:on:5:1 fec10x2:on:10:2" +# LOSS_PROFILES="fade:35:250:1750:40:10:70 handoff:80:750:5000:60:20:85" +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +LK_DIR="${LK_DIR:-$REPO_ROOT}" +RUST_DIR="${RUST_DIR:-$REPO_ROOT/../rust-sdks5}" +SFU_BIN="${SFU_BIN:-/tmp/flexfec-livekit-server}" +OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results/$(date +%Y%m%d-%H%M%S)}" + +DURATION="${DURATION:-45}" +SFU_BOOT="${SFU_BOOT:-3}" +PUB_BOOT="${PUB_BOOT:-4}" +SUB_BOOT="${SUB_BOOT:-5}" + +SCENARIOS="${SCENARIOS:-uplink downlink}" +FEC_CONFIGS="${FEC_CONFIGS:-rtx:off:0:0 fec5x1:on:5:1 fec8x1:on:8:1 fec10x2:on:10:2}" +LOSS_PROFILES="${LOSS_PROFILES:-fade-short:35:250:1750:40:10:70 fade-long:70:750:5000:60:20:85}" + +ROOM_PREFIX="${ROOM_PREFIX:-flexfec-bench}" +RUST_LOG="${RUST_LOG:-info}" +PAYLOAD_TYPE="${PAYLOAD_TYPE:-49}" +GO_BIN="${GO_BIN:-go}" +REBUILD_RUST="${REBUILD_RUST:-0}" +DRY_RUN="${DRY_RUN:-0}" + +PUB_BIN="$RUST_DIR/target/debug/publisher" +SUB_BIN="$RUST_DIR/target/debug/subscriber" +SUMMARY_CSV="$OUT_DIR/summary.csv" + +declare -a CASE_PIDS=() +IMPAIR_PID="" +NETNS_ACTIVE=0 +CURRENT_LEG="" + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "error: sweep-linux.sh must run on Linux" >&2 + exit 1 +fi + +if [[ "$(id -u)" == "0" ]]; then + SUDO=() +elif [[ -n "${SUDO_ASKPASS:-}" ]]; then + SUDO=(sudo -A) +else + SUDO=(sudo) +fi + +log() { + printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +safe_name() { + printf '%s' "$1" | tr -c 'A-Za-z0-9_.=-' '_' +} + +ms_to_seconds() { + awk "BEGIN { printf \"%.3f\", $1 / 1000 }" +} + +csv_quote() { + local s="${1:-}" + s="${s//\"/\"\"}" + printf '"%s"' "$s" +} + +run_netns() { + "${SUDO[@]}" bash "$SCRIPT_DIR/netns.sh" "$@" +} + +run_netem() { + "${SUDO[@]}" bash "$SCRIPT_DIR/netem.sh" "$@" +} + +case_contains_scenario() { + local want="$1" + for scenario in $SCENARIOS; do + [[ "$scenario" == "$want" ]] && return 0 + done + return 1 +} + +prepare_sudo() { + if [[ "${#SUDO[@]}" -gt 0 ]]; then + log "checking sudo access for tc/ip netem setup" + "${SUDO[@]}" -v + fi +} + +build_artifacts() { + [[ -d "$RUST_DIR" ]] || die "rust-sdks checkout not found at $RUST_DIR (set RUST_DIR)" + + log "building SFU: $SFU_BIN" + (cd "$LK_DIR" && "$GO_BIN" build -o "$SFU_BIN" ./cmd/server) + + if [[ "$REBUILD_RUST" == "1" || ! -x "$PUB_BIN" || ! -x "$SUB_BIN" ]]; then + log "building local_video publisher/subscriber in $RUST_DIR" + (cd "$RUST_DIR" && cargo build -p local_video --features desktop --bin publisher --bin subscriber) + else + log "using existing local_video binaries in $RUST_DIR/target/debug" + fi +} + +write_summary_header() { + mkdir -p "$OUT_DIR" + cat >"$SUMMARY_CSV" <<'CSV' +timestamp,leg,profile,fec_label,fec_enabled,num_media_packets,num_fec_packets,loss_pct,burst_ms,gap_ms,one_way_delay_ms,jitter_ms,loss_correlation_pct,sfu_packets_recovered,subscriber_fec_accepted_total,video_quality_last,frame_latency_last,case_dir +CSV +} + +write_config() { + local leg="$1" + local fec_enabled="$2" + local num_media="$3" + local num_fec="$4" + local config="$5" + + local publisher=false + local subscriber=false + local bind_addresses="" + local includes="" + + if [[ "$leg" == "uplink" ]]; then + publisher="$fec_enabled" + bind_addresses=$'bind_addresses:\n - 10.200.0.1\n - 127.0.0.1' + includes=$' - 10.200.0.0/24\n - 127.0.0.0/8' + else + subscriber="$fec_enabled" + bind_addresses=$'bind_addresses:\n - 127.0.0.1' + includes=$' - 127.0.0.0/8' + fi + + if [[ "$fec_enabled" == "false" ]]; then + num_media=5 + num_fec=1 + fi + + cat >"$config" </dev/null + for _ in {1..20}; do + kill -0 "$IMPAIR_PID" 2>/dev/null || break + sleep 0.1 + done + kill -9 "$IMPAIR_PID" 2>/dev/null + wait "$IMPAIR_PID" 2>/dev/null + IMPAIR_PID="" + fi + + case "$CURRENT_LEG" in + uplink) run_netns clear >/dev/null 2>&1 ;; + downlink) run_netem stop >/dev/null 2>&1 ;; + esac + set -e +} + +stop_case_processes() { + set +e + if [[ "${#CASE_PIDS[@]}" -gt 0 ]]; then + kill "${CASE_PIDS[@]}" 2>/dev/null + sleep 1 + kill -9 "${CASE_PIDS[@]}" 2>/dev/null + wait "${CASE_PIDS[@]}" 2>/dev/null + fi + CASE_PIDS=() + + pkill -f "$SFU_BIN" 2>/dev/null + pkill -f "$PUB_BIN" 2>/dev/null + pkill -f "$SUB_BIN" 2>/dev/null + set -e +} + +cleanup_all() { + set +e + stop_impairment + set +e + stop_case_processes + set +e + if [[ "$NETNS_ACTIVE" == "1" ]]; then + run_netns down >/dev/null 2>&1 + fi +} + +start_uplink_burst() { + local case_dir="$1" + local loss="$2" + local burst_ms="$3" + local gap_ms="$4" + local owd_ms="$5" + local jitter_ms="$6" + local corr_pct="$7" + local burst_s gap_s + burst_s="$(ms_to_seconds "$burst_ms")" + gap_s="$(ms_to_seconds "$gap_ms")" + + ( + trap 'exit 0' INT TERM + while true; do + run_netns shape "$loss" "$owd_ms" "$jitter_ms" "$corr_pct" + sleep "$burst_s" + run_netns shape 0 "$owd_ms" "$jitter_ms" 0 + sleep "$gap_s" + done + ) >"$case_dir/netem.log" 2>&1 & + IMPAIR_PID="$!" +} + +start_downlink_burst() { + local case_dir="$1" + local loss="$2" + local burst_ms="$3" + local gap_ms="$4" + local owd_ms="$5" + + run_netem burst "$loss" "$burst_ms" "$gap_ms" down "$owd_ms" >"$case_dir/netem.log" 2>&1 & + IMPAIR_PID="$!" +} + +start_impairment() { + local leg="$1" + local case_dir="$2" + local loss="$3" + local burst_ms="$4" + local gap_ms="$5" + local owd_ms="$6" + local jitter_ms="$7" + local corr_pct="$8" + + CURRENT_LEG="$leg" + if [[ "$leg" == "uplink" ]]; then + start_uplink_burst "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" + else + start_downlink_burst "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" + fi +} + +last_line() { + local pattern="$1" + local file="$2" + grep -iE "$pattern" "$file" 2>/dev/null | tail -1 || true +} + +extract_key_number() { + local key="$1" + local line="$2" + printf '%s' "$line" | sed -nE "s/.*${key}\"?[:= ]+([0-9]+).*/\1/p" +} + +summarize_case() { + local leg="$1" + local profile="$2" + local fec_label="$3" + local fec_enabled="$4" + local num_media="$5" + local num_fec="$6" + local loss="$7" + local burst_ms="$8" + local gap_ms="$9" + local owd_ms="${10}" + local jitter_ms="${11}" + local corr_pct="${12}" + local case_dir="${13}" + + local sfu_log="$case_dir/sfu.log" + local sub_log="$case_dir/subscriber.log" + local sfu_recovery_line video_quality frame_latency accepted_line recovered accepted + + sfu_recovery_line="$(last_line 'flexfec recovery stats' "$sfu_log")" + video_quality="$(last_line 'Video quality' "$sub_log")" + frame_latency="$(last_line 'frame latency' "$sub_log")" + accepted_line="$(last_line 'accepted_recovery_payload_total|accepted_total|Video FEC recovery payload accepted' "$sub_log")" + + recovered="$(extract_key_number 'packetsRecovered' "$sfu_recovery_line")" + accepted="$(extract_key_number 'accepted_recovery_payload_total' "$accepted_line")" + if [[ -z "$accepted" ]]; then + accepted="$(extract_key_number 'accepted_total' "$accepted_line")" + fi + + { + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,' \ + "$(date -Iseconds)" "$leg" "$profile" "$fec_label" "$fec_enabled" \ + "$num_media" "$num_fec" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" \ + "${recovered:-0}" "${accepted:-0}" + csv_quote "$video_quality" + printf ',' + csv_quote "$frame_latency" + printf ',' + csv_quote "$case_dir" + printf '\n' + } >>"$SUMMARY_CSV" +} + +run_case() { + local leg="$1" + local config_spec="$2" + local profile_spec="$3" + local fec_label fec_mode num_media num_fec + local profile loss burst_ms gap_ms owd_ms jitter_ms corr_pct + + IFS=: read -r fec_label fec_mode num_media num_fec <<<"$config_spec" + IFS=: read -r profile loss burst_ms gap_ms owd_ms jitter_ms corr_pct <<<"$profile_spec" + + local fec_enabled=false + [[ "$fec_mode" == "on" ]] && fec_enabled=true + + local case_name case_dir config sfu_host room + case_name="$(safe_name "${leg}_${profile}_${fec_label}")" + case_dir="$OUT_DIR/$case_name" + config="$case_dir/sfu.yaml" + room="${ROOM_PREFIX}-${case_name}" + mkdir -p "$case_dir" + write_config "$leg" "$fec_enabled" "$num_media" "$num_fec" "$config" + + if [[ "$leg" == "uplink" ]]; then + sfu_host="10.200.0.1" + run_netns clear >/dev/null 2>&1 || true + else + sfu_host="127.0.0.1" + run_netem stop >/dev/null 2>&1 || true + fi + + log "case: leg=$leg profile=$profile fec=$fec_label loss=${loss}% burst=${burst_ms}ms gap=${gap_ms}ms owd=${owd_ms}ms" + + if [[ "$DRY_RUN" == "1" ]]; then + return + fi + + stop_case_processes + CASE_PIDS=() + IMPAIR_PID="" + CURRENT_LEG="" + + ( + cd "$LK_DIR" + "$SFU_BIN" --dev --config "$config" + ) >"$case_dir/sfu.log" 2>&1 & + CASE_PIDS+=("$!") + sleep "$SFU_BOOT" + + local url="ws://$sfu_host:7880" + local pub_args=(--flex-fec --room-name "$room" --identity "${case_name}-pub" --test-pattern 1 --attach-timestamp --attach-frame-id) + if [[ "$leg" == "uplink" ]]; then + run_netns exec env \ + LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ + "$PUB_BIN" "${pub_args[@]}" >"$case_dir/publisher.log" 2>&1 & + else + LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ + "$PUB_BIN" "${pub_args[@]}" >"$case_dir/publisher.log" 2>&1 & + fi + CASE_PIDS+=("$!") + sleep "$PUB_BOOT" + + local sub_display sub_xauth + sub_display="${DISPLAY:-${SUB_DISPLAY:-:1}}" + sub_xauth="${XAUTHORITY:-${SUB_XAUTHORITY:-/run/user/$(id -u)/gdm/Xauthority}}" + DISPLAY="$sub_display" XAUTHORITY="$sub_xauth" \ + LIVEKIT_URL="$url" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ + "$SUB_BIN" --room-name "$room" --identity "${case_name}-sub" >"$case_dir/subscriber.log" 2>&1 & + CASE_PIDS+=("$!") + sleep "$SUB_BOOT" + + start_impairment "$leg" "$case_dir" "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" + sleep "$DURATION" + + stop_impairment + stop_case_processes + summarize_case "$leg" "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" \ + "$loss" "$burst_ms" "$gap_ms" "$owd_ms" "$jitter_ms" "$corr_pct" "$case_dir" +} + +main() { + prepare_sudo + build_artifacts + write_summary_header + + if case_contains_scenario uplink; then + log "setting up robot netns for uplink benchmarks" + run_netns up + NETNS_ACTIVE=1 + fi + + log "results: $OUT_DIR" + for scenario in $SCENARIOS; do + case "$scenario" in + uplink | downlink) ;; + *) die "unknown scenario '$scenario' (use uplink/downlink)" ;; + esac + for profile_spec in $LOSS_PROFILES; do + for config_spec in $FEC_CONFIGS; do + run_case "$scenario" "$config_spec" "$profile_spec" + done + done + done + + cleanup_all + log "summary: $SUMMARY_CSV" +} + +trap cleanup_all EXIT INT TERM +main "$@" diff --git a/scripts/flexfec/sweep-remote-publish.sh b/scripts/flexfec/sweep-remote-publish.sh new file mode 100755 index 000000000..506bbd96c --- /dev/null +++ b/scripts/flexfec/sweep-remote-publish.sh @@ -0,0 +1,339 @@ +#!/usr/bin/env bash +# +# Focused tele-op publish-leg benchmark: +# - this computer runs local_video publisher + subscriber +# - DC-Linux runs the SFU +# - the SFU applies publish-side packet loss only, with no added delay +# +# This models a robot publishing over an already-real public/LAN path where the link RTT is +# real and should not be inflated by the test harness. Subscriber/downlink metrics are still +# collected, but FlexFEC generation toward the subscriber is disabled by default. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +REMOTE="${REMOTE:-dc@DC-Linux}" +REMOTE_LK_DIR="${REMOTE_LK_DIR:-/home/dc/workspace/livekit-flexfec-bench}" +REMOTE_GO_BIN="${REMOTE_GO_BIN:-/home/dc/go1.26/bin/go}" +REMOTE_SFU_BIN="${REMOTE_SFU_BIN:-/tmp/flexfec-livekit-server}" +REMOTE_WORK_DIR="${REMOTE_WORK_DIR:-/tmp/flexfec-remote-publish}" +REMOTE_BIND_ADDRESSES="${REMOTE_BIND_ADDRESSES:-10.104.0.88 100.69.171.8 127.0.0.1}" +REMOTE_ICE_INCLUDES="${REMOTE_ICE_INCLUDES:-10.104.0.0/16 100.64.0.0/10 127.0.0.0/8}" +SSH_OPTS="${SSH_OPTS:--o ControlPath=/tmp/dc-linux-ctrl}" + +RUST_DIR="${RUST_DIR:-$REPO_ROOT/../rust-sdks5}" +PUB_BIN="${PUB_BIN:-$RUST_DIR/target/debug/publisher}" +SUB_BIN="${SUB_BIN:-$RUST_DIR/target/debug/subscriber}" +SFU_HOST="${SFU_HOST:-DC-Linux}" +LIVEKIT_URL="${LIVEKIT_URL:-ws://$SFU_HOST:7880}" + +OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results/remote-publish-$(date +%Y%m%d-%H%M%S)}" +SUMMARY_CSV="$OUT_DIR/summary.csv" + +DURATION="${DURATION:-60}" +SFU_BOOT="${SFU_BOOT:-3}" +SUB_BOOT="${SUB_BOOT:-4}" +PUB_BOOT="${PUB_BOOT:-4}" +ROOM_PREFIX="${ROOM_PREFIX:-teleop-publish}" +RUST_LOG="${RUST_LOG:-info}" +PAYLOAD_TYPE="${PAYLOAD_TYPE:-49}" +DROP_LOG="${DROP_LOG:-1}" +REBUILD_RUST="${REBUILD_RUST:-0}" +DRY_RUN="${DRY_RUN:-0}" + +FEC_CONFIGS="${FEC_CONFIGS:-rtx:off:0:0 fec6x1:on:6:1 fec8x1:on:8:1 fec5x1:on:5:1}" +LOSS_PCTS="${LOSS_PCTS:-0.1 0.5 1 2 5}" +BURST_MS="${BURST_MS:-100 200 500}" +BURST_GAP_MS="${BURST_GAP_MS:-2000}" + +PUB_EXTRA="${PUB_EXTRA:---codec vp8 --encoder software}" +SUB_EXTRA="${SUB_EXTRA:-}" + +declare -a LOCAL_PIDS=() +REMOTE_SFU_PID="" +REMOTE_CASE_DIR="" + +log() { + printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +ssh_remote() { + # shellcheck disable=SC2086 + ssh $SSH_OPTS "$REMOTE" "$@" +} + +rsync_from_remote() { + local src="$1" + local dst="$2" + # shellcheck disable=SC2086 + rsync -az -e "ssh $SSH_OPTS" "$REMOTE:$src" "$dst" +} + +safe_name() { + printf '%s' "$1" | tr -c 'A-Za-z0-9_.=-' '_' +} + +csv_quote() { + local s="${1:-}" + s="${s//\"/\"\"}" + printf '"%s"' "$s" +} + +shell_quote() { + printf '%q' "$1" +} + +loss_fraction() { + awk "BEGIN { printf \"%.6f\", $1 / 100 }" +} + +write_summary_header() { + mkdir -p "$OUT_DIR" + cat >"$SUMMARY_CSV" <<'CSV' +timestamp,profile,fec_label,fec_enabled,num_media_packets,num_fec_packets,loss_pct,burst_ms,gap_ms,sfu_packets_recovered,sfu_drop_count,sfu_marker_count,subscriber_fec_accepted_total,video_quality_last,frame_latency_last,case_dir +CSV +} + +write_remote_config() { + local fec_enabled="$1" + local num_media="$2" + local num_fec="$3" + local remote_config="$4" + local bind_addresses_yaml="" + local ice_includes_yaml="" + + if [[ "$fec_enabled" == "false" ]]; then + num_media=5 + num_fec=1 + fi + + local bind_address + for bind_address in $REMOTE_BIND_ADDRESSES; do + bind_addresses_yaml+=" - $bind_address"$'\n' + done + + local ice_include + for ice_include in $REMOTE_ICE_INCLUDES; do + ice_includes_yaml+=" - $ice_include"$'\n' + done + + ssh_remote "cat > '$remote_config' <<'EOF' +port: 7880 + +bind_addresses: +$bind_addresses_yaml + +rtc: + udp_port: 7882 + tcp_port: 7881 + use_external_ip: false + enable_loopback_candidate: true + ips: + includes: +$ice_includes_yaml + + flexfec: + subscriber: false + publisher: $fec_enabled + payload_type: $PAYLOAD_TYPE + num_media_packets: $num_media + num_fec_packets: $num_fec + +logging: + level: debug + json: false +EOF" +} + +stop_local_processes() { + set +e + if [[ "${#LOCAL_PIDS[@]}" -gt 0 ]]; then + kill "${LOCAL_PIDS[@]}" 2>/dev/null + sleep 1 + kill -9 "${LOCAL_PIDS[@]}" 2>/dev/null + wait "${LOCAL_PIDS[@]}" 2>/dev/null + fi + LOCAL_PIDS=() + set -e +} + +stop_remote_sfu() { + set +e + if [[ -n "${REMOTE_SFU_PID:-}" ]]; then + ssh_remote "kill '$REMOTE_SFU_PID' 2>/dev/null || true; sleep 1; kill -9 '$REMOTE_SFU_PID' 2>/dev/null || true; wait '$REMOTE_SFU_PID' 2>/dev/null || true" >/dev/null 2>&1 + REMOTE_SFU_PID="" + fi + ssh_remote "pkill -f '$REMOTE_SFU_BIN' 2>/dev/null || true" >/dev/null 2>&1 + set -e +} + +cleanup_all() { + set +e + stop_local_processes + stop_remote_sfu +} + +last_line() { + local pattern="$1" + local file="$2" + grep -iE "$pattern" "$file" 2>/dev/null | tail -1 || true +} + +extract_key_number() { + local key="$1" + local line="$2" + printf '%s' "$line" | sed -nE "s/.*${key}\"?[:= ]+([0-9]+).*/\1/p" +} + +summarize_case() { + local profile="$1" + local fec_label="$2" + local fec_enabled="$3" + local num_media="$4" + local num_fec="$5" + local loss="$6" + local burst_ms="$7" + local gap_ms="$8" + local case_dir="$9" + + local sfu_log="$case_dir/sfu.log" + local sub_log="$case_dir/subscriber.log" + local sfu_recovery_line video_quality frame_latency accepted_line recovered accepted drops markers + + sfu_recovery_line="$(last_line 'flexfec recovery stats' "$sfu_log")" + video_quality="$(last_line 'Video quality' "$sub_log")" + frame_latency="$(last_line 'frame latency' "$sub_log")" + accepted_line="$(last_line 'accepted_recovery_payload_total|accepted_total|Video FEC recovery payload accepted' "$sub_log")" + + recovered="$(extract_key_number 'packetsRecovered' "$sfu_recovery_line")" + accepted="$(extract_key_number 'accepted_recovery_payload_total' "$accepted_line")" + if [[ -z "$accepted" ]]; then + accepted="$(extract_key_number 'accepted_total' "$accepted_line")" + fi + drops="$(grep -c 'uplink impairment dropped RTP packet' "$sfu_log" 2>/dev/null || true)" + markers="$(grep -c 'uplink impairment received frame marker' "$sfu_log" 2>/dev/null || true)" + + { + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,' \ + "$(date -Iseconds)" "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" \ + "$loss" "$burst_ms" "$gap_ms" "${recovered:-0}" "${drops:-0}" "${markers:-0}" "${accepted:-0}" + csv_quote "$video_quality" + printf ',' + csv_quote "$frame_latency" + printf ',' + csv_quote "$case_dir" + printf '\n' + } >>"$SUMMARY_CSV" +} + +build_artifacts() { + [[ -d "$RUST_DIR" ]] || die "rust-sdks checkout not found at $RUST_DIR (set RUST_DIR)" + + log "building remote SFU: $REMOTE:$REMOTE_SFU_BIN" + ssh_remote "cd '$REMOTE_LK_DIR' && '$REMOTE_GO_BIN' build -o '$REMOTE_SFU_BIN' ./cmd/server" + + if [[ "$REBUILD_RUST" == "1" || ! -x "$PUB_BIN" || ! -x "$SUB_BIN" ]]; then + log "building local_video publisher/subscriber in $RUST_DIR" + (cd "$RUST_DIR" && cargo build -p local_video --features desktop --bin publisher --bin subscriber) + else + log "using existing local_video binaries in $RUST_DIR/target/debug" + fi +} + +make_profiles() { + for loss in $LOSS_PCTS; do + for burst in $BURST_MS; do + printf 'loss%s_b%s:%s:%s:%s\n' \ + "$(printf '%s' "$loss" | tr '.' 'p')" "$burst" "$loss" "$burst" "$BURST_GAP_MS" + done + done +} + +run_case() { + local config_spec="$1" + local profile_spec="$2" + local fec_label fec_mode num_media num_fec profile loss burst_ms gap_ms + IFS=: read -r fec_label fec_mode num_media num_fec <<<"$config_spec" + IFS=: read -r profile loss burst_ms gap_ms <<<"$profile_spec" + + local fec_enabled=false + [[ "$fec_mode" == "on" ]] && fec_enabled=true + + local case_name case_dir remote_case config room pub_args + case_name="$(safe_name "${profile}_${fec_label}")" + case_dir="$OUT_DIR/$case_name" + remote_case="$REMOTE_WORK_DIR/$case_name" + config="$remote_case/sfu.yaml" + room="${ROOM_PREFIX}-${case_name}" + mkdir -p "$case_dir" + + log "case: profile=$profile fec=$fec_label loss=${loss}% burst=${burst_ms}ms gap=${gap_ms}ms url=$LIVEKIT_URL" + + if [[ "$DRY_RUN" == "1" ]]; then + return + fi + + stop_local_processes + stop_remote_sfu + LOCAL_PIDS=() + + ssh_remote "mkdir -p '$remote_case'" + write_remote_config "$fec_enabled" "$num_media" "$num_fec" "$config" + + local loss_frac + loss_frac="$(loss_fraction "$loss")" + local remote_start_cmd + remote_start_cmd="cd $(shell_quote "$REMOTE_LK_DIR") || exit; env LK_PUB_LOSS=$(shell_quote "$loss_frac") LK_PUB_DELAY_MS=0 LK_PUB_BURST_MS=$(shell_quote "$burst_ms") LK_PUB_GAP_MS=$(shell_quote "$gap_ms") LK_PUB_DROP_LOG=$(shell_quote "$DROP_LOG") $(shell_quote "$REMOTE_SFU_BIN") --dev --config $(shell_quote "$config") > $(shell_quote "$remote_case/sfu.log") 2>&1 < /dev/null & pid=\$!; disown \"\$pid\"; echo \"\$pid\"" + REMOTE_SFU_PID="$( + ssh_remote "bash -lc $(shell_quote "$remote_start_cmd")" + )" + sleep "$SFU_BOOT" + + LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ + "$SUB_BIN" --room-name "$room" --identity "${case_name}-sub" $SUB_EXTRA >"$case_dir/subscriber.log" 2>&1 & + LOCAL_PIDS+=("$!") + sleep "$SUB_BOOT" + + pub_args=(--room-name "$room" --identity "${case_name}-pub" --test-pattern 1 --attach-timestamp --attach-frame-id) + if [[ "$fec_enabled" == "true" ]]; then + pub_args=(--flex-fec "${pub_args[@]}") + fi + LIVEKIT_URL="$LIVEKIT_URL" LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret RUST_LOG="$RUST_LOG" \ + "$PUB_BIN" "${pub_args[@]}" $PUB_EXTRA >"$case_dir/publisher.log" 2>&1 & + LOCAL_PIDS+=("$!") + sleep "$PUB_BOOT" + + sleep "$DURATION" + + stop_local_processes + stop_remote_sfu + rsync_from_remote "$remote_case/" "$case_dir/" + grep 'uplink impairment dropped RTP packet' "$case_dir/sfu.log" >"$case_dir/dropped-packets.log" 2>/dev/null || true + grep 'uplink impairment received frame marker' "$case_dir/sfu.log" >"$case_dir/frame-markers.log" 2>/dev/null || true + + summarize_case "$profile" "$fec_label" "$fec_enabled" "$num_media" "$num_fec" "$loss" "$burst_ms" "$gap_ms" "$case_dir" +} + +main() { + build_artifacts + write_summary_header + + log "results: $OUT_DIR" + for profile_spec in $(make_profiles); do + for config_spec in $FEC_CONFIGS; do + run_case "$config_spec" "$profile_spec" + done + done + + cleanup_all + log "summary: $SUMMARY_CSV" +} + +trap cleanup_all EXIT INT TERM +main "$@"