From 21424fadd45951d43b8dbd10a466df7d3d12bd2f Mon Sep 17 00:00:00 2001
From: boks1971 <raja.gobi@tutanota.com>
Date: Sun, 4 Oct 2026 13:42:39 +0530
Subject: [PATCH] fix: skip RTX/FEC when picking a fallback video codec

selectAlternativeVideoCodec walked the enabled codecs and returned the
first one with a video/ prefix. video/rtx and video/flexfec-03 both have
that prefix, so when the requested codec was not enabled the track could
fall back to RTX. RTX is a retransmission stream, not a real codec, so
the track got registered as video/rtx and subscribers saw a blank tile.

Skip RTX and FEC entries so the fallback is always a valid video codec.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 pkg/rtc/mediaengine.go               |  4 ++++
 pkg/rtc/mediaengine_test.go          | 17 +++++++++++++++
 pkg/rtc/participant_internal_test.go | 32 ++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go
index 8f44593f7..3ac28b43f 100644
--- a/pkg/rtc/mediaengine.go
+++ b/pkg/rtc/mediaengine.go
@@ -162,6 +162,10 @@ func isCodecEnabledWithFmtp(codecs []*livekit.Codec, cap webrtc.RTPCodecCapabili
 
 func selectAlternativeVideoCodec(enabledCodecs []*livekit.Codec) string {
 	for _, c := range enabledCodecs {
+		// RTX and FEC are not real codecs, skip them
+		if mime.IsMimeTypeStringRTX(c.Mime) || mime.IsMimeTypeStringFlexFEC03(c.Mime) {
+			continue
+		}
 		if mime.IsMimeTypeStringVideo(c.Mime) {
 			return c.Mime
 		}
diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go
index 0b7424394..b4648112b 100644
--- a/pkg/rtc/mediaengine_test.go
+++ b/pkg/rtc/mediaengine_test.go
@@ -73,6 +73,23 @@ func TestIsCodecEnabled(t *testing.T) {
 	})
 }
 
+func TestSelectAlternativeVideoCodec(t *testing.T) {
+	// RTX and FEC have the video/ prefix but are not real codecs. The fallback must skip them.
+	t.Run("skips rtx and fec, picks real codec", func(t *testing.T) {
+		enabled := []*livekit.Codec{
+			{Mime: mime.MimeTypeRTX.String()},
+			{Mime: mime.MimeTypeFlexFEC03.String()},
+			{Mime: mime.MimeTypeH264.String()},
+		}
+		require.Equal(t, mime.MimeTypeH264.String(), selectAlternativeVideoCodec(enabled))
+	})
+
+	t.Run("falls back to vp8 when only rtx is enabled", func(t *testing.T) {
+		enabled := []*livekit.Codec{{Mime: mime.MimeTypeRTX.String()}}
+		require.Equal(t, mime.MimeTypeVP8.String(), selectAlternativeVideoCodec(enabled))
+	})
+}
+
 type offeredCodec struct {
 	payloadType webrtc.PayloadType
 	name        string
diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go
index c2167eb53..5827d3f9f 100644
--- a/pkg/rtc/participant_internal_test.go
+++ b/pkg/rtc/participant_internal_test.go
@@ -79,6 +79,38 @@ func TestIsReady(t *testing.T) {
 	}
 }
 
+// Disabling a codec can leave the publish list with only RTX and FEC. The
+// fallback must still pick a real codec, or the track registers as RTX and
+// subscribers see a blank tile.
+func TestPublishCodecFallbackSkipsRTX(t *testing.T) {
+	h264 := &livekit.Codec{Mime: mime.MimeTypeH264.String()}
+	rtx := &livekit.Codec{Mime: mime.MimeTypeRTX.String()}
+
+	newParticipant := func() *ParticipantImpl {
+		return &ParticipantImpl{params: ParticipantParams{Logger: logger.GetLogger()}}
+	}
+
+	t.Run("h264 enabled, rtx is last and not chosen", func(t *testing.T) {
+		p := newParticipant()
+		p.setupEnabledCodecs([]*livekit.Codec{h264, rtx}, nil, nil)
+
+		require.Equal(t, mime.MimeTypeH264.String(), selectAlternativeVideoCodec(p.enabledPublishCodecs))
+	})
+
+	t.Run("h264 disabled leaves rtx only, fallback avoids rtx", func(t *testing.T) {
+		p := newParticipant()
+		disabled := &livekit.DisabledCodecs{Publish: []*livekit.Codec{{Mime: mime.MimeTypeH264.String()}}}
+		p.setupEnabledCodecs([]*livekit.Codec{h264, rtx}, nil, disabled)
+
+		require.Len(t, p.enabledPublishCodecs, 1)
+		require.True(t, mime.IsMimeTypeStringRTX(p.enabledPublishCodecs[0].Mime))
+
+		alt := selectAlternativeVideoCodec(p.enabledPublishCodecs)
+		require.False(t, mime.IsMimeTypeStringRTX(alt), "fallback must not be rtx, got %q", alt)
+		require.Equal(t, mime.MimeTypeVP8.String(), alt)
+	})
+}
+
 func TestSupportsMoving(t *testing.T) {
 	t.Run("current protocol version", func(t *testing.T) {
 		p := newParticipantForTestWithOpts("test", &participantOpts{protocolVersion: types.CurrentProtocol})