fix: skip RTX/FEC when picking a fallback video codec (#4930)

selectAlternativeVideoCodec walked the enabled codecs and returned the
first one with a video/ prefix. video/rtx and video/flexfec-03 both have
that prefix, so when the requested codec was not enabled the track could
fall back to RTX. RTX is a retransmission stream, not a real codec, so
the track got registered as video/rtx and subscribers saw a blank tile.

Skip RTX and FEC entries so the fallback is always a valid video codec.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

pkg/rtc/mediaengine.go

@@ -162,6 +162,10 @@ func isCodecEnabledWithFmtp(codecs []*livekit.Codec, cap webrtc.RTPCodecCapabili
 
 func selectAlternativeVideoCodec(enabledCodecs []*livekit.Codec) string {
 	for _, c := range enabledCodecs {
+		// RTX and FEC are not real codecs, skip them
+		if mime.IsMimeTypeStringRTX(c.Mime) || mime.IsMimeTypeStringFlexFEC03(c.Mime) {
+			continue
+		}
 		if mime.IsMimeTypeStringVideo(c.Mime) {
 			return c.Mime
 		}

pkg/rtc/mediaengine_test.go

@@ -73,6 +73,23 @@ func TestIsCodecEnabled(t *testing.T) {
 	})
 }
 
+func TestSelectAlternativeVideoCodec(t *testing.T) {
+	// RTX and FEC have the video/ prefix but are not real codecs. The fallback must skip them.
+	t.Run("skips rtx and fec, picks real codec", func(t *testing.T) {
+		enabled := []*livekit.Codec{
+			{Mime: mime.MimeTypeRTX.String()},
+			{Mime: mime.MimeTypeFlexFEC03.String()},
+			{Mime: mime.MimeTypeH264.String()},
+		}
+		require.Equal(t, mime.MimeTypeH264.String(), selectAlternativeVideoCodec(enabled))
+	})
+
+	t.Run("falls back to vp8 when only rtx is enabled", func(t *testing.T) {
+		enabled := []*livekit.Codec{{Mime: mime.MimeTypeRTX.String()}}
+		require.Equal(t, mime.MimeTypeVP8.String(), selectAlternativeVideoCodec(enabled))
+	})
+}
+
 type offeredCodec struct {
 	payloadType webrtc.PayloadType
 	name        string

pkg/rtc/participant_internal_test.go

@@ -79,6 +79,38 @@ func TestIsReady(t *testing.T) {
 	}
 }
 
+// Disabling a codec can leave the publish list with only RTX and FEC. The
+// fallback must still pick a real codec, or the track registers as RTX and
+// subscribers see a blank tile.
+func TestPublishCodecFallbackSkipsRTX(t *testing.T) {
+	h264 := &livekit.Codec{Mime: mime.MimeTypeH264.String()}
+	rtx := &livekit.Codec{Mime: mime.MimeTypeRTX.String()}
+
+	newParticipant := func() *ParticipantImpl {
+		return &ParticipantImpl{params: ParticipantParams{Logger: logger.GetLogger()}}
+	}
+
+	t.Run("h264 enabled, rtx is last and not chosen", func(t *testing.T) {
+		p := newParticipant()
+		p.setupEnabledCodecs([]*livekit.Codec{h264, rtx}, nil, nil)
+
+		require.Equal(t, mime.MimeTypeH264.String(), selectAlternativeVideoCodec(p.enabledPublishCodecs))
+	})
+
+	t.Run("h264 disabled leaves rtx only, fallback avoids rtx", func(t *testing.T) {
+		p := newParticipant()
+		disabled := &livekit.DisabledCodecs{Publish: []*livekit.Codec{{Mime: mime.MimeTypeH264.String()}}}
+		p.setupEnabledCodecs([]*livekit.Codec{h264, rtx}, nil, disabled)
+
+		require.Len(t, p.enabledPublishCodecs, 1)
+		require.True(t, mime.IsMimeTypeStringRTX(p.enabledPublishCodecs[0].Mime))
+
+		alt := selectAlternativeVideoCodec(p.enabledPublishCodecs)
+		require.False(t, mime.IsMimeTypeStringRTX(alt), "fallback must not be rtx, got %q", alt)
+		require.Equal(t, mime.MimeTypeVP8.String(), alt)
+	})
+}
+
 func TestSupportsMoving(t *testing.T) {
 	t.Run("current protocol version", func(t *testing.T) {
 		p := newParticipantForTestWithOpts("test", &participantOpts{protocolVersion: types.CurrentProtocol})