sfu: find the RTX payload type when its fmtp line has more than apt (#5135)

FindRTXPayloadType compared the whole fmtp line with "apt=<pt>", so an
RTX codec negotiated as "apt=96;rtx-time=3000" (RFC 4588) was never
found. The down track then stored RTX payload type 0, sent
retransmissions on the primary SSRC and probed with padding even though
RTX was negotiated.

Compare only the apt parameter of the fmtp line.

pkg/sfu/utils/helpers.go

@@ -17,6 +17,8 @@ package utils
 import (
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/pion/interceptor"
 	"github.com/pion/rtp"
@@ -47,12 +49,17 @@ func CodecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []web
 	return webrtc.RTPCodecParameters{}, webrtc.ErrCodecNotFound
 }
 
-// Given a CodecParameters find the RTX CodecParameters if one exists
+// Given a CodecParameters find the RTX CodecParameters if one exists.
+// The RTX fmtp line can carry other parameters next to apt, for example
+// "apt=96;rtx-time=3000" (RFC 4588), so only the apt value is compared.
 func FindRTXPayloadType(needle webrtc.PayloadType, haystack []webrtc.RTPCodecParameters) webrtc.PayloadType {
-	aptStr := fmt.Sprintf("apt=%d", needle)
+	aptStr := strconv.Itoa(int(needle))
 	for _, c := range haystack {
-		if aptStr == c.SDPFmtpLine {
-			return c.PayloadType
+		for param := range strings.SplitSeq(c.SDPFmtpLine, ";") {
+			key, value, found := strings.Cut(param, "=")
+			if found && strings.EqualFold(strings.TrimSpace(key), "apt") && strings.TrimSpace(value) == aptStr {
+				return c.PayloadType
+			}
 		}
 	}
 

pkg/sfu/utils/helpers_test.go

@@ -0,0 +1,55 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/codecs/mime"
+)
+
+func TestFindRTXPayloadType(t *testing.T) {
+	rtx := func(pt webrtc.PayloadType, fmtp string) webrtc.RTPCodecParameters {
+		return webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mime.MimeTypeRTX.String(), ClockRate: 90000, SDPFmtpLine: fmtp},
+			PayloadType:        pt,
+		}
+	}
+	vp8 := webrtc.RTPCodecParameters{
+		RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP8.String(), ClockRate: 90000},
+		PayloadType:        96,
+	}
+
+	tests := []struct {
+		name     string
+		haystack []webrtc.RTPCodecParameters
+		expected webrtc.PayloadType
+	}{
+		{name: "apt only", haystack: []webrtc.RTPCodecParameters{vp8, rtx(97, "apt=96")}, expected: 97},
+		{name: "apt with rtx-time", haystack: []webrtc.RTPCodecParameters{vp8, rtx(97, "apt=96;rtx-time=3000")}, expected: 97},
+		{name: "rtx-time before apt", haystack: []webrtc.RTPCodecParameters{vp8, rtx(97, "rtx-time=3000; apt=96")}, expected: 97},
+		{name: "other codec", haystack: []webrtc.RTPCodecParameters{vp8, rtx(99, "apt=98"), rtx(97, "apt=96")}, expected: 97},
+		{name: "apt prefix", haystack: []webrtc.RTPCodecParameters{vp8, rtx(97, "apt=960")}, expected: 0},
+		{name: "no rtx", haystack: []webrtc.RTPCodecParameters{vp8}, expected: 0},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expected, FindRTXPayloadType(96, tc.haystack))
+		})
+	}
+}