Support more h264 profiles (#4708)

* Register h264 main profile if enabled explicitly We don't support the h264 main profile for compatibility, user can enabled it by set fmtp explicitly in codec config to enable it if want to use it in special scenario. * go mod
2026-07-28 20:49:26 +00:00 · 2026-07-28 16:18:19 +08:00
parent b8a073cb68
commit 436a0cc3d3
4 changed files with 214 additions and 12 deletions
@@ -21,7 +21,7 @@ require (
 	github.com/jxskiss/base62 v1.1.0
 	github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731
 	github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0
-	github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe
+	github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6
 	github.com/livekit/psrpc v0.7.2
 	github.com/mackerelio/go-osstat v0.2.8
 	github.com/magefile/mage v1.17.2
@@ -160,10 +160,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT
 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
 github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 h1:XHNNzebIKZRkLimla/hFGrAIX5EMWHctrgt3hLw7s+I=
 github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU=
-github.com/livekit/protocol v1.50.4 h1:Pzg9p1lpu9TcxUFJqIlUGNP6m7mX8PBT+ETxXRpnmZ8=
-github.com/livekit/protocol v1.50.4/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs=
-github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe h1:Vdw2fb0B0iFaAt45JHho11qb60+yiR/ZHyLBjDegka0=
-github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs=
+github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6 h1:oavqDhmS2U5zFPs3S+jxTL6tEqsHxATxzfw3B/CKC+w=
+github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs=
 github.com/livekit/psrpc v0.7.2 h1:6oZ+NODJ2pLyaT6VqDq1F4Qc/3TpDUSpyphj/P9MhQc=
 github.com/livekit/psrpc v0.7.2/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw=
 github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4=
@@ -25,6 +25,11 @@ import (
 	"github.com/livekit/protocol/livekit"
 )

+type codecToRegister struct {
+	webrtc.RTPCodecParameters
+	strictFmtp bool
+}
+
 func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error {
 	// audio codecs
 	if IsCodecEnabled(codecs, protoCodecs.OpusCodecParameters.RTPCodecCapability) {
@@ -55,20 +60,36 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac

 	// video codecs
 	rtxEnabled := IsCodecEnabled(codecs, protoCodecs.VideoRTXCodecParameters.RTPCodecCapability)
+
+	var codecsToRegister []codecToRegister
 	for _, codec := range protoCodecs.VideoCodecsParameters {
+		codecsToRegister = append(codecsToRegister, codecToRegister{RTPCodecParameters: codec, strictFmtp: false})
+	}
+	codecsToRegister = append(codecsToRegister,
+		codecToRegister{
+			RTPCodecParameters: protoCodecs.H264ProfileLevelId4d001fPacketizationMode0CodecParameters,
+			strictFmtp:         true,
+		},
+		codecToRegister{
+			RTPCodecParameters: protoCodecs.H264ProfileLevelId4d001fPacketizationMode1CodecParameters,
+			strictFmtp:         true,
+		},
+	)
+
+	for _, codec := range codecsToRegister {
 		if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == protoCodecs.H264HighProfileFmtp {
 			continue
 		}
 		if mime.IsMimeTypeStringRTX(codec.MimeType) {
 			continue
 		}
-		if !IsCodecEnabled(codecs, codec.RTPCodecCapability) {
+		if !isCodecEnabledWithFmtp(codecs, codec.RTPCodecCapability, codec.strictFmtp) {
 			continue
 		}

 		cp := codec
 		cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Video
-		if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil {
+		if err := me.RegisterCodec(cp.RTPCodecParameters, webrtc.RTPCodecTypeVideo); err != nil {
 			return err
 		}

@@ -76,10 +97,10 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 			continue
 		}

-		cp = protoCodecs.VideoRTXCodecParameters
-		cp.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType)
-		cp.PayloadType = codec.PayloadType + 1
-		if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil {
+		cpRtx := protoCodecs.VideoRTXCodecParameters
+		cpRtx.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType)
+		cpRtx.PayloadType = codec.PayloadType + 1
+		if err := me.RegisterCodec(cpRtx, webrtc.RTPCodecTypeVideo); err != nil {
 			return err
 		}
 	}
@@ -116,11 +137,15 @@ func createMediaEngine(codecs []*livekit.Codec, config DirectionConfig, filterOu
 }

 func IsCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool {
+	return isCodecEnabledWithFmtp(codecs, cap, false)
+}
+
+func isCodecEnabledWithFmtp(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability, strictFmtp bool) bool {
 	for _, codec := range codecs {
 		if !mime.IsMimeTypeStringEqual(codec.Mime, cap.MimeType) {
 			continue
 		}
-		if codec.FmtpLine == "" || strings.EqualFold(codec.FmtpLine, cap.SDPFmtpLine) {
+		if (!strictFmtp && codec.FmtpLine == "") || strings.EqualFold(codec.FmtpLine, cap.SDPFmtpLine) {
 			return true
 		}
 	}
@@ -15,6 +15,8 @@
 package rtc

 import (
+	"strconv"
+	"strings"
 	"testing"

 	"github.com/pion/webrtc/v4"
@@ -22,6 +24,13 @@ import (

 	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/config"
+)
+
+const (
+	h264MainProfilePacketizationMode0Fmtp = "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f"
+	h264MainProfilePacketizationMode1Fmtp = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f"
 )

 func TestIsCodecEnabled(t *testing.T) {
@@ -38,4 +47,174 @@ func TestIsCodecEnabled(t *testing.T) {
 		require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String()}))
 		require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP8.String()}))
 	})
+
+	t.Run("strict fmtp requires an exact fmtp, mime alone is not enough", func(t *testing.T) {
+		cap := webrtc.RTPCodecCapability{
+			MimeType:    mime.MimeTypeH264.String(),
+			SDPFmtpLine: h264MainProfilePacketizationMode1Fmtp,
+		}
+
+		// mime-only config opts into every non-strict codec, but never a strict one
+		mimeOnly := []*livekit.Codec{{Mime: "video/h264"}}
+		require.True(t, isCodecEnabledWithFmtp(mimeOnly, cap, false))
+		require.False(t, isCodecEnabledWithFmtp(mimeOnly, cap, true))
+
+		// spelling the fmtp out enables it
+		explicit := []*livekit.Codec{{Mime: "video/h264", FmtpLine: h264MainProfilePacketizationMode1Fmtp}}
+		require.True(t, isCodecEnabledWithFmtp(explicit, cap, true))
+
+		// ...but only for the profile that was asked for
+		require.False(t, isCodecEnabledWithFmtp(explicit, webrtc.RTPCodecCapability{
+			MimeType:    mime.MimeTypeH264.String(),
+			SDPFmtpLine: h264MainProfilePacketizationMode0Fmtp,
+		}, true))
+	})
+}
+
+type offeredCodec struct {
+	payloadType webrtc.PayloadType
+	name        string
+	fmtp        string
+}
+
+func (o offeredCodec) String() string {
+	return strconv.Itoa(int(o.payloadType)) + " " + o.name + " " + o.fmtp
+}
+
+// offeredVideoCodecs returns the video codecs an offer built from enabledCodecs carries.
+// It goes through a real offer rather than inspecting the MediaEngine so that payload
+// type assignment, which is what collides, is covered too.
+func offeredVideoCodecs(t *testing.T, enabledCodecs []*livekit.Codec, filterOutH264HighProfile bool) []offeredCodec {
+	t.Helper()
+
+	me, err := createMediaEngine(enabledCodecs, DirectionConfig{}, filterOutH264HighProfile)
+	require.NoError(t, err)
+
+	pc, err := webrtc.NewAPI(webrtc.WithMediaEngine(me)).NewPeerConnection(webrtc.Configuration{})
+	require.NoError(t, err)
+	defer pc.Close()
+
+	_, err = pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo)
+	require.NoError(t, err)
+
+	offer, err := pc.CreateOffer(nil)
+	require.NoError(t, err)
+
+	var codecs []offeredCodec
+	indexByPayloadType := make(map[webrtc.PayloadType]int)
+	for _, line := range strings.Split(offer.SDP, "\r\n") {
+		attr, value, found := strings.Cut(line, ":")
+		if !found || (attr != "a=rtpmap" && attr != "a=fmtp") {
+			continue
+		}
+
+		rawPayloadType, rest, _ := strings.Cut(value, " ")
+		payloadType, err := strconv.Atoi(rawPayloadType)
+		require.NoError(t, err)
+
+		if attr == "a=rtpmap" {
+			indexByPayloadType[webrtc.PayloadType(payloadType)] = len(codecs)
+			codecs = append(codecs, offeredCodec{payloadType: webrtc.PayloadType(payloadType), name: rest})
+			continue
+		}
+
+		i, ok := indexByPayloadType[webrtc.PayloadType(payloadType)]
+		require.True(t, ok, "fmtp for payload type %d without an rtpmap", payloadType)
+		codecs[i].fmtp = rest
+	}
+
+	require.NotEmpty(t, codecs)
+	return codecs
+}
+
+func hasOfferedFmtp(codecs []offeredCodec, mimeType mime.MimeType, fmtp string) bool {
+	for _, c := range codecs {
+		name, _, _ := strings.Cut(c.name, "/")
+		if mime.NormalizeMimeType("video/"+name) == mimeType && c.fmtp == fmtp {
+			return true
+		}
+	}
+	return false
+}
+
+// defaultEnabledCodecs mirrors what roomallocator derives from config, so these tests
+// track the shipped default codec set instead of a hand-maintained copy of it.
+func defaultEnabledCodecs() []*livekit.Codec {
+	codecs := make([]*livekit.Codec, 0, len(config.DefaultConfig.Room.EnabledCodecs))
+	for _, c := range config.DefaultConfig.Room.EnabledCodecs {
+		codecs = append(codecs, &livekit.Codec{Mime: c.Mime, FmtpLine: c.FmtpLine})
+	}
+	return codecs
+}
+
+func TestRegisterCodecsH264Profiles(t *testing.T) {
+	t.Run("baseline and constrained baseline are offered by default", func(t *testing.T) {
+		codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), false)
+
+		for _, fmtp := range []string{
+			"level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
+			"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+			"level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f",
+			"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
+			"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032",
+		} {
+			require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, fmtp), "missing %q in %v", fmtp, codecs)
+		}
+	})
+
+	t.Run("main profile is not offered unless explicitly enabled", func(t *testing.T) {
+		codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), false)
+
+		require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, h264MainProfilePacketizationMode0Fmtp), "%v", codecs)
+		require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, h264MainProfilePacketizationMode1Fmtp), "%v", codecs)
+	})
+
+	t.Run("main profile is offered when explicitly enabled", func(t *testing.T) {
+		for _, fmtp := range []string{
+			h264MainProfilePacketizationMode0Fmtp,
+			h264MainProfilePacketizationMode1Fmtp,
+		} {
+			t.Run(fmtp, func(t *testing.T) {
+				enabled := append(defaultEnabledCodecs(), &livekit.Codec{
+					Mime:     mime.MimeTypeH264.String(),
+					FmtpLine: fmtp,
+				})
+
+				codecs := offeredVideoCodecs(t, enabled, false)
+				require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, fmtp), "missing %q in %v", fmtp, codecs)
+			})
+		}
+	})
+
+	t.Run("high profile is filtered out for the answerer", func(t *testing.T) {
+		highProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032"
+
+		codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), true)
+		require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, highProfileFmtp), "%v", codecs)
+
+		// the other profiles survive the filter
+		require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264,
+			"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f"), "%v", codecs)
+	})
+}
+
+// Every video codec claims its own payload type plus the next one for its RTX stream,
+// so a codec whose payload type lands on another's RTX slot makes pion refuse to build
+// the MediaEngine at all.
+func TestRegisterCodecsPayloadTypesAreUnique(t *testing.T) {
+	enabled := append(defaultEnabledCodecs(),
+		&livekit.Codec{Mime: mime.MimeTypeH264.String(), FmtpLine: h264MainProfilePacketizationMode0Fmtp},
+		&livekit.Codec{Mime: mime.MimeTypeH264.String(), FmtpLine: h264MainProfilePacketizationMode1Fmtp},
+	)
+
+	for _, filterOutH264HighProfile := range []bool{false, true} {
+		codecs := offeredVideoCodecs(t, enabled, filterOutH264HighProfile)
+
+		seen := make(map[webrtc.PayloadType]offeredCodec, len(codecs))
+		for _, c := range codecs {
+			previous, duplicate := seen[c.payloadType]
+			require.False(t, duplicate, "payload type %d claimed by both %v and %v", c.payloadType, previous, c)
+			seen[c.payloadType] = c
+		}
+	}
 }