diff --git a/go.mod b/go.mod index 2377393c1..bc75e4cef 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 - github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe + github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6 github.com/livekit/psrpc v0.7.2 github.com/mackerelio/go-osstat v0.2.8 github.com/magefile/mage v1.17.2 diff --git a/go.sum b/go.sum index a78136c04..3b7d805ca 100644 --- a/go.sum +++ b/go.sum @@ -160,10 +160,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 h1:XHNNzebIKZRkLimla/hFGrAIX5EMWHctrgt3hLw7s+I= github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.50.4 h1:Pzg9p1lpu9TcxUFJqIlUGNP6m7mX8PBT+ETxXRpnmZ8= -github.com/livekit/protocol v1.50.4/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= -github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe h1:Vdw2fb0B0iFaAt45JHho11qb60+yiR/ZHyLBjDegka0= -github.com/livekit/protocol v1.50.5-0.20260726190139-cdb9dafee4fe/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= +github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6 h1:oavqDhmS2U5zFPs3S+jxTL6tEqsHxATxzfw3B/CKC+w= +github.com/livekit/protocol v1.50.5-0.20260728075222-28e604c046c6/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= github.com/livekit/psrpc v0.7.2 h1:6oZ+NODJ2pLyaT6VqDq1F4Qc/3TpDUSpyphj/P9MhQc= github.com/livekit/psrpc v0.7.2/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index cfc53eaaf..e3fc1ecd3 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -25,6 +25,11 @@ import ( "github.com/livekit/protocol/livekit" ) +type codecToRegister struct { + webrtc.RTPCodecParameters + strictFmtp bool +} + func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { // audio codecs if IsCodecEnabled(codecs, protoCodecs.OpusCodecParameters.RTPCodecCapability) { @@ -55,20 +60,36 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac // video codecs rtxEnabled := IsCodecEnabled(codecs, protoCodecs.VideoRTXCodecParameters.RTPCodecCapability) + + var codecsToRegister []codecToRegister for _, codec := range protoCodecs.VideoCodecsParameters { + codecsToRegister = append(codecsToRegister, codecToRegister{RTPCodecParameters: codec, strictFmtp: false}) + } + codecsToRegister = append(codecsToRegister, + codecToRegister{ + RTPCodecParameters: protoCodecs.H264ProfileLevelId4d001fPacketizationMode0CodecParameters, + strictFmtp: true, + }, + codecToRegister{ + RTPCodecParameters: protoCodecs.H264ProfileLevelId4d001fPacketizationMode1CodecParameters, + strictFmtp: true, + }, + ) + + for _, codec := range codecsToRegister { if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == protoCodecs.H264HighProfileFmtp { continue } if mime.IsMimeTypeStringRTX(codec.MimeType) { continue } - if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { + if !isCodecEnabledWithFmtp(codecs, codec.RTPCodecCapability, codec.strictFmtp) { continue } cp := codec cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Video - if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + if err := me.RegisterCodec(cp.RTPCodecParameters, webrtc.RTPCodecTypeVideo); err != nil { return err } @@ -76,10 +97,10 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac continue } - cp = protoCodecs.VideoRTXCodecParameters - cp.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType) - cp.PayloadType = codec.PayloadType + 1 - if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + cpRtx := protoCodecs.VideoRTXCodecParameters + cpRtx.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType) + cpRtx.PayloadType = codec.PayloadType + 1 + if err := me.RegisterCodec(cpRtx, webrtc.RTPCodecTypeVideo); err != nil { return err } } @@ -116,11 +137,15 @@ func createMediaEngine(codecs []*livekit.Codec, config DirectionConfig, filterOu } func IsCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool { + return isCodecEnabledWithFmtp(codecs, cap, false) +} + +func isCodecEnabledWithFmtp(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability, strictFmtp bool) bool { for _, codec := range codecs { if !mime.IsMimeTypeStringEqual(codec.Mime, cap.MimeType) { continue } - if codec.FmtpLine == "" || strings.EqualFold(codec.FmtpLine, cap.SDPFmtpLine) { + if (!strictFmtp && codec.FmtpLine == "") || strings.EqualFold(codec.FmtpLine, cap.SDPFmtpLine) { return true } } diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index aa7036875..79af78b8c 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -15,6 +15,8 @@ package rtc import ( + "strconv" + "strings" "testing" "github.com/pion/webrtc/v4" @@ -22,6 +24,13 @@ import ( "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + + "github.com/livekit/livekit-server/pkg/config" +) + +const ( + h264MainProfilePacketizationMode0Fmtp = "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f" + h264MainProfilePacketizationMode1Fmtp = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f" ) func TestIsCodecEnabled(t *testing.T) { @@ -38,4 +47,174 @@ func TestIsCodecEnabled(t *testing.T) { require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String()})) require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP8.String()})) }) + + t.Run("strict fmtp requires an exact fmtp, mime alone is not enough", func(t *testing.T) { + cap := webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH264.String(), + SDPFmtpLine: h264MainProfilePacketizationMode1Fmtp, + } + + // mime-only config opts into every non-strict codec, but never a strict one + mimeOnly := []*livekit.Codec{{Mime: "video/h264"}} + require.True(t, isCodecEnabledWithFmtp(mimeOnly, cap, false)) + require.False(t, isCodecEnabledWithFmtp(mimeOnly, cap, true)) + + // spelling the fmtp out enables it + explicit := []*livekit.Codec{{Mime: "video/h264", FmtpLine: h264MainProfilePacketizationMode1Fmtp}} + require.True(t, isCodecEnabledWithFmtp(explicit, cap, true)) + + // ...but only for the profile that was asked for + require.False(t, isCodecEnabledWithFmtp(explicit, webrtc.RTPCodecCapability{ + MimeType: mime.MimeTypeH264.String(), + SDPFmtpLine: h264MainProfilePacketizationMode0Fmtp, + }, true)) + }) +} + +type offeredCodec struct { + payloadType webrtc.PayloadType + name string + fmtp string +} + +func (o offeredCodec) String() string { + return strconv.Itoa(int(o.payloadType)) + " " + o.name + " " + o.fmtp +} + +// offeredVideoCodecs returns the video codecs an offer built from enabledCodecs carries. +// It goes through a real offer rather than inspecting the MediaEngine so that payload +// type assignment, which is what collides, is covered too. +func offeredVideoCodecs(t *testing.T, enabledCodecs []*livekit.Codec, filterOutH264HighProfile bool) []offeredCodec { + t.Helper() + + me, err := createMediaEngine(enabledCodecs, DirectionConfig{}, filterOutH264HighProfile) + require.NoError(t, err) + + pc, err := webrtc.NewAPI(webrtc.WithMediaEngine(me)).NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + _, err = pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo) + require.NoError(t, err) + + offer, err := pc.CreateOffer(nil) + require.NoError(t, err) + + var codecs []offeredCodec + indexByPayloadType := make(map[webrtc.PayloadType]int) + for _, line := range strings.Split(offer.SDP, "\r\n") { + attr, value, found := strings.Cut(line, ":") + if !found || (attr != "a=rtpmap" && attr != "a=fmtp") { + continue + } + + rawPayloadType, rest, _ := strings.Cut(value, " ") + payloadType, err := strconv.Atoi(rawPayloadType) + require.NoError(t, err) + + if attr == "a=rtpmap" { + indexByPayloadType[webrtc.PayloadType(payloadType)] = len(codecs) + codecs = append(codecs, offeredCodec{payloadType: webrtc.PayloadType(payloadType), name: rest}) + continue + } + + i, ok := indexByPayloadType[webrtc.PayloadType(payloadType)] + require.True(t, ok, "fmtp for payload type %d without an rtpmap", payloadType) + codecs[i].fmtp = rest + } + + require.NotEmpty(t, codecs) + return codecs +} + +func hasOfferedFmtp(codecs []offeredCodec, mimeType mime.MimeType, fmtp string) bool { + for _, c := range codecs { + name, _, _ := strings.Cut(c.name, "/") + if mime.NormalizeMimeType("video/"+name) == mimeType && c.fmtp == fmtp { + return true + } + } + return false +} + +// defaultEnabledCodecs mirrors what roomallocator derives from config, so these tests +// track the shipped default codec set instead of a hand-maintained copy of it. +func defaultEnabledCodecs() []*livekit.Codec { + codecs := make([]*livekit.Codec, 0, len(config.DefaultConfig.Room.EnabledCodecs)) + for _, c := range config.DefaultConfig.Room.EnabledCodecs { + codecs = append(codecs, &livekit.Codec{Mime: c.Mime, FmtpLine: c.FmtpLine}) + } + return codecs +} + +func TestRegisterCodecsH264Profiles(t *testing.T) { + t.Run("baseline and constrained baseline are offered by default", func(t *testing.T) { + codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), false) + + for _, fmtp := range []string{ + "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f", + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032", + } { + require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, fmtp), "missing %q in %v", fmtp, codecs) + } + }) + + t.Run("main profile is not offered unless explicitly enabled", func(t *testing.T) { + codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), false) + + require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, h264MainProfilePacketizationMode0Fmtp), "%v", codecs) + require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, h264MainProfilePacketizationMode1Fmtp), "%v", codecs) + }) + + t.Run("main profile is offered when explicitly enabled", func(t *testing.T) { + for _, fmtp := range []string{ + h264MainProfilePacketizationMode0Fmtp, + h264MainProfilePacketizationMode1Fmtp, + } { + t.Run(fmtp, func(t *testing.T) { + enabled := append(defaultEnabledCodecs(), &livekit.Codec{ + Mime: mime.MimeTypeH264.String(), + FmtpLine: fmtp, + }) + + codecs := offeredVideoCodecs(t, enabled, false) + require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, fmtp), "missing %q in %v", fmtp, codecs) + }) + } + }) + + t.Run("high profile is filtered out for the answerer", func(t *testing.T) { + highProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" + + codecs := offeredVideoCodecs(t, defaultEnabledCodecs(), true) + require.False(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, highProfileFmtp), "%v", codecs) + + // the other profiles survive the filter + require.True(t, hasOfferedFmtp(codecs, mime.MimeTypeH264, + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f"), "%v", codecs) + }) +} + +// Every video codec claims its own payload type plus the next one for its RTX stream, +// so a codec whose payload type lands on another's RTX slot makes pion refuse to build +// the MediaEngine at all. +func TestRegisterCodecsPayloadTypesAreUnique(t *testing.T) { + enabled := append(defaultEnabledCodecs(), + &livekit.Codec{Mime: mime.MimeTypeH264.String(), FmtpLine: h264MainProfilePacketizationMode0Fmtp}, + &livekit.Codec{Mime: mime.MimeTypeH264.String(), FmtpLine: h264MainProfilePacketizationMode1Fmtp}, + ) + + for _, filterOutH264HighProfile := range []bool{false, true} { + codecs := offeredVideoCodecs(t, enabled, filterOutH264HighProfile) + + seen := make(map[webrtc.PayloadType]offeredCodec, len(codecs)) + for _, c := range codecs { + previous, duplicate := seen[c.payloadType] + require.False(t, duplicate, "payload type %d claimed by both %v and %v", c.payloadType, previous, c) + seen[c.payloadType] = c + } + } }