From 0c33b8c671db470125e9c50b68aa723febacdbc0 Mon Sep 17 00:00:00 2001 From: Anunay Maheshwari Date: Tue, 20 Jan 2026 20:54:32 +0530 Subject: [PATCH] chore: move codecs/mime stuff to protocol (#4255) --- go.mod | 8 +- go.sum | 18 +- pkg/clientconfiguration/conf.go | 2 +- pkg/config/config.go | 2 +- pkg/rtc/dynacast/dynacastmanager_test.go | 2 +- pkg/rtc/dynacast/dynacastmanageraudio.go | 2 +- pkg/rtc/dynacast/dynacastmanagerbase.go | 2 +- pkg/rtc/dynacast/dynacastmanagervideo.go | 2 +- pkg/rtc/dynacast/dynacastqualityaudio.go | 2 +- pkg/rtc/dynacast/dynacastqualityvideo.go | 2 +- pkg/rtc/dynacast/interfaces.go | 2 +- pkg/rtc/mediaengine.go | 147 +------ pkg/rtc/mediaengine_test.go | 2 +- pkg/rtc/mediatrack.go | 2 +- pkg/rtc/mediatrack_test.go | 2 +- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/mediatracksubscriptions.go | 2 +- pkg/rtc/participant.go | 2 +- pkg/rtc/participant_internal_test.go | 7 +- pkg/rtc/participant_sdp.go | 2 +- pkg/rtc/subscribedtrack.go | 2 +- pkg/rtc/transport.go | 2 +- pkg/rtc/transport_test.go | 2 +- pkg/rtc/types/interfaces.go | 2 +- .../typesfakes/fake_local_media_track.go | 2 +- pkg/rtc/types/typesfakes/fake_media_track.go | 2 +- pkg/rtc/utils.go | 2 +- pkg/rtc/wrappedreceiver.go | 7 +- pkg/sfu/buffer/buffer_base.go | 2 +- pkg/sfu/buffer/videolayerutils.go | 2 +- pkg/sfu/buffer/videolayerutils_test.go | 2 +- pkg/sfu/connectionquality/connectionstats.go | 2 +- .../connectionquality/connectionstats_test.go | 2 +- pkg/sfu/downtrack.go | 2 +- pkg/sfu/forwarder.go | 2 +- pkg/sfu/mime/mimetype.go | 365 ------------------ pkg/sfu/receiver.go | 2 +- pkg/sfu/receiver_base.go | 2 +- pkg/sfu/streamtrackermanager.go | 2 +- pkg/sfu/utils/helpers.go | 2 +- pkg/telemetry/events.go | 2 +- .../telemetryfakes/fake_telemetry_service.go | 2 +- pkg/telemetry/telemetryservice.go | 2 +- test/client/client.go | 2 +- test/client/trackwriter.go | 2 +- test/singlenode_test.go | 2 +- 46 files changed, 71 insertions(+), 561 deletions(-) delete mode 100644 pkg/sfu/mime/mimetype.go diff --git a/go.mod b/go.mod index 5318b50f8..a8ebe140f 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 github.com/livekit/mediatransportutil v0.0.0-20260113174415-2e8ba344fca3 - github.com/livekit/protocol v1.44.0 + github.com/livekit/protocol v1.44.1-0.20260120144210-9d3e62e3ff71 github.com/livekit/psrpc v0.7.1 github.com/mackerelio/go-osstat v0.2.6 github.com/magefile/mage v1.15.0 @@ -113,7 +113,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lithammer/shortuuid/v4 v4.2.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -152,8 +152,8 @@ require ( golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/tools v0.41.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260112192933-99fd39fd28a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260112192933-99fd39fd28a9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3 // indirect google.golang.org/grpc v1.78.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index b45eac943..430536258 100644 --- a/go.sum +++ b/go.sum @@ -154,8 +154,8 @@ github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -177,10 +177,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260113174415-2e8ba344fca3 h1:v1Xc/q/547TjLX7Nw5y2vXNnmV0XYFAbhTJrtErQeDA= github.com/livekit/mediatransportutil v0.0.0-20260113174415-2e8ba344fca3/go.mod h1:QBx/KHV6Vv00ggibg/WrOlqrkTciEA2Hc9DGWYr3Q9U= -github.com/livekit/protocol v1.43.5-0.20260114074149-a8bb8204ce69 h1:cD82r488SxGYL5MX1lLuLLjmdnNoC+u5TIepxQmSB40= -github.com/livekit/protocol v1.43.5-0.20260114074149-a8bb8204ce69/go.mod h1:BLJHYHErQTu3+fnmfGrzN6CbHxNYiooFIIYGYxXxotw= -github.com/livekit/protocol v1.44.0 h1:OB2IiwmntSkdXWszajcpS6vPXnld8KU7XS11ekRq1Gg= -github.com/livekit/protocol v1.44.0/go.mod h1:BLJHYHErQTu3+fnmfGrzN6CbHxNYiooFIIYGYxXxotw= +github.com/livekit/protocol v1.44.1-0.20260120144210-9d3e62e3ff71 h1:92p9LMmeJKVjDAQP4fdeDtkWy3nDxvVbGnDdhkuQ0CM= +github.com/livekit/protocol v1.44.1-0.20260120144210-9d3e62e3ff71/go.mod h1:BLJHYHErQTu3+fnmfGrzN6CbHxNYiooFIIYGYxXxotw= github.com/livekit/psrpc v0.7.1 h1:ms37az0QTD3UXIWuUC5D/SkmKOlRMVRsI261eBWu/Vw= github.com/livekit/psrpc v0.7.1/go.mod h1:bZ4iHFQptTkbPnB0LasvRNu/OBYXEu1NA6O5BMFo9kk= github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0= @@ -486,10 +484,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260112192933-99fd39fd28a9 h1:4DKBrmaqeptdEzp21EfrOEh8LE7PJ5ywH6wydSbOfGY= -google.golang.org/genproto/googleapis/api v0.0.0-20260112192933-99fd39fd28a9/go.mod h1:dd646eSK+Dk9kxVBl1nChEOhJPtMXriCcVb4x3o6J+E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260112192933-99fd39fd28a9 h1:IY6/YYRrFUk0JPp0xOVctvFIVuRnjccihY5kxf5g0TE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260112192933-99fd39fd28a9/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 h1:X9z6obt+cWRX8XjDVOn+SZWhWe5kZHm46TThU9j+jss= +google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3/go.mod h1:dd646eSK+Dk9kxVBl1nChEOhJPtMXriCcVb4x3o6J+E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3 h1:C4WAdL+FbjnGlpp2S+HMVhBeCq2Lcib4xZqfPNF6OoQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260114163908-3f89685c29c3/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index a8f8ff61c..7fb5ae8d5 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -15,7 +15,7 @@ package clientconfiguration import ( - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils/must" ) diff --git a/pkg/config/config.go b/pkg/config/config.go index 87aaafc5b..baa608d33 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -27,6 +27,7 @@ import ( "gopkg.in/yaml.v3" "github.com/livekit/mediatransportutil/pkg/rtcconfig" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" redisLiveKit "github.com/livekit/protocol/redis" @@ -38,7 +39,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe" "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" ) diff --git a/pkg/rtc/dynacast/dynacastmanager_test.go b/pkg/rtc/dynacast/dynacastmanager_test.go index 9bb876db0..6fffcbe31 100644 --- a/pkg/rtc/dynacast/dynacastmanager_test.go +++ b/pkg/rtc/dynacast/dynacastmanager_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/dynacast/dynacastmanageraudio.go b/pkg/rtc/dynacast/dynacastmanageraudio.go index 542a58a1d..2e9bf6e85 100644 --- a/pkg/rtc/dynacast/dynacastmanageraudio.go +++ b/pkg/rtc/dynacast/dynacastmanageraudio.go @@ -18,7 +18,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" ) var _ DynacastManager = (*dynacastManagerAudio)(nil) diff --git a/pkg/rtc/dynacast/dynacastmanagerbase.go b/pkg/rtc/dynacast/dynacastmanagerbase.go index 94812acbc..a8e251ca2 100644 --- a/pkg/rtc/dynacast/dynacastmanagerbase.go +++ b/pkg/rtc/dynacast/dynacastmanagerbase.go @@ -19,9 +19,9 @@ import ( "golang.org/x/exp/maps" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/logger" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/utils" ) diff --git a/pkg/rtc/dynacast/dynacastmanagervideo.go b/pkg/rtc/dynacast/dynacastmanagervideo.go index 93120c84e..e2c871213 100644 --- a/pkg/rtc/dynacast/dynacastmanagervideo.go +++ b/pkg/rtc/dynacast/dynacastmanagervideo.go @@ -19,11 +19,11 @@ import ( "github.com/bep/debounce" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) var _ DynacastManager = (*dynacastManagerVideo)(nil) diff --git a/pkg/rtc/dynacast/dynacastqualityaudio.go b/pkg/rtc/dynacast/dynacastqualityaudio.go index 2414b5b66..817481d1d 100644 --- a/pkg/rtc/dynacast/dynacastqualityaudio.go +++ b/pkg/rtc/dynacast/dynacastqualityaudio.go @@ -17,7 +17,7 @@ package dynacast import ( "sync" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/dynacast/dynacastqualityvideo.go b/pkg/rtc/dynacast/dynacastqualityvideo.go index 47f9f388c..d2095043a 100644 --- a/pkg/rtc/dynacast/dynacastqualityvideo.go +++ b/pkg/rtc/dynacast/dynacastqualityvideo.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/dynacast/interfaces.go b/pkg/rtc/dynacast/interfaces.go index 42d1147aa..c544ff54a 100644 --- a/pkg/rtc/dynacast/interfaces.go +++ b/pkg/rtc/dynacast/interfaces.go @@ -15,10 +15,10 @@ package dynacast import ( + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) type DynacastManagerListener interface { diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index e1e7180a2..b0c452545 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -20,153 +20,28 @@ import ( "github.com/pion/webrtc/v4" - "github.com/livekit/livekit-server/pkg/sfu/mime" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) -var ( - OpusCodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeOpus.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "minptime=10;useinbandfec=1", - }, - PayloadType: 111, - } - - RedCodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRED.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "111/111", - }, - PayloadType: 63, - } - - PCMUCodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypePCMU.String(), - ClockRate: 8000, - }, - PayloadType: 0, - } - - PCMACodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypePCMA.String(), - ClockRate: 8000, - }, - PayloadType: 8, - } - - videoRTXCodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRTX.String(), - ClockRate: 90000, - }, - } - - vp8CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP8.String(), - ClockRate: 90000, - }, - PayloadType: 96, - } - - vp9ProfileId0CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=0", - }, - PayloadType: 98, - } - - vp9ProfileId1CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=1", - }, - PayloadType: 100, - } - - h264ProfileLevelId42e01fPacketizationMode0CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", - }, - PayloadType: 125, - } - - h264ProfileLevelId42e01fPacketizationMode1CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", - }, - PayloadType: 108, - } - - h264HighProfileFmtp = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" - h264HighProfileCodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: h264HighProfileFmtp, - }, - PayloadType: 123, - } - - av1CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeAV1.String(), - ClockRate: 90000, - }, - PayloadType: 35, - } - - h265CodecParameters = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH265.String(), - ClockRate: 90000, - }, - PayloadType: 116, - } - - videoCodecsParameters = []webrtc.RTPCodecParameters{ - vp8CodecParameters, - vp9ProfileId0CodecParameters, - vp9ProfileId1CodecParameters, - h264ProfileLevelId42e01fPacketizationMode0CodecParameters, - h264ProfileLevelId42e01fPacketizationMode1CodecParameters, - h264HighProfileCodecParameters, - av1CodecParameters, - h265CodecParameters, - } -) - func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { // audio codecs - if IsCodecEnabled(codecs, OpusCodecParameters.RTPCodecCapability) { - cp := OpusCodecParameters + if IsCodecEnabled(codecs, protoCodecs.OpusCodecParameters.RTPCodecCapability) { + cp := protoCodecs.OpusCodecParameters cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Audio if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeAudio); err != nil { return err } - if IsCodecEnabled(codecs, RedCodecParameters.RTPCodecCapability) { - if err := me.RegisterCodec(RedCodecParameters, webrtc.RTPCodecTypeAudio); err != nil { + if IsCodecEnabled(codecs, protoCodecs.RedCodecParameters.RTPCodecCapability) { + if err := me.RegisterCodec(protoCodecs.RedCodecParameters, webrtc.RTPCodecTypeAudio); err != nil { return err } } } - for _, codec := range []webrtc.RTPCodecParameters{PCMUCodecParameters, PCMACodecParameters} { + for _, codec := range []webrtc.RTPCodecParameters{protoCodecs.PCMUCodecParameters, protoCodecs.PCMACodecParameters} { if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { continue } @@ -179,9 +54,9 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac } // video codecs - rtxEnabled := IsCodecEnabled(codecs, videoRTXCodecParameters.RTPCodecCapability) - for _, codec := range videoCodecsParameters { - if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp { + rtxEnabled := IsCodecEnabled(codecs, protoCodecs.VideoRTXCodecParameters.RTPCodecCapability) + for _, codec := range protoCodecs.VideoCodecsParameters { + if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == protoCodecs.H264HighProfileFmtp { continue } if mime.IsMimeTypeStringRTX(codec.MimeType) { @@ -201,7 +76,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac continue } - cp = videoRTXCodecParameters + cp = protoCodecs.VideoRTXCodecParameters cp.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType) cp.PayloadType = codec.PayloadType + 1 if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index 122861a87..aa7036875 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -20,7 +20,7 @@ import ( "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index cc6ad170d..7c43edc2d 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -24,6 +24,7 @@ import ( "github.com/pion/webrtc/v4" "go.uber.org/atomic" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability/roomobs" @@ -36,7 +37,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" "github.com/livekit/livekit-server/pkg/sfu/interceptor" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry" util "github.com/livekit/mediatransportutil" ) diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 7839a943d..bd3b09dd6 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index b109dfdf0..9c4d64906 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -28,6 +28,7 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -35,7 +36,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" "github.com/livekit/livekit-server/pkg/telemetry" sutils "github.com/livekit/livekit-server/pkg/utils" diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index fc529b041..c4dd38cd7 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -19,7 +19,7 @@ import ( "slices" "sync" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/pion/webrtc/v4" diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a436098f3..950b14a96 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -39,6 +39,7 @@ import ( "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability" @@ -61,7 +62,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" "github.com/livekit/livekit-server/pkg/sfu/interceptor" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index f4d265361..a48879801 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -26,9 +26,10 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes" "github.com/livekit/protocol/auth" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability/roomobs" @@ -625,10 +626,10 @@ func TestPreferAudioCodecForRed(t *testing.T) { participant.SetMigrateState(types.MigrateStateComplete) me := webrtc.MediaEngine{} - opusCodecParameters := OpusCodecParameters + opusCodecParameters := protoCodecs.OpusCodecParameters opusCodecParameters.RTPCodecCapability.RTCPFeedback = []webrtc.RTCPFeedback{{Type: webrtc.TypeRTCPFBNACK}} require.NoError(t, me.RegisterCodec(opusCodecParameters, webrtc.RTPCodecTypeAudio)) - redCodecParameters := RedCodecParameters + redCodecParameters := protoCodecs.RedCodecParameters redCodecParameters.RTPCodecCapability.RTCPFeedback = []webrtc.RTCPFeedback{{Type: webrtc.TypeRTCPFBNACK}} require.NoError(t, me.RegisterCodec(redCodecParameters, webrtc.RTPCodecTypeAudio)) diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 828863f32..79cb595cd 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -24,7 +24,7 @@ import ( "github.com/pion/webrtc/v4" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" lksdp "github.com/livekit/protocol/sdp" diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 6daa69d54..67eed6cc8 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -25,6 +25,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry" sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability/roomobs" @@ -33,7 +34,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) const ( diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 1dbd2a4e2..66aa9b01c 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -49,7 +49,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/datachannel" sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" @@ -58,6 +57,7 @@ import ( "github.com/livekit/livekit-server/pkg/utils" lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/logger/pionlogger" diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index d27ec0aff..c48a2e4d2 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -28,8 +28,8 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index ec8b00e7f..4f0d6c698 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -22,6 +22,7 @@ import ( "github.com/pion/webrtc/v4" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability/roomobs" @@ -31,7 +32,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/telemetry" diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index a327509de..a06094ec2 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -6,7 +6,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index ba44585f1..95a4d0649 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -6,7 +6,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index 1a584b310..9653cc46d 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -23,7 +23,7 @@ import ( "github.com/pion/webrtc/v4" "google.golang.org/protobuf/proto" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index ad3a3b9ab..b6550d931 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -23,12 +23,13 @@ import ( "golang.org/x/exp/maps" "golang.org/x/exp/slices" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) // wrapper around WebRTC receiver, overriding its ID @@ -64,10 +65,10 @@ func NewWrappedReceiver(params WrappedReceiverParams) *WrappedReceiver { normalizedMimeType := mime.NormalizeMimeType(codecs[0].MimeType) if normalizedMimeType == mime.MimeTypeRED { // if upstream is opus/red, then add opus to match clients that don't support red - codecs = append(codecs, OpusCodecParameters) + codecs = append(codecs, protoCodecs.OpusCodecParameters) } else if !params.DisableRed && normalizedMimeType == mime.MimeTypeOpus { // if upstream is opus only and red enabled, add red to match clients that support red - codecs = append(codecs, RedCodecParameters) + codecs = append(codecs, protoCodecs.RedCodecParameters) // prefer red codec codecs[0], codecs[1] = codecs[1], codecs[0] } diff --git a/pkg/sfu/buffer/buffer_base.go b/pkg/sfu/buffer/buffer_base.go index 673285aeb..263bb6c76 100644 --- a/pkg/sfu/buffer/buffer_base.go +++ b/pkg/sfu/buffer/buffer_base.go @@ -31,13 +31,13 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/sfu/audio" - "github.com/livekit/livekit-server/pkg/sfu/mime" act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/mediatransportutil/pkg/bucket" "github.com/livekit/mediatransportutil/pkg/nack" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 2754b59a8..d80c0a77a 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -17,7 +17,7 @@ package buffer import ( "slices" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index a7c5ad26d..9382ec4e0 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 0668b9ca2..f0c71bdbf 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -22,11 +22,11 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" ) diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 73ba4aecd..13b61a645 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 361056e3d..f604278f4 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -33,6 +33,7 @@ import ( "go.uber.org/atomic" "go.uber.org/zap/zapcore" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" @@ -41,7 +42,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/bwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index d52bcaa04..f96413005 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -27,6 +27,7 @@ import ( "go.uber.org/zap/zapcore" "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -34,7 +35,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/codecmunger" - "github.com/livekit/livekit-server/pkg/sfu/mime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" diff --git a/pkg/sfu/mime/mimetype.go b/pkg/sfu/mime/mimetype.go deleted file mode 100644 index d336bccf0..000000000 --- a/pkg/sfu/mime/mimetype.go +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mime - -import ( - "strings" - - "github.com/pion/webrtc/v4" - - "github.com/livekit/protocol/observability/roomobs" -) - -const ( - MimeTypePrefixAudio = "audio/" - MimeTypePrefixVideo = "video/" -) - -type MimeTypeCodec int - -const ( - MimeTypeCodecUnknown MimeTypeCodec = iota - MimeTypeCodecH264 - MimeTypeCodecH265 - MimeTypeCodecOpus - MimeTypeCodecRED - MimeTypeCodecVP8 - MimeTypeCodecVP9 - MimeTypeCodecAV1 - MimeTypeCodecG722 - MimeTypeCodecPCMU - MimeTypeCodecPCMA - MimeTypeCodecRTX - MimeTypeCodecFlexFEC - MimeTypeCodecULPFEC -) - -func (m MimeTypeCodec) String() string { - switch m { - case MimeTypeCodecUnknown: - return "MimeTypeCodecUnknown" - case MimeTypeCodecH264: - return "H264" - case MimeTypeCodecH265: - return "H265" - case MimeTypeCodecOpus: - return "opus" - case MimeTypeCodecRED: - return "red" - case MimeTypeCodecVP8: - return "VP8" - case MimeTypeCodecVP9: - return "VP9" - case MimeTypeCodecAV1: - return "AV1" - case MimeTypeCodecG722: - return "G722" - case MimeTypeCodecPCMU: - return "PCMU" - case MimeTypeCodecPCMA: - return "PCMA" - case MimeTypeCodecRTX: - return "rtx" - case MimeTypeCodecFlexFEC: - return "flexfec" - case MimeTypeCodecULPFEC: - return "ulpfec" - } - - return "MimeTypeCodecUnknown" -} - -func (m MimeTypeCodec) ToMimeType() MimeType { - switch m { - case MimeTypeCodecUnknown: - return MimeTypeUnknown - case MimeTypeCodecH264: - return MimeTypeH264 - case MimeTypeCodecH265: - return MimeTypeH265 - case MimeTypeCodecOpus: - return MimeTypeOpus - case MimeTypeCodecRED: - return MimeTypeRED - case MimeTypeCodecVP8: - return MimeTypeVP8 - case MimeTypeCodecVP9: - return MimeTypeVP9 - case MimeTypeCodecAV1: - return MimeTypeAV1 - case MimeTypeCodecG722: - return MimeTypeG722 - case MimeTypeCodecPCMU: - return MimeTypePCMU - case MimeTypeCodecPCMA: - return MimeTypePCMA - case MimeTypeCodecRTX: - return MimeTypeRTX - case MimeTypeCodecFlexFEC: - return MimeTypeFlexFEC - case MimeTypeCodecULPFEC: - return MimeTypeULPFEC - } - - return MimeTypeUnknown -} - -func NormalizeMimeTypeCodec(codec string) MimeTypeCodec { - switch { - case strings.EqualFold(codec, "h264"): - return MimeTypeCodecH264 - case strings.EqualFold(codec, "h265"): - return MimeTypeCodecH265 - case strings.EqualFold(codec, "opus"): - return MimeTypeCodecOpus - case strings.EqualFold(codec, "red"): - return MimeTypeCodecRED - case strings.EqualFold(codec, "vp8"): - return MimeTypeCodecVP8 - case strings.EqualFold(codec, "vp9"): - return MimeTypeCodecVP9 - case strings.EqualFold(codec, "av1"): - return MimeTypeCodecAV1 - case strings.EqualFold(codec, "g722"): - return MimeTypeCodecG722 - case strings.EqualFold(codec, "pcmu"): - return MimeTypeCodecPCMU - case strings.EqualFold(codec, "pcma"): - return MimeTypeCodecPCMA - case strings.EqualFold(codec, "rtx"): - return MimeTypeCodecRTX - case strings.EqualFold(codec, "flexfec"): - return MimeTypeCodecFlexFEC - case strings.EqualFold(codec, "ulpfec"): - return MimeTypeCodecULPFEC - } - - return MimeTypeCodecUnknown -} - -func GetMimeTypeCodec(mime string) MimeTypeCodec { - i := strings.IndexByte(mime, '/') - if i == -1 { - return MimeTypeCodecUnknown - } - - return NormalizeMimeTypeCodec(mime[i+1:]) -} - -func IsMimeTypeCodecStringOpus(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecOpus -} - -func IsMimeTypeCodecStringRED(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED -} - -func IsMimeTypeCodecStringPCMA(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMA -} - -func IsMimeTypeCodecStringPCMU(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecPCMU -} - -func IsMimeTypeCodecStringH264(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264 -} - -type MimeType int - -const ( - MimeTypeUnknown MimeType = iota - MimeTypeH264 - MimeTypeH265 - MimeTypeOpus - MimeTypeRED - MimeTypeVP8 - MimeTypeVP9 - MimeTypeAV1 - MimeTypeG722 - MimeTypePCMU - MimeTypePCMA - MimeTypeRTX - MimeTypeFlexFEC - MimeTypeULPFEC -) - -func (m MimeType) String() string { - switch m { - case MimeTypeUnknown: - return "MimeTypeUnknown" - case MimeTypeH264: - return webrtc.MimeTypeH264 - case MimeTypeH265: - return webrtc.MimeTypeH265 - case MimeTypeOpus: - return webrtc.MimeTypeOpus - case MimeTypeRED: - return "audio/red" - case MimeTypeVP8: - return webrtc.MimeTypeVP8 - case MimeTypeVP9: - return webrtc.MimeTypeVP9 - case MimeTypeAV1: - return webrtc.MimeTypeAV1 - case MimeTypeG722: - return webrtc.MimeTypeG722 - case MimeTypePCMU: - return webrtc.MimeTypePCMU - case MimeTypePCMA: - return webrtc.MimeTypePCMA - case MimeTypeRTX: - return webrtc.MimeTypeRTX - case MimeTypeFlexFEC: - return webrtc.MimeTypeFlexFEC - case MimeTypeULPFEC: - return "video/ulpfec" - } - - return "MimeTypeUnknown" -} - -func (m MimeType) ReporterType() roomobs.MimeType { - switch m { - case MimeTypeUnknown: - return roomobs.MimeTypeUndefined - case MimeTypeH264: - return roomobs.MimeTypeVideoH264 - case MimeTypeH265: - return roomobs.MimeTypeVideoH265 - case MimeTypeOpus: - return roomobs.MimeTypeAudioOpus - case MimeTypeRED: - return roomobs.MimeTypeAudioRed - case MimeTypeVP8: - return roomobs.MimeTypeVideoVp8 - case MimeTypeVP9: - return roomobs.MimeTypeVideoVp9 - case MimeTypeAV1: - return roomobs.MimeTypeVideoAv1 - case MimeTypeG722: - return roomobs.MimeTypeAudioG722 - case MimeTypePCMU: - return roomobs.MimeTypeAudioPcmu - case MimeTypePCMA: - return roomobs.MimeTypeAudioPcma - case MimeTypeRTX: - return roomobs.MimeTypeVideoRtx - case MimeTypeFlexFEC: - return roomobs.MimeTypeVideoFlexfec - case MimeTypeULPFEC: - return roomobs.MimeTypeVideoUlpfec - } - - return roomobs.MimeTypeUndefined -} - -func NormalizeMimeType(mime string) MimeType { - switch { - case strings.EqualFold(mime, webrtc.MimeTypeH264): - return MimeTypeH264 - case strings.EqualFold(mime, webrtc.MimeTypeH265): - return MimeTypeH265 - case strings.EqualFold(mime, webrtc.MimeTypeOpus): - return MimeTypeOpus - case strings.EqualFold(mime, "audio/red"): - return MimeTypeRED - case strings.EqualFold(mime, webrtc.MimeTypeVP8): - return MimeTypeVP8 - case strings.EqualFold(mime, webrtc.MimeTypeVP9): - return MimeTypeVP9 - case strings.EqualFold(mime, webrtc.MimeTypeAV1): - return MimeTypeAV1 - case strings.EqualFold(mime, webrtc.MimeTypeG722): - return MimeTypeG722 - case strings.EqualFold(mime, webrtc.MimeTypePCMU): - return MimeTypePCMU - case strings.EqualFold(mime, webrtc.MimeTypePCMA): - return MimeTypePCMA - case strings.EqualFold(mime, webrtc.MimeTypeRTX): - return MimeTypeRTX - case strings.EqualFold(mime, webrtc.MimeTypeFlexFEC): - return MimeTypeFlexFEC - case strings.EqualFold(mime, "video/ulpfec"): - return MimeTypeULPFEC - } - - return MimeTypeUnknown -} - -func IsMimeTypeStringEqual(mime1 string, mime2 string) bool { - return NormalizeMimeType(mime1) == NormalizeMimeType(mime2) -} - -func IsMimeTypeStringAudio(mime string) bool { - return strings.HasPrefix(mime, MimeTypePrefixAudio) -} - -func IsMimeTypeAudio(mimeType MimeType) bool { - return strings.HasPrefix(mimeType.String(), MimeTypePrefixAudio) -} - -func IsMimeTypeStringVideo(mime string) bool { - return strings.HasPrefix(mime, MimeTypePrefixVideo) -} - -func IsMimeTypeVideo(mimeType MimeType) bool { - return strings.HasPrefix(mimeType.String(), MimeTypePrefixVideo) -} - -func IsMimeTypeSVCCapable(mimeType MimeType) bool { - switch mimeType { - case MimeTypeAV1, MimeTypeVP9: - return true - } - return false -} - -func IsMimeTypeStringSVCCapable(mime string) bool { - return IsMimeTypeSVCCapable(NormalizeMimeType(mime)) -} - -func IsMimeTypeStringRED(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeRED -} - -func IsMimeTypeStringOpus(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeOpus -} - -func IsMimeTypeStringPCMA(mime string) bool { - return NormalizeMimeType(mime) == MimeTypePCMA -} - -func IsMimeTypeStringPCMU(mime string) bool { - return NormalizeMimeType(mime) == MimeTypePCMU -} - -func IsMimeTypeStringRTX(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeRTX -} - -func IsMimeTypeStringVP8(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeVP8 -} - -func IsMimeTypeStringVP9(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeVP9 -} - -func IsMimeTypeStringH264(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeH264 -} diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 5a1482e75..68886a208 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -22,12 +22,12 @@ import ( "github.com/pion/rtcp" "github.com/pion/webrtc/v4" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" ) diff --git a/pkg/sfu/receiver_base.go b/pkg/sfu/receiver_base.go index ac878aaef..28b0993e2 100644 --- a/pkg/sfu/receiver_base.go +++ b/pkg/sfu/receiver_base.go @@ -27,6 +27,7 @@ import ( "go.uber.org/atomic" "github.com/livekit/mediatransportutil/pkg/bucket" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -34,7 +35,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index b10267392..2bc246a53 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -23,12 +23,12 @@ import ( "github.com/frostbyte73/core" "go.uber.org/atomic" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) diff --git a/pkg/sfu/utils/helpers.go b/pkg/sfu/utils/helpers.go index bfd40194d..7ce7eaac8 100644 --- a/pkg/sfu/utils/helpers.go +++ b/pkg/sfu/utils/helpers.go @@ -22,7 +22,7 @@ import ( "github.com/pion/rtp" "github.com/pion/webrtc/v4" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index a6c006450..30cee7106 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -20,8 +20,8 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index bc9109db5..855f80024 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -5,8 +5,8 @@ import ( "context" "sync" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/observability/roomobs" ) diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index cd48cd384..a22a31a6a 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -19,8 +19,8 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/webhook" diff --git a/test/client/client.go b/test/client/client.go index f98e82d2a..02d188fb2 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -40,6 +40,7 @@ import ( "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/signalling" @@ -49,7 +50,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) type SignalRequestHandler func(msg *livekit.SignalRequest) error diff --git a/test/client/trackwriter.go b/test/client/trackwriter.go index eae2da44f..09dad4560 100644 --- a/test/client/trackwriter.go +++ b/test/client/trackwriter.go @@ -26,7 +26,7 @@ import ( "github.com/pion/webrtc/v4/pkg/media/ivfreader" "github.com/pion/webrtc/v4/pkg/media/oggreader" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/logger" ) diff --git a/test/singlenode_test.go b/test/singlenode_test.go index 03b9e4635..3808c053a 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -33,13 +33,13 @@ import ( "go.uber.org/atomic" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/sfu/datachannel" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/testutils" testclient "github.com/livekit/livekit-server/test/client" )