From 8c2fc0bcd9bc4b5c10ea4d65d16bf6ad4a0f2406 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 23 Jun 2025 22:39:12 +0800 Subject: [PATCH] Fix svc encoding for chrome mobile on iOS (#3751) The browser could send rtp packets of svc encoding without DD extension while the sdp negotiates it, sfu detects extension in rtp packet for this case. --- pkg/clientconfiguration/conf.go | 15 --------------- pkg/rtc/mediatrack.go | 1 - pkg/sfu/buffer/buffer.go | 4 ++-- pkg/sfu/forwarder.go | 18 ++++++++++++++---- pkg/sfu/receiver.go | 27 +++------------------------ 5 files changed, 19 insertions(+), 46 deletions(-) diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index 98515a1ed..a8f8ff61c 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -49,21 +49,6 @@ var StaticConfigurations = []ConfigurationItem{ }, Merge: true, }, - // disable advanced codecs when publishing using JS SDK from iOS, - // seeing publish failures (no DD header extension found) when Chrome Mobile publishes VP9, - // being defensive and disabling advanced codecs - { - Match: must.Get(NewScriptMatch(`c.os == "ios" && c.sdk == "js"`)), - Configuration: &livekit.ClientConfiguration{ - DisabledCodecs: &livekit.DisabledCodecs{ - Publish: []*livekit.Codec{ - {Mime: mime.MimeTypeVP9.String()}, - {Mime: mime.MimeTypeAV1.String()}, - }, - }, - }, - Merge: true, - }, { Match: must.Get(NewScriptMatch(`(c.device_model == "xiaomi 2201117ti" && c.os == "android") || ((c.browser == "firefox" || c.browser == "firefox mobile") && (c.os == "linux" || c.os == "android"))`)), diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 90e50c0a2..595bc39f4 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -316,7 +316,6 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe sfu.WithPliThrottleConfig(t.params.PLIThrottleConfig), sfu.WithAudioConfig(t.params.AudioConfig), sfu.WithLoadBalanceThreshold(20), - sfu.WithStreamTrackers(), sfu.WithForwardStats(t.params.ForwardStats), ) newWR.OnCloseHandler(func() { diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index bf2b5c0fb..5deff6beb 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -890,8 +890,8 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat ddVal, videoLayer, err := b.ddParser.Parse(ep.Packet) if err != nil { if errors.Is(err, ErrDDExtentionNotFound) { - if b.mime == mime.MimeTypeVP8 { - b.logger.Infow("dd extension not found for vp8 packet, disable dd parser") + if b.mime == mime.MimeTypeVP8 || b.mime == mime.MimeTypeVP9 { + b.logger.Infow("dd extension not found, disable dd parser") b.ddParser = nil b.createFrameRateCalculator() } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 5251e7e90..e1eb05d4c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -231,6 +231,7 @@ type Forwarder struct { dummyStartTSOffset uint64 refInfos [buffer.DefaultMaxLayerSpatial + 1]refInfo refIsSVC bool + isDDAvailable bool provisional *VideoAllocationProvisional @@ -341,8 +342,8 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ case mime.MimeTypeVP9: // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - isDDAvailable := ddAvailable(extensions) - if isDDAvailable { + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { if f.vls != nil { f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) } else { @@ -359,8 +360,8 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ case mime.MimeTypeAV1: // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - isDDAvailable := ddAvailable(extensions) - if isDDAvailable { + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { if f.vls != nil { f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) } else { @@ -2002,6 +2003,15 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in result := f.vls.Select(extPkt, layer) if !result.IsSelected { + if f.isDDAvailable && extPkt.DependencyDescriptor == nil { + f.isDDAvailable = false + switch f.mime { + case mime.MimeTypeVP9: + f.vls = videolayerselector.NewVP9FromOther(f.vls) + case mime.MimeTypeAV1: + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + } + } tp.shouldDrop = true if f.started && result.IsRelevant { // call to update highest incoming sequence number and other internal structures diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 3a3f8503a..645146ab1 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -34,7 +34,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" "github.com/livekit/livekit-server/pkg/sfu/mime" - dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) @@ -175,7 +174,6 @@ type WebRTCReceiver struct { onCloseHandler func() closeOnce sync.Once closed atomic.Bool - useTrackers bool trackInfo atomic.Pointer[livekit.TrackInfo] onRTCP func([]rtcp.Packet) @@ -219,14 +217,6 @@ func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts { } } -// WithStreamTrackers enables StreamTracker use for simulcast -func WithStreamTrackers() ReceiverOpts { - return func(w *WebRTCReceiver) *WebRTCReceiver { - w.useTrackers = true - return w - } -} - // WithLoadBalanceThreshold enables parallelization of packet writes when downTracks exceeds threshold // Value should be between 3 and 150. // For a server handling a few large rooms, use a smaller value (required to handle very large (250+ participant) rooms). @@ -296,15 +286,6 @@ func NewWebRTCReceiver( w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig) w.streamTrackerManager.SetListener(w) - // SVC-TODO: Handle DD for non-SVC cases??? - if w.isSVC { - for _, ext := range receiver.GetParameters().HeaderExtensions { - if ext.URI == dd.ExtensionURI { - w.streamTrackerManager.AddDependencyDescriptorTrackers() - break - } - } - } return w } @@ -452,10 +433,6 @@ func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) erro buff.SetRTT(rtt) buff.SetPaused(w.streamTrackerManager.IsPaused()) - if w.Kind() == webrtc.RTPCodecTypeVideo && w.useTrackers { - w.streamTrackerManager.AddTracker(layer) - } - go w.forwardRTP(layer, buff) return nil } @@ -769,7 +746,6 @@ func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) { w.logger.Errorw("invalid layer", nil, "layer", layer) return } - spatialTrackers[layer] = w.streamTrackerManager.GetTracker(layer) pktBuf := make([]byte, bucket.MaxPktSize) for { @@ -822,6 +798,9 @@ func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) { if spatialTrackers[spatialLayer] == nil { spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer) if spatialTrackers[spatialLayer] == nil { + if w.isSVC && pkt.DependencyDescriptor != nil { + w.streamTrackerManager.AddDependencyDescriptorTrackers() + } spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer) } }