report video size from media data for whip (#4211)

pkg/rtc/mediatrack.go

@@ -75,26 +75,27 @@ type MediaTrack struct {
 }
 
 type MediaTrackParams struct {
-	ParticipantID                   func() livekit.ParticipantID
-	ParticipantIdentity             livekit.ParticipantIdentity
-	ParticipantVersion              uint32
-	ParticipantCountry              string
-	BufferFactory                   *buffer.Factory
-	ReceiverConfig                  ReceiverConfig
-	SubscriberConfig                DirectionConfig
-	PLIThrottleConfig               sfu.PLIThrottleConfig
-	AudioConfig                     sfu.AudioConfig
-	VideoConfig                     config.VideoConfig
-	Telemetry                       telemetry.TelemetryService
-	Logger                          logger.Logger
-	Reporter                        roomobs.TrackReporter
-	SimTracks                       map[uint32]interceptor.SimulcastTrackInfo
-	OnRTCP                          func([]rtcp.Packet)
-	ForwardStats                    *sfu.ForwardStats
-	OnTrackEverSubscribed           func(livekit.TrackID)
-	ShouldRegressCodec              func() bool
-	PreferVideoSizeFromMedia        bool
-	EnableRTPStreamRestartDetection bool
+	ParticipantID                    func() livekit.ParticipantID
+	ParticipantIdentity              livekit.ParticipantIdentity
+	ParticipantVersion               uint32
+	ParticipantCountry               string
+	BufferFactory                    *buffer.Factory
+	ReceiverConfig                   ReceiverConfig
+	SubscriberConfig                 DirectionConfig
+	PLIThrottleConfig                sfu.PLIThrottleConfig
+	AudioConfig                      sfu.AudioConfig
+	VideoConfig                      config.VideoConfig
+	Telemetry                        telemetry.TelemetryService
+	Logger                           logger.Logger
+	Reporter                         roomobs.TrackReporter
+	SimTracks                        map[uint32]interceptor.SimulcastTrackInfo
+	OnRTCP                           func([]rtcp.Packet)
+	ForwardStats                     *sfu.ForwardStats
+	OnTrackEverSubscribed            func(livekit.TrackID)
+	ShouldRegressCodec               func() bool
+	PreferVideoSizeFromMedia         bool
+	EnableRTPStreamRestartDetection  bool
+	UpdateTrackInfoByVideoSizeChange bool
 }
 
 func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack {
@@ -506,6 +507,10 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe
 
 		// update subscriber video layers when video size changes
 		newWR.OnVideoSizeChanged(func() {
+			if t.params.UpdateTrackInfoByVideoSizeChange {
+				t.MediaTrackReceiver.UpdateVideoSize(mimeType, newWR.VideoSizes())
+			}
+
 			t.MediaTrackSubscriptions.UpdateVideoLayers()
 		})
 	}

pkg/rtc/mediatrackreceiver.go

@@ -965,6 +965,52 @@ func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTr
 	t.params.Logger.Debugw("updated video track", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo))
 }
 
+func (t *MediaTrackReceiver) UpdateVideoSize(mimeType mime.MimeType, sizes []buffer.VideoSize) {
+	var changed bool
+	t.lock.Lock()
+	trackInfo := t.TrackInfo()
+	clonedInfo := utils.CloneProto(trackInfo)
+	var maxWidth, maxHeight uint32
+	for _, size := range sizes {
+		if size.Width > maxWidth {
+			maxWidth = size.Width
+			maxHeight = size.Height
+		}
+	}
+
+	if clonedInfo.Width != maxWidth || clonedInfo.Height != maxHeight {
+		clonedInfo.Width = maxWidth
+		clonedInfo.Height = maxHeight
+		changed = true
+	}
+
+	for _, c := range clonedInfo.Codecs {
+		if mime.NormalizeMimeType(c.MimeType) == mimeType {
+			for i, l := range c.Layers {
+				if i < len(sizes) && (sizes[i].Width != 0 || sizes[i].Height != 0) &&
+					(l.Width != sizes[i].Width || l.Height != sizes[i].Height) {
+					l.Width = sizes[i].Width
+					l.Height = sizes[i].Height
+					changed = true
+				}
+			}
+		}
+	}
+
+	if !changed {
+		t.lock.Unlock()
+		return
+	}
+
+	t.trackInfo.Store(clonedInfo)
+	t.lock.Unlock()
+
+	t.updateTrackInfoOfReceivers()
+
+	t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), clonedInfo)
+	t.params.Logger.Debugw("updated video sizes", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo))
+}
+
 func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo {
 	return t.trackInfo.Load()
 }

pkg/rtc/participant.go

@@ -3284,8 +3284,9 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo)
 		ShouldRegressCodec: func() bool {
 			return p.helper().ShouldRegressCodec()
 		},
-		PreferVideoSizeFromMedia:        p.params.PreferVideoSizeFromMedia,
-		EnableRTPStreamRestartDetection: p.params.EnableRTPStreamRestartDetection,
+		PreferVideoSizeFromMedia:         p.params.PreferVideoSizeFromMedia,
+		EnableRTPStreamRestartDetection:  p.params.EnableRTPStreamRestartDetection,
+		UpdateTrackInfoByVideoSizeChange: p.params.UseOneShotSignallingMode,
 	}, ti)
 
 	mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange)