mirror of
https://github.com/livekit/livekit.git
synced 2026-07-31 16:00:22 +00:00
Merge remote-tracking branch 'origin/master' into raja_pc
This commit is contained in:
boks1971
+1
-1
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -23,7 +23,7 @@ require (
|
||||
github.com/jxskiss/base62 v1.1.0
|
||||
github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731
|
||||
github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded
|
||||
github.com/livekit/protocol v1.39.4-0.20250813045730-236ffc7e5e12
|
||||
github.com/livekit/protocol v1.40.0
|
||||
github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741
|
||||
github.com/mackerelio/go-osstat v0.2.6
|
||||
github.com/magefile/mage v1.15.0
|
||||
@@ -32,7 +32,7 @@ require (
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/ory/dockertest/v3 v3.12.0
|
||||
github.com/pion/datachannel v1.5.10
|
||||
github.com/pion/dtls/v3 v3.0.6
|
||||
github.com/pion/dtls/v3 v3.0.7
|
||||
github.com/pion/ice/v4 v4.0.10
|
||||
github.com/pion/interceptor v0.1.40
|
||||
github.com/pion/rtcp v1.2.15
|
||||
@@ -40,7 +40,7 @@ require (
|
||||
github.com/pion/sctp v1.8.39
|
||||
github.com/pion/sdp/v3 v3.0.15
|
||||
github.com/pion/transport/v3 v3.0.7
|
||||
github.com/pion/turn/v4 v4.1.0
|
||||
github.com/pion/turn/v4 v4.1.1
|
||||
github.com/pion/webrtc/v4 v4.1.3
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.23.0
|
||||
|
||||
@@ -170,8 +170,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT
|
||||
github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
|
||||
github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded h1:ylZPdnlX1RW9Z15SD4mp87vT2D2shsk0hpLJwSPcq3g=
|
||||
github.com/livekit/mediatransportutil v0.0.0-20250519131108-fb90f5acfded/go.mod h1:mSNtYzSf6iY9xM3UX42VEI+STHvMgHmrYzEHPcdhB8A=
|
||||
github.com/livekit/protocol v1.39.4-0.20250813045730-236ffc7e5e12 h1:Ya6XP1yroaBk6SfYJa+wzkkUWHFZ00RrV0mam+wwAfI=
|
||||
github.com/livekit/protocol v1.39.4-0.20250813045730-236ffc7e5e12/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU=
|
||||
github.com/livekit/protocol v1.40.0 h1:FzCv17ivkxI4ySE0uFXSPPVPBVepf736Cgnmi054z2o=
|
||||
github.com/livekit/protocol v1.40.0/go.mod h1:YlgUxAegtU8jZ0tVXoIV/4fHeHqqLvS+6JnPKDbpFPU=
|
||||
github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741 h1:KKL1u94l6dF9u4cBwnnfozk27GH1txWy2SlvkfgmzoY=
|
||||
github.com/livekit/psrpc v0.6.1-0.20250726180611-3915e005e741/go.mod h1:AuDC5uOoEjQJEc69v4Li3t77Ocz0e0NdjQEuFfO+vfk=
|
||||
github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0=
|
||||
@@ -235,8 +235,8 @@ github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCy
|
||||
github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE=
|
||||
github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
|
||||
github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
|
||||
github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E=
|
||||
github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU=
|
||||
github.com/pion/dtls/v3 v3.0.7 h1:bItXtTYYhZwkPFk4t1n3Kkf5TDrfj6+4wG+CZR8uI9Q=
|
||||
github.com/pion/dtls/v3 v3.0.7/go.mod h1:uDlH5VPrgOQIw59irKYkMudSFprY9IEFCqz/eTz16f8=
|
||||
github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
|
||||
github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
|
||||
github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
|
||||
@@ -261,8 +261,8 @@ github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
|
||||
github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
|
||||
github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
|
||||
github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
|
||||
github.com/pion/turn/v4 v4.1.0 h1:+J56+aS8Bi6B4zij3ah6VvJpRuy8W8FtExR0OJPiTdM=
|
||||
github.com/pion/turn/v4 v4.1.0/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8=
|
||||
github.com/pion/turn/v4 v4.1.1 h1:9UnY2HB99tpDyz3cVVZguSxcqkJ1DsTSZ+8TGruh4fc=
|
||||
github.com/pion/turn/v4 v4.1.1/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8=
|
||||
github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c=
|
||||
github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
|
||||
+36
-29
@@ -64,24 +64,25 @@ type MediaTrack struct {
|
||||
}
|
||||
|
||||
type MediaTrackParams struct {
|
||||
ParticipantID func() livekit.ParticipantID
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
ParticipantVersion uint32
|
||||
ParticipantCountry string
|
||||
BufferFactory *buffer.Factory
|
||||
ReceiverConfig ReceiverConfig
|
||||
SubscriberConfig DirectionConfig
|
||||
PLIThrottleConfig sfu.PLIThrottleConfig
|
||||
AudioConfig sfu.AudioConfig
|
||||
VideoConfig config.VideoConfig
|
||||
Telemetry telemetry.TelemetryService
|
||||
Logger logger.Logger
|
||||
Reporter roomobs.TrackReporter
|
||||
SimTracks map[uint32]SimulcastTrackInfo
|
||||
OnRTCP func([]rtcp.Packet)
|
||||
ForwardStats *sfu.ForwardStats
|
||||
OnTrackEverSubscribed func(livekit.TrackID)
|
||||
ShouldRegressCodec func() bool
|
||||
ParticipantID func() livekit.ParticipantID
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
ParticipantVersion uint32
|
||||
ParticipantCountry string
|
||||
BufferFactory *buffer.Factory
|
||||
ReceiverConfig ReceiverConfig
|
||||
SubscriberConfig DirectionConfig
|
||||
PLIThrottleConfig sfu.PLIThrottleConfig
|
||||
AudioConfig sfu.AudioConfig
|
||||
VideoConfig config.VideoConfig
|
||||
Telemetry telemetry.TelemetryService
|
||||
Logger logger.Logger
|
||||
Reporter roomobs.TrackReporter
|
||||
SimTracks map[uint32]SimulcastTrackInfo
|
||||
OnRTCP func([]rtcp.Packet)
|
||||
ForwardStats *sfu.ForwardStats
|
||||
OnTrackEverSubscribed func(livekit.TrackID)
|
||||
ShouldRegressCodec func() bool
|
||||
PreferVideoSizeFromMedia bool
|
||||
}
|
||||
|
||||
func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack {
|
||||
@@ -96,17 +97,18 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack {
|
||||
}
|
||||
|
||||
t.MediaTrackReceiver = NewMediaTrackReceiver(MediaTrackReceiverParams{
|
||||
MediaTrack: t,
|
||||
IsRelayed: false,
|
||||
ParticipantID: params.ParticipantID,
|
||||
ParticipantIdentity: params.ParticipantIdentity,
|
||||
ParticipantVersion: params.ParticipantVersion,
|
||||
ReceiverConfig: params.ReceiverConfig,
|
||||
SubscriberConfig: params.SubscriberConfig,
|
||||
AudioConfig: params.AudioConfig,
|
||||
Telemetry: params.Telemetry,
|
||||
Logger: params.Logger,
|
||||
RegressionTargetCodec: t.regressionTargetCodec,
|
||||
MediaTrack: t,
|
||||
IsRelayed: false,
|
||||
ParticipantID: params.ParticipantID,
|
||||
ParticipantIdentity: params.ParticipantIdentity,
|
||||
ParticipantVersion: params.ParticipantVersion,
|
||||
ReceiverConfig: params.ReceiverConfig,
|
||||
SubscriberConfig: params.SubscriberConfig,
|
||||
AudioConfig: params.AudioConfig,
|
||||
Telemetry: params.Telemetry,
|
||||
Logger: params.Logger,
|
||||
RegressionTargetCodec: t.regressionTargetCodec,
|
||||
PreferVideoSizeFromMedia: params.PreferVideoSizeFromMedia,
|
||||
}, ti)
|
||||
|
||||
if ti.Type == livekit.TrackType_AUDIO {
|
||||
@@ -456,6 +458,11 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe
|
||||
newWR.AddOnCodecStateChange(func(codec webrtc.RTPCodecParameters, state sfu.ReceiverCodecState) {
|
||||
t.MediaTrackReceiver.HandleReceiverCodecChange(newWR, codec, state)
|
||||
})
|
||||
|
||||
// update subscriber video layers when video size changes
|
||||
newWR.OnVideoSizeChanged(func() {
|
||||
t.MediaTrackSubscriptions.UpdateVideoLayers()
|
||||
})
|
||||
}
|
||||
|
||||
if newCodec && t.enableRegression() {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/mime"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
func TestTrackInfo(t *testing.T) {
|
||||
@@ -52,7 +53,9 @@ func TestTrackInfo(t *testing.T) {
|
||||
|
||||
func TestGetQualityForDimension(t *testing.T) {
|
||||
t.Run("landscape source", func(t *testing.T) {
|
||||
mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
|
||||
mt := NewMediaTrack(MediaTrackParams{
|
||||
Logger: logger.GetLogger(),
|
||||
}, &livekit.TrackInfo{
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Width: 1080,
|
||||
Height: 720,
|
||||
@@ -66,7 +69,9 @@ func TestGetQualityForDimension(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("portrait source", func(t *testing.T) {
|
||||
mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
|
||||
mt := NewMediaTrack(MediaTrackParams{
|
||||
Logger: logger.GetLogger(),
|
||||
}, &livekit.TrackInfo{
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Width: 540,
|
||||
Height: 960,
|
||||
@@ -79,7 +84,9 @@ func TestGetQualityForDimension(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("layers provided", func(t *testing.T) {
|
||||
mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
|
||||
mt := NewMediaTrack(MediaTrackParams{
|
||||
Logger: logger.GetLogger(),
|
||||
}, &livekit.TrackInfo{
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Width: 1080,
|
||||
Height: 720,
|
||||
@@ -114,7 +121,9 @@ func TestGetQualityForDimension(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("highest layer with smallest dimensions", func(t *testing.T) {
|
||||
mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
|
||||
mt := NewMediaTrack(MediaTrackParams{
|
||||
Logger: logger.GetLogger(),
|
||||
}, &livekit.TrackInfo{
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Width: 1080,
|
||||
Height: 720,
|
||||
@@ -148,7 +157,9 @@ func TestGetQualityForDimension(t *testing.T) {
|
||||
require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700))
|
||||
require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1200, 800))
|
||||
|
||||
mt = NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
|
||||
mt = NewMediaTrack(MediaTrackParams{
|
||||
Logger: logger.GetLogger(),
|
||||
}, &livekit.TrackInfo{
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Width: 1080,
|
||||
Height: 720,
|
||||
|
||||
@@ -119,17 +119,18 @@ func (r *simulcastReceiver) IsRegressed() bool {
|
||||
// -----------------------------------------------------
|
||||
|
||||
type MediaTrackReceiverParams struct {
|
||||
MediaTrack types.MediaTrack
|
||||
IsRelayed bool
|
||||
ParticipantID func() livekit.ParticipantID
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
ParticipantVersion uint32
|
||||
ReceiverConfig ReceiverConfig
|
||||
SubscriberConfig DirectionConfig
|
||||
AudioConfig sfu.AudioConfig
|
||||
Telemetry telemetry.TelemetryService
|
||||
Logger logger.Logger
|
||||
RegressionTargetCodec mime.MimeType
|
||||
MediaTrack types.MediaTrack
|
||||
IsRelayed bool
|
||||
ParticipantID func() livekit.ParticipantID
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
ParticipantVersion uint32
|
||||
ReceiverConfig ReceiverConfig
|
||||
SubscriberConfig DirectionConfig
|
||||
AudioConfig sfu.AudioConfig
|
||||
Telemetry telemetry.TelemetryService
|
||||
Logger logger.Logger
|
||||
RegressionTargetCodec mime.MimeType
|
||||
PreferVideoSizeFromMedia bool
|
||||
}
|
||||
|
||||
type MediaTrackReceiver struct {
|
||||
@@ -924,7 +925,12 @@ func (t *MediaTrackReceiver) GetQualityForDimension(mimeType mime.MimeType, widt
|
||||
|
||||
trackInfo := t.TrackInfo()
|
||||
|
||||
if trackInfo.Height == 0 {
|
||||
var mediaSizes []buffer.VideoSize
|
||||
if receiver := t.Receiver(mimeType); receiver != nil {
|
||||
mediaSizes = receiver.VideoSizes()
|
||||
}
|
||||
|
||||
if trackInfo.Height == 0 && len(mediaSizes) == 0 {
|
||||
return quality
|
||||
}
|
||||
origSize := trackInfo.Height
|
||||
@@ -935,12 +941,36 @@ func (t *MediaTrackReceiver) GetQualityForDimension(mimeType mime.MimeType, widt
|
||||
requestedSize = width
|
||||
}
|
||||
|
||||
if origSize == 0 {
|
||||
for i := len(mediaSizes) - 1; i >= 0; i-- {
|
||||
if mediaSizes[i].Height > 0 {
|
||||
origSize = mediaSizes[i].Height
|
||||
if mediaSizes[i].Width < mediaSizes[i].Height {
|
||||
origSize = mediaSizes[i].Width
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// default sizes representing qualities low - high
|
||||
layerSizes := []uint32{180, 360, origSize}
|
||||
var providedSizes []uint32
|
||||
for _, layer := range buffer.GetVideoLayersForMimeType(mimeType, trackInfo) {
|
||||
providedSizes = append(providedSizes, layer.Height)
|
||||
}
|
||||
|
||||
if len(providedSizes) == 0 || providedSizes[0] == 0 || t.params.PreferVideoSizeFromMedia {
|
||||
if len(mediaSizes) > 0 {
|
||||
providedSizes = providedSizes[:0]
|
||||
for _, size := range mediaSizes {
|
||||
providedSizes = append(providedSizes, size.Height)
|
||||
}
|
||||
} else {
|
||||
t.params.Logger.Debugw("no video sizes provided by receiver, using track info sizes")
|
||||
}
|
||||
}
|
||||
|
||||
if len(providedSizes) > 0 {
|
||||
layerSizes = providedSizes
|
||||
// comparing height always
|
||||
|
||||
@@ -169,9 +169,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr *
|
||||
|
||||
if !sub.Hidden() {
|
||||
downTrack.OnBindAndConnected(func() {
|
||||
if err == nil {
|
||||
t.params.MediaTrack.OnTrackSubscribed()
|
||||
}
|
||||
t.params.MediaTrack.OnTrackSubscribed()
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -209,6 +209,7 @@ type ParticipantParams struct {
|
||||
DisableCodecRegression bool
|
||||
LastPubReliableSeq uint32
|
||||
Country string
|
||||
PreferVideoSizeFromMedia bool
|
||||
}
|
||||
|
||||
type ParticipantImpl struct {
|
||||
@@ -3291,6 +3292,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo)
|
||||
ShouldRegressCodec: func() bool {
|
||||
return p.helper().ShouldRegressCodec()
|
||||
},
|
||||
PreferVideoSizeFromMedia: p.params.PreferVideoSizeFromMedia,
|
||||
}, ti)
|
||||
|
||||
mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange)
|
||||
|
||||
@@ -471,6 +471,14 @@ func (d *DummyReceiver) CodecState() sfu.ReceiverCodecState {
|
||||
return sfu.ReceiverCodecStateNormal
|
||||
}
|
||||
|
||||
func (d *DummyReceiver) VideoSizes() []buffer.VideoSize {
|
||||
if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
|
||||
return r.VideoSizes()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// --------------------------------------------
|
||||
|
||||
type DummyRedReceiver struct {
|
||||
|
||||
@@ -72,6 +72,12 @@ type ExtPacket struct {
|
||||
IsOutOfOrder bool
|
||||
}
|
||||
|
||||
// VideoSize represents video resolution
|
||||
type VideoSize struct {
|
||||
Width uint32
|
||||
Height uint32
|
||||
}
|
||||
|
||||
// Buffer contains all packets
|
||||
type Buffer struct {
|
||||
sync.RWMutex
|
||||
@@ -125,6 +131,10 @@ type Buffer struct {
|
||||
onFpsChanged func()
|
||||
onFinalRtpStats func(*livekit.RTPStats)
|
||||
onCodecChange func(webrtc.RTPCodecParameters)
|
||||
onVideoSizeChanged func([]VideoSize)
|
||||
|
||||
// video size tracking for multiple spatial layers
|
||||
currentVideoSize [DefaultMaxLayerSpatial + 1]VideoSize
|
||||
|
||||
// logger
|
||||
logger logger.Logger
|
||||
@@ -886,6 +896,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
}
|
||||
|
||||
ep.Temporal = 0
|
||||
var videoSize []VideoSize
|
||||
if b.ddParser != nil {
|
||||
ddVal, videoLayer, err := b.ddParser.Parse(ep.Packet)
|
||||
if err != nil {
|
||||
@@ -901,6 +912,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
} else if ddVal != nil {
|
||||
ep.DependencyDescriptor = ddVal
|
||||
ep.VideoLayer = videoLayer
|
||||
videoSize = ExtractDependencyDescriptorVideoSize(ddVal.Descriptor)
|
||||
// DD-TODO : notify active decode target change if changed.
|
||||
}
|
||||
}
|
||||
@@ -915,6 +927,12 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
ep.KeyFrame = vp8Packet.IsKeyFrame
|
||||
if ep.DependencyDescriptor == nil {
|
||||
ep.Temporal = int32(vp8Packet.TID)
|
||||
|
||||
if ep.KeyFrame {
|
||||
if sz := ExtractVP8VideoSize(&vp8Packet, rtpPacket.Payload); sz.Width > 0 && sz.Height > 0 {
|
||||
videoSize = append(videoSize, sz)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// vp8 with DependencyDescriptor enabled, use the TID from the descriptor
|
||||
vp8Packet.TID = uint8(ep.Temporal)
|
||||
@@ -935,17 +953,36 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
Temporal: int32(vp9Packet.TID),
|
||||
}
|
||||
ep.Payload = vp9Packet
|
||||
ep.KeyFrame = IsVP9KeyFrame(&vp9Packet, rtpPacket.Payload)
|
||||
|
||||
if ep.KeyFrame {
|
||||
for i := 0; i < len(vp9Packet.Width); i++ {
|
||||
videoSize = append(videoSize, VideoSize{
|
||||
Width: uint32(vp9Packet.Width[i]),
|
||||
Height: uint32(vp9Packet.Height[i]),
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ep.KeyFrame = IsVP9KeyFrame(nil, rtpPacket.Payload)
|
||||
}
|
||||
ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload)
|
||||
|
||||
case mime.MimeTypeH264:
|
||||
ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload)
|
||||
ep.Spatial = InvalidLayerSpatial // h.264 don't have spatial scalability, reset to invalid
|
||||
|
||||
// Check H264 key frame video size
|
||||
if ep.KeyFrame {
|
||||
if sz := ExtractH264VideoSize(rtpPacket.Payload); sz.Width > 0 && sz.Height > 0 {
|
||||
videoSize = append(videoSize, sz)
|
||||
}
|
||||
}
|
||||
|
||||
case mime.MimeTypeAV1:
|
||||
ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload)
|
||||
|
||||
case mime.MimeTypeH265:
|
||||
ep.KeyFrame = IsH265KeyFrame(rtpPacket.Payload)
|
||||
if ep.DependencyDescriptor == nil {
|
||||
if len(rtpPacket.Payload) < 2 {
|
||||
b.logger.Warnw("invalid H265 packet", nil)
|
||||
@@ -955,8 +992,13 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
Temporal: int32(rtpPacket.Payload[1]&0x07) - 1,
|
||||
}
|
||||
ep.Spatial = InvalidLayerSpatial
|
||||
|
||||
if ep.KeyFrame {
|
||||
if sz := ExtractH265VideoSize(rtpPacket.Payload); sz.Width > 0 && sz.Height > 0 {
|
||||
videoSize = append(videoSize, sz)
|
||||
}
|
||||
}
|
||||
}
|
||||
ep.KeyFrame = IsH265KeyFrame(rtpPacket.Payload)
|
||||
}
|
||||
|
||||
if ep.KeyFrame {
|
||||
@@ -974,6 +1016,10 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowStat
|
||||
}
|
||||
}
|
||||
|
||||
if len(videoSize) > 0 {
|
||||
b.checkVideoSizeChange(videoSize)
|
||||
}
|
||||
|
||||
return ep
|
||||
}
|
||||
|
||||
@@ -1237,6 +1283,40 @@ func (b *Buffer) OnFpsChanged(f func()) {
|
||||
b.Unlock()
|
||||
}
|
||||
|
||||
func (b *Buffer) OnVideoSizeChanged(fn func([]VideoSize)) {
|
||||
b.Lock()
|
||||
b.onVideoSizeChanged = fn
|
||||
b.Unlock()
|
||||
}
|
||||
|
||||
// checkVideoSizeChange checks if video size has changed for a specific spatial layer and fires callback
|
||||
func (b *Buffer) checkVideoSizeChange(videoSizes []VideoSize) {
|
||||
if len(videoSizes) > len(b.currentVideoSize) {
|
||||
b.logger.Warnw("video size index out of range", nil, "newSize", videoSizes, "currentVideoSize", b.currentVideoSize)
|
||||
return
|
||||
}
|
||||
|
||||
if len(videoSizes) < len(b.currentVideoSize) {
|
||||
videoSizes = append(videoSizes, make([]VideoSize, len(b.currentVideoSize)-len(videoSizes))...)
|
||||
}
|
||||
|
||||
changed := false
|
||||
for i, sz := range videoSizes {
|
||||
if b.currentVideoSize[i].Width != sz.Width || b.currentVideoSize[i].Height != sz.Height {
|
||||
changed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if changed {
|
||||
b.logger.Debugw("video size changed", "from", b.currentVideoSize, "to", videoSizes)
|
||||
copy(b.currentVideoSize[:], videoSizes[:])
|
||||
if b.onVideoSizeChanged != nil {
|
||||
go b.onVideoSizeChanged(videoSizes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) GetTemporalLayerFpsForSpatial(layer int32) []float32 {
|
||||
if int(layer) >= len(b.frameRateCalculator) {
|
||||
return nil
|
||||
|
||||
@@ -217,3 +217,16 @@ func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDe
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []VideoSize {
|
||||
if dd.AttachedStructure == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
videoSizes := make([]VideoSize, 0, len(dd.AttachedStructure.Resolutions))
|
||||
for _, res := range dd.AttachedStructure.Resolutions {
|
||||
videoSizes = append(videoSizes, VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)})
|
||||
}
|
||||
|
||||
return videoSizes
|
||||
}
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SPSInfo holds parsed SPS parameters
|
||||
type SPSInfo struct {
|
||||
ChromaFormatIDC uint
|
||||
PicWidthInLumaSamples uint
|
||||
PicHeightInLumaSamples uint
|
||||
ConformanceWindowFlag bool
|
||||
ConfWinLeftOffset uint
|
||||
ConfWinRightOffset uint
|
||||
ConfWinTopOffset uint
|
||||
ConfWinBottomOffset uint
|
||||
CodedWidth, CodedHeight uint // Raw coded resolution
|
||||
DisplayWidth, DisplayHeight uint // Resolution after conformance window cropping
|
||||
}
|
||||
|
||||
// -------- BitReader --------
|
||||
type BitReader struct {
|
||||
data []byte
|
||||
pos int // bit position
|
||||
}
|
||||
|
||||
func NewBitReader(data []byte) *BitReader {
|
||||
return &BitReader{data: data}
|
||||
}
|
||||
|
||||
func (br *BitReader) left() int {
|
||||
return len(br.data)*8 - br.pos
|
||||
}
|
||||
|
||||
func (br *BitReader) ReadBits(n int) (uint, error) {
|
||||
if n < 0 || br.left() < n {
|
||||
return 0, errors.New("not enough bits")
|
||||
}
|
||||
var v uint
|
||||
for i := 0; i < n; i++ {
|
||||
bytePos := br.pos / 8
|
||||
bitPos := 7 - (br.pos % 8)
|
||||
bit := (br.data[bytePos] >> bitPos) & 1
|
||||
v = (v << 1) | uint(bit)
|
||||
br.pos++
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (br *BitReader) ReadFlag() (bool, error) {
|
||||
b, err := br.ReadBits(1)
|
||||
return b == 1, err
|
||||
}
|
||||
|
||||
func (br *BitReader) ReadUE() (uint, error) {
|
||||
// Unsigned Exp-Golomb
|
||||
zeros := 0
|
||||
for {
|
||||
bit, err := br.ReadBits(1)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == 0 {
|
||||
zeros++
|
||||
continue
|
||||
}
|
||||
break // hit the stop bit '1'
|
||||
}
|
||||
if zeros == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
info, err := br.ReadBits(zeros)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return (1<<zeros - 1) + info, nil
|
||||
}
|
||||
|
||||
func (br *BitReader) ReadSE() (int, error) {
|
||||
ueVal, err := br.ReadUE()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
k := int(ueVal)
|
||||
var val int
|
||||
if k%2 == 0 {
|
||||
val = -int(k / 2)
|
||||
} else {
|
||||
val = (k + 1) / 2
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// ------------------------- H265 -------------------------
|
||||
|
||||
// stripStartCode removes 00 00 01 or 00 00 00 01 if present.
|
||||
func stripStartCode(b []byte) []byte {
|
||||
if len(b) >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0x00 && b[3] == 0x01 {
|
||||
return b[4:]
|
||||
}
|
||||
if len(b) >= 3 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0x01 {
|
||||
return b[3:]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// removeEmulationPreventionBytes removes 0x03 after 0x0000
|
||||
func removeEmulationPreventionBytes(data []byte) []byte {
|
||||
out := make([]byte, 0, len(data))
|
||||
for i := 0; i < len(data); i++ {
|
||||
if i > 1 && data[i] == 0x03 && data[i-1] == 0x00 && data[i-2] == 0x00 {
|
||||
continue
|
||||
}
|
||||
out = append(out, data[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseH265SPS parses a full H.265 SPS NAL unit
|
||||
func parseH265SPS(nal []byte) (*SPSInfo, error) {
|
||||
// Optional start code
|
||||
nal = stripStartCode(nal)
|
||||
|
||||
// Remove emulation prevention bytes across the NAL
|
||||
rbsp := removeEmulationPreventionBytes(nal)
|
||||
|
||||
br := NewBitReader(rbsp)
|
||||
|
||||
// ---- NAL header (16 bits): forbidden_zero_bit(1), nal_unit_type(6), nuh_layer_id(6), nuh_temporal_id_plus1(3)
|
||||
if _, err := br.ReadBits(1); err != nil { // forbidden_zero_bit
|
||||
return nil, err
|
||||
}
|
||||
nalUnitType, err := br.ReadBits(6)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(6); err != nil { // nuh_layer_id
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(3); err != nil { // nuh_temporal_id_plus1
|
||||
return nil, err
|
||||
}
|
||||
// 33 = SPS
|
||||
if nalUnitType != 33 {
|
||||
return nil, fmt.Errorf("not an HEVC SPS NAL (type=%d)", nalUnitType)
|
||||
}
|
||||
|
||||
// ---- sps_video_parameter_set_id u(4), sps_max_sub_layers_minus1 u(3), sps_temporal_id_nesting_flag u(1)
|
||||
if _, err = br.ReadBits(4); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxSubLayersMinus1, err := br.ReadBits(3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// ---- profile_tier_level(1, max_sub_layers_minus1)
|
||||
// general_profile_space u(2), general_tier_flag u(1), general_profile_idc u(5)
|
||||
if _, err = br.ReadBits(2 + 1 + 5); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// general_profile_compatibility_flags u(32)
|
||||
if _, err = br.ReadBits(32); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// general_constraint_indicator_flags u(48)
|
||||
if _, err = br.ReadBits(16); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(32); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// general_level_idc u(8)
|
||||
if _, err = br.ReadBits(8); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
subLayerProfilePresentFlag := make([]bool, maxSubLayersMinus1)
|
||||
subLayerLevelPresentFlag := make([]bool, maxSubLayersMinus1)
|
||||
for i := uint(0); i < maxSubLayersMinus1; i++ {
|
||||
f1, err := br.ReadFlag()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f2, err := br.ReadFlag()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
subLayerProfilePresentFlag[i] = f1
|
||||
subLayerLevelPresentFlag[i] = f2
|
||||
}
|
||||
if maxSubLayersMinus1 > 0 {
|
||||
// reserved_zero_2bits for i = maxSubLayersMinus1 .. 7
|
||||
for i := maxSubLayersMinus1; i < 8; i++ {
|
||||
if _, err := br.ReadBits(2); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := uint(0); i < maxSubLayersMinus1; i++ {
|
||||
if subLayerProfilePresentFlag[i] {
|
||||
if _, err = br.ReadBits(2 + 1 + 5); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(32); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = br.ReadBits(48); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if subLayerLevelPresentFlag[i] {
|
||||
if _, err = br.ReadBits(8); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Now the core SPS fields we need
|
||||
_, err = br.ReadUE() // sps_seq_parameter_set_id
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chromaFormatIDC, err := br.ReadUE()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chromaFormatIDC == 3 {
|
||||
// separate_colour_plane_flag u(1)
|
||||
if _, err := br.ReadFlag(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
picW, err := br.ReadUE() // pic_width_in_luma_samples
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
picH, err := br.ReadUE() // pic_height_in_luma_samples
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
confFlag, err := br.ReadFlag()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var l, r, t, b uint
|
||||
if confFlag {
|
||||
if l, err = br.ReadUE(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, err = br.ReadUE(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if t, err = br.ReadUE(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b, err = br.ReadUE(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// crop unit size depends on chroma_format_idc
|
||||
subWidthC, subHeightC := getSubWidthC(chromaFormatIDC), getSubHeightC(chromaFormatIDC)
|
||||
|
||||
info := &SPSInfo{
|
||||
ChromaFormatIDC: chromaFormatIDC,
|
||||
PicWidthInLumaSamples: picW,
|
||||
PicHeightInLumaSamples: picH,
|
||||
ConformanceWindowFlag: confFlag,
|
||||
ConfWinLeftOffset: l,
|
||||
ConfWinRightOffset: r,
|
||||
ConfWinTopOffset: t,
|
||||
ConfWinBottomOffset: b,
|
||||
CodedWidth: picW,
|
||||
CodedHeight: picH,
|
||||
}
|
||||
|
||||
if confFlag {
|
||||
w := int(picW) - int(l+r)*int(subWidthC)
|
||||
h := int(picH) - int(t+b)*int(subHeightC)
|
||||
if w < 0 {
|
||||
w = 0
|
||||
}
|
||||
if h < 0 {
|
||||
h = 0
|
||||
}
|
||||
info.DisplayWidth = uint(w)
|
||||
info.DisplayHeight = uint(h)
|
||||
} else {
|
||||
info.DisplayWidth = picW
|
||||
info.DisplayHeight = picH
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func getSubWidthC(chromaFormatIDC uint) uint {
|
||||
if chromaFormatIDC == 1 || chromaFormatIDC == 2 {
|
||||
return 2
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func getSubHeightC(chromaFormatIDC uint) uint {
|
||||
if chromaFormatIDC == 1 {
|
||||
return 2
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func ExtractH265VideoSize(payload []byte) VideoSize {
|
||||
if len(payload) < 2 {
|
||||
return VideoSize{}
|
||||
}
|
||||
nalType := (payload[0] >> 1) & 0x3F
|
||||
|
||||
var spsNalu []byte
|
||||
switch nalType {
|
||||
case 33: // SPS
|
||||
spsNalu = payload
|
||||
case 48: // Aggregation Packet (AP)
|
||||
// skip 2-byte header
|
||||
i := 2
|
||||
for i+2 <= len(payload) {
|
||||
nalSize := int(payload[i])<<8 | int(payload[i+1])
|
||||
i += 2
|
||||
if i+nalSize > len(payload) {
|
||||
break
|
||||
}
|
||||
nalUnit := payload[i : i+nalSize]
|
||||
nt := (nalUnit[0] >> 1) & 0x3F
|
||||
if nt == 33 {
|
||||
spsNalu = nalUnit
|
||||
break
|
||||
}
|
||||
i += nalSize
|
||||
}
|
||||
}
|
||||
|
||||
if len(spsNalu) > 0 {
|
||||
info, err := parseH265SPS(spsNalu)
|
||||
if err != nil {
|
||||
return VideoSize{}
|
||||
}
|
||||
return VideoSize{Width: uint32(info.DisplayWidth), Height: uint32(info.DisplayHeight)}
|
||||
}
|
||||
|
||||
return VideoSize{}
|
||||
}
|
||||
|
||||
// ------------------------- H264 -------------------------
|
||||
|
||||
// parseH264SPS parses a full H.264 SPS NAL unit into SPSInfo
|
||||
func parseH264SPS(nal []byte) (*SPSInfo, error) {
|
||||
if len(nal) < 1 {
|
||||
return nil, errors.New("empty SPS NAL")
|
||||
}
|
||||
nal = stripStartCode(nal)
|
||||
nalType := nal[0] & 0x1F
|
||||
if nalType != 7 {
|
||||
return nil, fmt.Errorf("not an SPS NAL (type=%d)", nalType)
|
||||
}
|
||||
|
||||
rbsp := removeEmulationPreventionBytes(nal[1:]) // skip NAL header
|
||||
br := NewBitReader(rbsp)
|
||||
|
||||
profileIDC, _ := br.ReadBits(8)
|
||||
_, _ = br.ReadBits(8) // constraint flags
|
||||
_, _ = br.ReadBits(8) // level_idc
|
||||
_, _ = br.ReadUE() // seq_parameter_set_id
|
||||
|
||||
chromaFormatIDC := uint(1)
|
||||
if profileIDC == 100 || profileIDC == 110 || profileIDC == 122 || profileIDC == 244 ||
|
||||
profileIDC == 44 || profileIDC == 83 || profileIDC == 86 || profileIDC == 118 || profileIDC == 128 {
|
||||
chromaFormatIDC, _ = br.ReadUE()
|
||||
if chromaFormatIDC == 3 {
|
||||
br.ReadFlag() // separate_colour_plane_flag
|
||||
}
|
||||
br.ReadUE() // bit_depth_luma_minus8
|
||||
br.ReadUE() // bit_depth_chroma_minus8
|
||||
br.ReadFlag() // qpprime_y_zero_transform_bypass_flag
|
||||
if v, _ := br.ReadFlag(); v { // seq_scaling_matrix_present_flag
|
||||
for i := 0; i < 8; i++ {
|
||||
br.ReadFlag()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
br.ReadUE() // log2_max_frame_num_minus4
|
||||
pocType, _ := br.ReadUE()
|
||||
if pocType == 0 {
|
||||
br.ReadUE()
|
||||
} else if pocType == 1 {
|
||||
br.ReadFlag()
|
||||
br.ReadSE()
|
||||
br.ReadSE()
|
||||
cnt, _ := br.ReadUE()
|
||||
for i := uint(0); i < cnt; i++ {
|
||||
br.ReadSE()
|
||||
}
|
||||
}
|
||||
|
||||
br.ReadUE() // max_num_ref_frames
|
||||
br.ReadFlag() // gaps_in_frame_num_value_allowed_flag
|
||||
|
||||
wMbs, _ := br.ReadUE()
|
||||
hMapUnits, _ := br.ReadUE()
|
||||
frameMbsOnly, _ := br.ReadFlag()
|
||||
if !frameMbsOnly {
|
||||
br.ReadFlag() // mb_adaptive_frame_field_flag
|
||||
}
|
||||
br.ReadFlag() // direct_8x8_inference_flag
|
||||
|
||||
var cropLeft, cropRight, cropTop, cropBottom uint
|
||||
if frameCropping, _ := br.ReadFlag(); frameCropping {
|
||||
cropLeft, _ = br.ReadUE()
|
||||
cropRight, _ = br.ReadUE()
|
||||
cropTop, _ = br.ReadUE()
|
||||
cropBottom, _ = br.ReadUE()
|
||||
}
|
||||
|
||||
width := (wMbs + 1) * 16
|
||||
height := (hMapUnits + 1) * 16
|
||||
if !frameMbsOnly {
|
||||
height *= 2
|
||||
}
|
||||
|
||||
subWidthC := getSubWidthC(chromaFormatIDC)
|
||||
subHeightC := getSubHeightC(chromaFormatIDC)
|
||||
cropUnitX := subWidthC
|
||||
cropUnitY := subHeightC
|
||||
if chromaFormatIDC == 0 {
|
||||
cropUnitX = 1
|
||||
if !frameMbsOnly {
|
||||
cropUnitY = 2
|
||||
} else {
|
||||
cropUnitY = 1
|
||||
}
|
||||
} else if !frameMbsOnly {
|
||||
cropUnitY *= 2
|
||||
}
|
||||
|
||||
info := &SPSInfo{
|
||||
ChromaFormatIDC: chromaFormatIDC,
|
||||
PicWidthInLumaSamples: width,
|
||||
PicHeightInLumaSamples: height,
|
||||
ConformanceWindowFlag: cropLeft+cropRight+cropTop+cropBottom > 0,
|
||||
ConfWinLeftOffset: cropLeft,
|
||||
ConfWinRightOffset: cropRight,
|
||||
ConfWinTopOffset: cropTop,
|
||||
ConfWinBottomOffset: cropBottom,
|
||||
CodedWidth: width,
|
||||
CodedHeight: height,
|
||||
DisplayWidth: width - (cropLeft+cropRight)*cropUnitX,
|
||||
DisplayHeight: height - (cropTop+cropBottom)*cropUnitY,
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// ExtractH264VideoSize extracts resolution from H.264 RTP payload
|
||||
func ExtractH264VideoSize(payload []byte) VideoSize {
|
||||
if len(payload) < 1 {
|
||||
return VideoSize{}
|
||||
}
|
||||
|
||||
parseNAL := func(nal []byte) VideoSize {
|
||||
info, err := parseH264SPS(nal)
|
||||
if err != nil {
|
||||
return VideoSize{}
|
||||
}
|
||||
return VideoSize{Width: uint32(info.DisplayWidth), Height: uint32(info.DisplayHeight)}
|
||||
}
|
||||
|
||||
nalType := payload[0] & 0x1F
|
||||
|
||||
switch nalType {
|
||||
case 7: // SPS NAL
|
||||
return parseNAL(payload)
|
||||
|
||||
case 28: // FU-A
|
||||
if len(payload) < 2 {
|
||||
return VideoSize{}
|
||||
}
|
||||
start := (payload[1] & 0x80) != 0
|
||||
if !start {
|
||||
return VideoSize{}
|
||||
}
|
||||
nalHeader := (payload[0] & 0xE0) | (payload[1] & 0x1F)
|
||||
sps := append([]byte{nalHeader}, payload[2:]...)
|
||||
return parseNAL(sps)
|
||||
|
||||
case 24, 25, 26, 27: // STAP-A/B, MTAP16, MTAP24
|
||||
offset := 1
|
||||
if nalType == 25 { // STAP-B has 16-bit DON
|
||||
offset += 2
|
||||
} else if nalType == 26 { // MTAP16
|
||||
offset += 3
|
||||
} else if nalType == 27 { // MTAP24
|
||||
offset += 4
|
||||
}
|
||||
|
||||
for offset+2 <= len(payload) {
|
||||
naluSize := int(payload[offset])<<8 | int(payload[offset+1])
|
||||
offset += 2
|
||||
if offset+naluSize > len(payload) {
|
||||
break
|
||||
}
|
||||
nalu := payload[offset : offset+naluSize]
|
||||
if nalu[0]&0x1F == 7 { // SPS
|
||||
return parseNAL(nalu)
|
||||
}
|
||||
offset += naluSize
|
||||
}
|
||||
return VideoSize{}
|
||||
|
||||
default:
|
||||
return VideoSize{}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExtractH26xVideoSize(t *testing.T) {
|
||||
type testcase struct {
|
||||
payload string
|
||||
width uint32
|
||||
height uint32
|
||||
isH264 bool
|
||||
}
|
||||
|
||||
testcases := []testcase{
|
||||
{"eAAOZ0LAH4xoBQBboB4RCNQABGjOPIA=", 1280, 720, true},
|
||||
{"eAAPZ0LAFoxoCgL3lgHhEI1AAARozjyA", 640, 360, true},
|
||||
{"eAAOZ0LADIxoFBl54B4RCNQABGjOPIA=", 320, 180, true},
|
||||
{"YAEAGkABDAP//wFgAAADALAAAAMAAAMAXQAAGwJAAC9CAQMBYAAAAwCwAAADAAADAF0AAKACgIAtFiBu5FIy5+E9C+ob1SmoCAgIH8IBBAAHRAHAcvBbJA==", 1280, 720, false},
|
||||
{"YAEAGkABDAP//wFgAAADALAAAAMAAAMAPwAAGwJAADBCAQMBYAAAAwCwAAADAAADAD8AAKAFAgFx8uIG7kUjLn4T0L6hvVKagICAgfwgEEAAB0QBwHLwWyQ=", 640, 360, false},
|
||||
{"QgEDAWAAAAMAsAAAAwAAAwA8AACgCggMHz4gM7kUhi5+E9C+ob1Q/qoI9VQT6qoK9VVBfqqqDPVVVKagICAgfwgEEA==", 320, 180, false},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
payload, err := base64.StdEncoding.DecodeString(tc.payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
var sz VideoSize
|
||||
if tc.isH264 {
|
||||
sz = ExtractH264VideoSize(payload)
|
||||
} else {
|
||||
sz = ExtractH265VideoSize(payload)
|
||||
}
|
||||
require.Equal(t, tc.width, sz.Width)
|
||||
require.Equal(t, tc.height, sz.Height)
|
||||
}
|
||||
}
|
||||
@@ -318,12 +318,15 @@ func IsH264KeyFrame(payload []byte) bool {
|
||||
// IsVP9KeyFrame detects if vp9 payload is a keyframe
|
||||
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
|
||||
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
|
||||
func IsVP9KeyFrame(payload []byte) bool {
|
||||
var vp9 codecs.VP9Packet
|
||||
_, err := vp9.Unmarshal(payload)
|
||||
if err != nil || len(vp9.Payload) < 1 {
|
||||
return false
|
||||
func IsVP9KeyFrame(vp9 *codecs.VP9Packet, payload []byte) bool {
|
||||
if vp9 == nil {
|
||||
vp9 = &codecs.VP9Packet{}
|
||||
_, err := vp9.Unmarshal(payload)
|
||||
if err != nil || len(vp9.Payload) < 1 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if !vp9.B {
|
||||
return false
|
||||
}
|
||||
@@ -452,4 +455,22 @@ func IsH265KeyFrame(payload []byte) (kf bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
// ExtractVP8VideoSize extracts video resolution from VP8 key frame
|
||||
func ExtractVP8VideoSize(vp8Packet *VP8, payload []byte) VideoSize {
|
||||
if !vp8Packet.IsKeyFrame || len(payload) < vp8Packet.HeaderSize+10 {
|
||||
return VideoSize{}
|
||||
}
|
||||
|
||||
vp8Payload := payload[vp8Packet.HeaderSize:]
|
||||
|
||||
// Check for VP8 start code
|
||||
if len(vp8Payload) < 10 || vp8Payload[3] != 0x9D || vp8Payload[4] != 0x01 || vp8Payload[5] != 0x2A {
|
||||
return VideoSize{}
|
||||
}
|
||||
|
||||
// Read width and height from bytes 6-9
|
||||
width := uint32(vp8Payload[6]) | (uint32(vp8Payload[7]) << 8)
|
||||
height := uint32(vp8Payload[8]) | (uint32(vp8Payload[9]) << 8)
|
||||
|
||||
return VideoSize{width & 0x3FFF, height & 0x3FFF}
|
||||
}
|
||||
|
||||
+45
-4
@@ -141,6 +141,9 @@ type TrackReceiver interface {
|
||||
|
||||
AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState))
|
||||
CodecState() ReceiverCodecState
|
||||
|
||||
// VideoSizes returns the video size parsed from rtp packet for each spatial layer.
|
||||
VideoSizes() []buffer.VideoSize
|
||||
}
|
||||
|
||||
type REDTransformer interface {
|
||||
@@ -181,10 +184,13 @@ type WebRTCReceiver struct {
|
||||
|
||||
onRTCP func([]rtcp.Packet)
|
||||
|
||||
bufferMu sync.RWMutex
|
||||
buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
|
||||
upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
|
||||
rtt uint32
|
||||
bufferMu sync.RWMutex
|
||||
buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
|
||||
upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
|
||||
videoSizeMu sync.RWMutex
|
||||
videoSizes [buffer.DefaultMaxLayerSpatial + 1]buffer.VideoSize
|
||||
onVideoSizeChanged func()
|
||||
rtt uint32
|
||||
|
||||
lbThreshold int
|
||||
|
||||
@@ -415,6 +421,21 @@ func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) erro
|
||||
rt.(REDTransformer).ForwardRTCPSenderReport(w.codec.PayloadType, layer, srData)
|
||||
}
|
||||
})
|
||||
buff.OnVideoSizeChanged(func(videoSize []buffer.VideoSize) {
|
||||
w.videoSizeMu.Lock()
|
||||
if w.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
|
||||
copy(w.videoSizes[:], videoSize)
|
||||
} else {
|
||||
w.videoSizes[layer] = videoSize[0]
|
||||
}
|
||||
w.logger.Debugw("video size changed", "size", w.videoSizes)
|
||||
cb := w.onVideoSizeChanged
|
||||
w.videoSizeMu.Unlock()
|
||||
|
||||
if cb != nil {
|
||||
cb()
|
||||
}
|
||||
})
|
||||
|
||||
if w.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 {
|
||||
buff.OnCodecChange(w.handleCodecChange)
|
||||
@@ -974,6 +995,26 @@ func (w *WebRTCReceiver) SetCodecState(state ReceiverCodecState) {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *WebRTCReceiver) VideoSizes() []buffer.VideoSize {
|
||||
var sizes []buffer.VideoSize
|
||||
w.videoSizeMu.RLock()
|
||||
defer w.videoSizeMu.RUnlock()
|
||||
for _, v := range w.videoSizes {
|
||||
if v.Width == 0 || v.Height == 0 {
|
||||
break
|
||||
}
|
||||
sizes = append(sizes, v)
|
||||
}
|
||||
|
||||
return sizes
|
||||
}
|
||||
|
||||
func (w *WebRTCReceiver) OnVideoSizeChanged(f func()) {
|
||||
w.videoSizeMu.Lock()
|
||||
w.onVideoSizeChanged = f
|
||||
w.videoSizeMu.Unlock()
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------
|
||||
|
||||
// closes all track senders in parallel, returns when all are closed
|
||||
|
||||
Reference in New Issue
Block a user