diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 34a13ab6c..a012f3f68 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -17,9 +17,9 @@ name: Test on: workflow_dispatch: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: test: @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version-file: "go.mod" - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@v2.4.1 diff --git a/CHANGELOG b/CHANGELOG index d8408a4c5..362fb85ce 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,46 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.6.0] - 2024-04-10 + +### Added +- Support for Participant.Kind. (#2505 #2626) +- Support XR request/response for rtt calculation (#2536) +- Added support for departureTimeout to keep the room open after participant depart (#2549) +- Added support for Egress Proxy (#2570) +- Added support for SIP DTMF data messages. (#2559) +- Add option to enable bitrate based scoring (#2600) +- Agent service: support for orchestration v2 & namespaces (#2545 #2641) +- Ability to disable audio loss proxying. (#2629) + +### Fixed +- Prevent multiple debounce of quality downgrade. (#2499) +- fix pli throttle locking (#2521) +- Use the correct snapshot id for PPS. (#2528) +- Validate SIP trunks and rules when creating new ones. (#2535) +- Remove subscriber if track closed while adding subscriber. (#2537) +- fix #2539, do not kill the keepaliveWorker task when the ping timeout occurs (#2555) +- Improved A/V sync, proper RTCP report past mute. (#2588) +- Protect duplicate subscription. (#2596) +- Fix twcc has chance to miss for firefox simulcast rtx (#2601) +- Limit playout delay change for high jitter (#2635) + +### Changed +- Replace reflect.Equal with generic sliceEqual (#2494) +- Some optimisations in the forwarding path. (#2035) +- Reduce heap for dependency descriptor in forwarding path. (#2496) +- Separate buffer size config for video and audio. (#2498) +- update pion/ice for tcpmux memory improvement (#2500) +- Close published track always. (#2508) +- use dynamic bucket size (#2524) +- Refactoring channel handling (#2532) +- Forward publisher sender report instead of generating. (#2572) +- Notify initial permissions (#2595) +- Replace sleep with sync.Cond to reduce jitter (#2603) +- Prevent large spikes in propagation delay (#2615) +- reduce gc from stream allocator rate monitor (#2638) + + ## [1.5.3] - 2024-02-17 ### Added diff --git a/cmd/server/main.go b/cmd/server/main.go index 3e8e3344d..427702954 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -272,7 +272,9 @@ func startServer(c *cli.Context) error { return err } - prometheus.Init(currentNode.Id, currentNode.Type, conf.Environment) + if err := prometheus.Init(currentNode.Id, currentNode.Type); err != nil { + return err + } server, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/go.mod b/go.mod index b15f7bf03..ed85a6780 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/livekit/livekit-server -go 1.21 +go 1.22 require ( github.com/avast/retry-go/v4 v4.5.1 @@ -16,26 +16,27 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/jellydator/ttlcache/v3 v3.2.0 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e - github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 + github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e + github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b + github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.14 - github.com/pion/interceptor v0.1.25 + github.com/pion/ice/v2 v2.3.19 + github.com/pion/interceptor v0.1.29 github.com/pion/rtcp v1.2.14 - github.com/pion/rtp v1.8.3 - github.com/pion/sctp v1.8.12 - github.com/pion/sdp/v3 v3.0.8 + github.com/pion/rtp v1.8.6 + github.com/pion/sctp v1.8.16 + github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 - github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.29 + github.com/pion/turn/v2 v2.1.6 + github.com/pion/webrtc/v3 v3.2.38 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 @@ -48,15 +49,16 @@ require ( github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 - golang.org/x/sync v0.6.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/sync v0.7.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -65,7 +67,6 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -73,13 +74,13 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.33.1 // indirect + github.com/nats-io/nats.go v1.34.1 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -97,13 +98,14 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sys v0.18.0 // indirect + go.uber.org/zap/exp v0.2.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.19.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/grpc v1.62.1 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/grpc v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 670d00671..28895fb95 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -8,8 +10,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= @@ -38,8 +40,6 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frostbyte73/core v0.0.10 h1:D4DQXdPb8ICayz0n75rs4UYTXrUSdxzUfeleuNJORsU= github.com/frostbyte73/core v0.0.10/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= @@ -50,20 +50,7 @@ github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -96,7 +83,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE= +github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= @@ -113,8 +101,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -130,12 +118,12 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= -github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e h1:XR7vPLN7c/R6R87UARoBW2csVKd7RuTXwG+XsjczbT0= -github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e/go.mod h1:G7Pa985GhZv2MCC3UnUocBhZfi3DsWA6WmlSkkpQYTM= -github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= -github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b h1:hPgkp/LJzhx+U2CHOc68yxGIyfFspagsyupAaqx1Ulw= +github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= +github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -165,23 +153,14 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= -github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= @@ -189,29 +168,27 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/ice/v2 v2.3.14 h1:A7UaEmalw12Fko8YO0qguUbWyE69BnN4mDEqT7cLWQI= -github.com/pion/ice/v2 v2.3.14/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= -github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= +github.com/pion/ice/v2 v2.3.19 h1:1GoMRTMnB6bCP4aGy2MjxK3w4laDkk+m7svJb/eqybc= +github.com/pion/ice/v2 v2.3.19/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= +github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw= +github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY= -github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= -github.com/pion/sdp/v3 v3.0.8 h1:yd/wkrS0nzXEAb+uwv1TL3SG/gzsTiXHVOtXtD7EKl0= -github.com/pion/sdp/v3 v3.0.8/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY= +github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -223,13 +200,14 @@ github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++ github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= +github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= -github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.29 h1:flXjxjlqpp3FjkpSSBKwv7UOfbUvan9+gFY6A5ZaAn4= -github.com/pion/webrtc/v3 v3.2.29/go.mod h1:M+5YSvBDPAkHHRwGXlplIFBQI5mXm6Y4byns1OpiX68= +github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= +github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.38 h1:oA52VJAJhOjSi1JpKjf0CM+cCiZ3b7jBxvsoOiajeDU= +github.com/pion/webrtc/v3 v3.2.38/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -253,7 +231,6 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -262,7 +239,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -283,7 +259,6 @@ github.com/urfave/negroni/v3 v3.1.0 h1:lzmuxGSpnJCT/ujgIAjkU3+LW3NX8alCglO/L6KjI github.com/urfave/negroni/v3 v3.1.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -297,8 +272,9 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= +go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= @@ -307,33 +283,28 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -346,36 +317,29 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -400,8 +364,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -427,29 +391,19 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -458,12 +412,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/agent/client.go b/pkg/agent/client.go new file mode 100644 index 000000000..94eeed949 --- /dev/null +++ b/pkg/agent/client.go @@ -0,0 +1,185 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "sync" + "time" + + "github.com/gammazero/workerpool" + "google.golang.org/protobuf/types/known/emptypb" + + serverutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils" + "github.com/livekit/psrpc" +) + +const ( + EnabledCacheTTL = 1 * time.Minute + RoomAgentTopic = "room" + PublisherAgentTopic = "publisher" + DefaultHandlerNamespace = "" + + CheckEnabledTimeout = 5 * time.Second +) + +type Client interface { + // LaunchJob starts a room or participant job on an agent. + // it will launch a job once for each worker in each namespace + LaunchJob(ctx context.Context, desc *JobDescription) + Stop() error +} + +type JobDescription struct { + JobType livekit.JobType + Room *livekit.Room + // only set for participant jobs + Participant *livekit.ParticipantInfo +} + +type agentClient struct { + client rpc.AgentInternalClient + + mu sync.RWMutex + + // cache response to avoid constantly checking with controllers + // cache is invalidated with AgentRegistered updates + roomNamespaces *serverutils.IncrementalDispatcher[string] + publisherNamespaces *serverutils.IncrementalDispatcher[string] + enabledExpiresAt time.Time + + workers *workerpool.WorkerPool + + invalidateSub psrpc.Subscription[*emptypb.Empty] + subDone chan struct{} +} + +func NewAgentClient(bus psrpc.MessageBus) (Client, error) { + client, err := rpc.NewAgentInternalClient(bus) + if err != nil { + return nil, err + } + + c := &agentClient{ + client: client, + workers: workerpool.New(50), + subDone: make(chan struct{}), + } + + sub, err := c.client.SubscribeWorkerRegistered(context.Background(), DefaultHandlerNamespace) + if err != nil { + return nil, err + } + + c.invalidateSub = sub + + go func() { + // invalidate cache + for range sub.Channel() { + c.mu.Lock() + c.roomNamespaces = nil + c.publisherNamespaces = nil + c.mu.Unlock() + } + + c.subDone <- struct{}{} + }() + + return c, nil +} + +func (c *agentClient) LaunchJob(ctx context.Context, desc *JobDescription) { + roomNamespaces, publisherNamespaces, needsRefresh := c.getOrCreateDispatchers() + + if needsRefresh { + go c.checkEnabled(ctx, roomNamespaces, publisherNamespaces) + } + + target := roomNamespaces + jobTypeTopic := RoomAgentTopic + if desc.JobType == livekit.JobType_JT_PUBLISHER { + target = publisherNamespaces + jobTypeTopic = PublisherAgentTopic + } + + target.ForEach(func(ns string) { + c.workers.Submit(func() { + _, err := c.client.JobRequest(ctx, ns, jobTypeTopic, &livekit.Job{ + Id: utils.NewGuid(utils.AgentJobPrefix), + Type: desc.JobType, + Room: desc.Room, + Participant: desc.Participant, + Namespace: ns, + }) + if err != nil { + logger.Errorw("failed to send job request", err, "namespace", ns, "jobType", jobTypeTopic) + } + }) + }) +} + +func (c *agentClient) getOrCreateDispatchers() (*serverutils.IncrementalDispatcher[string], *serverutils.IncrementalDispatcher[string], bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if time.Since(c.enabledExpiresAt) > EnabledCacheTTL || c.roomNamespaces == nil || c.publisherNamespaces == nil { + c.roomNamespaces = serverutils.NewIncrementalDispatcher[string]() + c.publisherNamespaces = serverutils.NewIncrementalDispatcher[string]() + return c.roomNamespaces, c.publisherNamespaces, true + } + return c.roomNamespaces, c.publisherNamespaces, false +} + +func (c *agentClient) checkEnabled(ctx context.Context, roomNamespaces, publisherNamespaces *serverutils.IncrementalDispatcher[string]) { + defer roomNamespaces.Done() + defer publisherNamespaces.Done() + resChan, err := c.client.CheckEnabled(ctx, &rpc.CheckEnabledRequest{}, psrpc.WithRequestTimeout(CheckEnabledTimeout)) + if err != nil { + logger.Errorw("failed to check enabled", err) + return + } + + roomNSMap := make(map[string]bool) + publisherNSMap := make(map[string]bool) + + for r := range resChan { + if r.Result.GetRoomEnabled() { + for _, ns := range r.Result.GetNamespaces() { + if _, ok := roomNSMap[ns]; !ok { + roomNamespaces.Add(ns) + roomNSMap[ns] = true + } + } + } + if r.Result.GetPublisherEnabled() { + for _, ns := range r.Result.GetNamespaces() { + if _, ok := publisherNSMap[ns]; !ok { + publisherNamespaces.Add(ns) + publisherNSMap[ns] = true + } + } + } + } +} + +func (c *agentClient) Stop() error { + _ = c.invalidateSub.Close() + <-c.subDone + return nil +} diff --git a/pkg/agent/job.go b/pkg/agent/job.go new file mode 100644 index 000000000..6edfe4a27 --- /dev/null +++ b/pkg/agent/job.go @@ -0,0 +1,86 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "sync" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +// Represents a job that is being executed by a worker +type Job struct { + id string + jobType livekit.JobType + status livekit.JobStatus + namespace string + + mu sync.Mutex + load float32 + + Logger logger.Logger +} + +func NewJob(id, namespace string, jobType livekit.JobType) *Job { + return &Job{ + id: id, + status: livekit.JobStatus_JS_UNKNOWN, + jobType: jobType, + namespace: namespace, + } +} + +func (j *Job) ID() string { + return j.id +} + +func (j *Job) Namespace() string { + return j.namespace +} + +func (j *Job) Type() livekit.JobType { + return j.jobType +} + +func (j *Job) WorkerLoad() float32 { + // Current load that this job is taking on its worker + j.mu.Lock() + defer j.mu.Unlock() + return j.load +} + +func (j *Job) UpdateStatus(req *livekit.UpdateJobStatus) { + j.mu.Lock() + + if req.Status != nil { + j.status = *req.Status // End of the job, SUCCESS or FAILURE + + if j.status == livekit.JobStatus_JS_FAILED { + j.Logger.Errorw("job failed", nil, "id", j.id, "type", j.jobType, "error", req.Error) + } + } + + j.load = req.Load + j.mu.Unlock() + + if req.Metadata != nil { + j.UpdateMetadata(req.GetMetadata()) + } +} + +func (j *Job) UpdateMetadata(metadata string) { + j.Logger.Debugw("job metadata", nil, "id", j.id, "metadata", metadata) +} diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go new file mode 100644 index 000000000..666681b39 --- /dev/null +++ b/pkg/agent/worker.go @@ -0,0 +1,388 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + + pagent "github.com/livekit/protocol/agent" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + putil "github.com/livekit/protocol/utils" +) + +type WorkerProtocolVersion int + +const CurrentProtocol = 1 + +const ( + registerTimeout = 10 * time.Second + assignJobTimeout = 10 * time.Second + pingFrequency = 10 * time.Second +) + +var ( + ErrWorkerClosed = errors.New("worker closed") + ErrWorkerNotAvailable = errors.New("worker not available") + ErrAvailabilityTimeout = errors.New("agent worker availability timeout") +) + +type sigConn interface { + WriteServerMessage(msg *livekit.ServerMessage) (int, error) +} + +type Worker struct { + id string + jobType livekit.JobType + version string + name string + namespace string + load float32 + permissions *livekit.ParticipantPermission + apiKey string + apiSecret string + serverInfo *livekit.ServerInfo + mu sync.Mutex + + protocolVersion WorkerProtocolVersion + registered atomic.Bool + status livekit.WorkerStatus + runningJobs map[string]*Job + + onWorkerRegistered func(w *Worker) + + conn *websocket.Conn + sigConn sigConn + closed chan struct{} + + availability map[string]chan *livekit.AvailabilityResponse + + ctx context.Context + cancel context.CancelFunc + + Logger logger.Logger +} + +func NewWorker( + protocolVersion WorkerProtocolVersion, + apiKey string, + apiSecret string, + serverInfo *livekit.ServerInfo, + conn *websocket.Conn, + sigConn sigConn, + logger logger.Logger, +) *Worker { + ctx, cancel := context.WithCancel(context.Background()) + + w := &Worker{ + id: putil.NewGuid(utils.AgentWorkerPrefix), + protocolVersion: protocolVersion, + apiKey: apiKey, + apiSecret: apiSecret, + serverInfo: serverInfo, + closed: make(chan struct{}), + runningJobs: make(map[string]*Job), + availability: make(map[string]chan *livekit.AvailabilityResponse), + conn: conn, + sigConn: sigConn, + ctx: ctx, + cancel: cancel, + Logger: logger, + } + + go func() { + <-time.After(registerTimeout) + if !w.registered.Load() && !w.IsClosed() { + w.Logger.Warnw("worker did not register in time", nil, "id", w.id) + w.Close() + } + }() + + return w +} + +func (w *Worker) sendRequest(req *livekit.ServerMessage) { + if _, err := w.sigConn.WriteServerMessage(req); err != nil { + w.Logger.Errorw("error writing to websocket", err) + } +} + +func (w *Worker) ID() string { + return w.id +} + +func (w *Worker) JobType() livekit.JobType { + w.mu.Lock() + defer w.mu.Unlock() + return w.jobType +} + +func (w *Worker) Namespace() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.namespace +} + +func (w *Worker) Status() livekit.WorkerStatus { + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +func (w *Worker) Load() float32 { + w.mu.Lock() + defer w.mu.Unlock() + return w.load +} + +func (w *Worker) OnWorkerRegistered(f func(w *Worker)) { + w.mu.Lock() + defer w.mu.Unlock() + w.onWorkerRegistered = f +} + +func (w *Worker) Registered() bool { + return w.registered.Load() +} + +func (w *Worker) RunningJobs() map[string]*Job { + jobs := make(map[string]*Job, len(w.runningJobs)) + w.mu.Lock() + defer w.mu.Unlock() + for k, v := range w.runningJobs { + jobs[k] = v + } + return jobs +} + +func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) error { + availCh := make(chan *livekit.AvailabilityResponse, 1) + + w.mu.Lock() + w.availability[job.Id] = availCh + w.mu.Unlock() + + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ + Availability: &livekit.AvailabilityRequest{Job: job}, + }}) + + // See handleAvailability for the response + select { + case res := <-availCh: + if !res.Available { + return ErrWorkerNotAvailable + } + + token, err := pagent.BuildAgentToken(w.apiKey, w.apiSecret, job.Room.Name, res.ParticipantIdentity, res.ParticipantName, res.ParticipantMetadata, w.permissions) + if err != nil { + w.Logger.Errorw("failed to build agent token", err) + return err + } + + // In OSS, Url is nil, and the used API Key is the same as the one used to connect the worker + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Assignment{ + Assignment: &livekit.JobAssignment{Job: job, Url: nil, Token: token}, + }}) + + // TODO(theomonnom): Check if an agent was successfully connected to the room before returning + return nil + case <-time.After(assignJobTimeout): + return ErrAvailabilityTimeout + case <-w.ctx.Done(): + return ErrWorkerClosed + case <-ctx.Done(): + return ctx.Err() + } +} + +func (w *Worker) UpdateStatus(status *livekit.UpdateWorkerStatus) { + w.mu.Lock() + if status.Status != nil { + w.status = status.GetStatus() + } + w.load = status.GetLoad() + w.mu.Unlock() + + if status.Metadata != nil { + w.UpdateMetadata(status.GetMetadata()) + } +} + +func (w *Worker) UpdateMetadata(metadata string) { + w.Logger.Debugw("worker metadata updated", nil, "metadata", metadata) +} + +func (w *Worker) IsClosed() bool { + select { + case <-w.closed: + return true + default: + return false + } +} + +func (w *Worker) Close() { + w.mu.Lock() + if w.IsClosed() { + w.mu.Unlock() + return + } + + w.Logger.Infow("closing worker") + + close(w.closed) + w.cancel() + _ = w.conn.Close() + w.mu.Unlock() +} + +func (w *Worker) HandleMessage(req *livekit.WorkerMessage) { + switch m := req.Message.(type) { + case *livekit.WorkerMessage_Register: + go w.handleRegister(m.Register) + case *livekit.WorkerMessage_Availability: + go w.handleAvailability(m.Availability) + case *livekit.WorkerMessage_UpdateJob: + go w.handleJobUpdate(m.UpdateJob) + case *livekit.WorkerMessage_SimulateJob: + go w.handleSimulateJob(m.SimulateJob) + case *livekit.WorkerMessage_Ping: + go w.handleWorkerPing(m.Ping) + case *livekit.WorkerMessage_UpdateWorker: + go w.handleWorkerStatus(m.UpdateWorker) + case *livekit.WorkerMessage_MigrateJob: + go w.handleMigrateJob(m.MigrateJob) + } +} + +func (w *Worker) handleRegister(req *livekit.RegisterWorkerRequest) { + if w.registered.Load() { + w.Logger.Warnw("worker already registered", nil, "id", w.id) + return + } + + w.mu.Lock() + onWorkerRegistered := w.onWorkerRegistered + w.jobType = req.Type + w.version = req.Version + w.name = req.Name + w.namespace = req.GetNamespace() + + if req.AllowedPermissions != nil { + w.permissions = req.AllowedPermissions + } else { + // Use default agent permissions + w.permissions = &livekit.ParticipantPermission{ + CanSubscribe: true, + CanPublish: true, + CanPublishData: true, + CanUpdateMetadata: true, + } + } + + w.status = livekit.WorkerStatus_WS_AVAILABLE + w.registered.Store(true) + w.mu.Unlock() + + w.Logger.Debugw("worker registered", "request", req) + + w.sendRequest(&livekit.ServerMessage{ + Message: &livekit.ServerMessage_Register{ + Register: &livekit.RegisterWorkerResponse{ + WorkerId: w.ID(), + ServerInfo: w.serverInfo, + }, + }, + }) + + if onWorkerRegistered != nil { + onWorkerRegistered(w) + } +} + +func (w *Worker) handleAvailability(res *livekit.AvailabilityResponse) { + w.mu.Lock() + defer w.mu.Unlock() + + availCh, ok := w.availability[res.JobId] + if !ok { + w.Logger.Warnw("received availability response for unknown job", nil, "jobId", res.JobId) + return + } + + availCh <- res + delete(w.availability, res.JobId) +} + +func (w *Worker) handleJobUpdate(update *livekit.UpdateJobStatus) { + w.mu.Lock() + job, ok := w.runningJobs[update.JobId] + w.mu.Unlock() + + if !ok { + w.Logger.Warnw("received job update for unknown job", nil, "jobId", update.JobId) + return + } + + job.UpdateStatus(update) +} + +func (w *Worker) handleSimulateJob(simulate *livekit.SimulateJobRequest) { + jobType := livekit.JobType_JT_ROOM + if simulate.Participant != nil { + jobType = livekit.JobType_JT_PUBLISHER + } + + job := &livekit.Job{ + Id: utils.NewGuid(utils.AgentJobPrefix), + Type: jobType, + Room: simulate.Room, + Participant: simulate.Participant, + Namespace: w.Namespace(), + } + + ctx := context.Background() + err := w.AssignJob(ctx, job) + if err != nil { + w.Logger.Errorw("failed to simulate job, assignment failed", err, "jobId", job.Id) + } + +} + +func (w *Worker) handleWorkerPing(ping *livekit.WorkerPing) { + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Pong{ + Pong: &livekit.WorkerPong{ + LastTimestamp: ping.Timestamp, + Timestamp: time.Now().UnixMilli(), + }, + }}) +} + +func (w *Worker) handleWorkerStatus(update *livekit.UpdateWorkerStatus) { + w.Logger.Debugw("worker status update", "status", update.Status, "load", update.Load) + w.UpdateStatus(update) +} + +func (w *Worker) handleMigrateJob(migrate *livekit.MigrateJobRequest) { + // TODO(theomonnom): On OSS this is not implemented + // We could maybe just move a specific job to another worker +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 82f338f9a..78cd0f1e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,7 +61,6 @@ type Config struct { Port uint32 `yaml:"port,omitempty"` BindAddresses []string `yaml:"bind_addresses,omitempty"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` - Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` Redis redisLiveKit.RedisConfig `yaml:"redis,omitempty"` Audio AudioConfig `yaml:"audio,omitempty"` @@ -191,6 +190,8 @@ type AudioConfig struct { SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"` // enable red encoding downtrack for opus only audio up track ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` + // enable proxying weakest subscriber loss to publisher in RTCP Receiver Report + EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"` } type StreamTrackerPacketConfig struct { @@ -572,10 +573,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Logging.ComponentLevels["pion"] = conf.Logging.PionLevel } - if conf.Development { - conf.Environment = "dev" - } - return &conf, nil } diff --git a/pkg/rtc/agentclient.go b/pkg/rtc/agentclient.go deleted file mode 100644 index 65be3ba7c..000000000 --- a/pkg/rtc/agentclient.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rtc - -import ( - "context" - "time" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/rpc" - "github.com/livekit/psrpc" -) - -const ( - RoomAgentTopic = "room" - PublisherAgentTopic = "publisher" -) - -type AgentClient interface { - CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) *rpc.CheckEnabledResponse - JobRequest(ctx context.Context, job *livekit.Job) -} - -type agentClient struct { - client rpc.AgentInternalClient -} - -func NewAgentClient(bus psrpc.MessageBus) (AgentClient, error) { - client, err := rpc.NewAgentInternalClient(bus) - if err != nil { - return nil, err - } - return &agentClient{client: client}, nil -} - -func (c *agentClient) CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) *rpc.CheckEnabledResponse { - res := &rpc.CheckEnabledResponse{} - resChan, err := c.client.CheckEnabled(ctx, req, psrpc.WithRequestTimeout(time.Second)) - if err != nil { - return res - } - - for r := range resChan { - if r.Err != nil { - continue - } - if r.Result.RoomEnabled { - res.RoomEnabled = true - if res.PublisherEnabled { - return res - } - } - if r.Result.PublisherEnabled { - res.PublisherEnabled = true - if res.RoomEnabled { - return res - } - } - } - - return res -} - -func (c *agentClient) JobRequest(ctx context.Context, job *livekit.Job) { - var topic string - var logError bool - switch job.Type { - case livekit.JobType_JT_ROOM: - topic = RoomAgentTopic - case livekit.JobType_JT_PUBLISHER: - topic = PublisherAgentTopic - logError = true - } - - _, err := c.client.JobRequest(ctx, topic, job) - if err != nil && logError { - logger.Warnw("agent job request failed", err) - } -} diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index f8a8054be..5c608d1f2 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -20,7 +20,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/mediatransportutil/pkg/rtcconfig" ) @@ -88,6 +88,7 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { sdp.SDESMidURI, sdp.SDESRTPStreamIDURI, sdp.AudioLevelURI, + //act.AbsCaptureTimeURI, }, Video: []string{ sdp.SDESMidURI, @@ -96,6 +97,7 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { frameMarking, dd.ExtensionURI, repairedRTPStreamID, + //act.AbsCaptureTimeURI, }, }, RTCPFeedback: RTCPFeedbackConfig{ @@ -115,7 +117,13 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { subscriberConfig := DirectionConfig{ StrictACKs: conf.RTC.StrictACKs, RTPHeaderExtension: RTPHeaderExtensionConfig{ - Video: []string{dd.ExtensionURI}, + Video: []string{ + dd.ExtensionURI, + //act.AbsCaptureTimeURI, + }, + Audio: []string{ + //act.AbsCaptureTimeURI, + }, }, RTCPFeedback: RTCPFeedbackConfig{ Video: []webrtc.RTCPFeedback{ diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index 55836b472..a9b1d623b 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -29,9 +29,22 @@ const ( videoRTXMimeType = "video/rtx" ) -var opusCodecCapability = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"} -var redCodecCapability = webrtc.RTPCodecCapability{MimeType: sfu.MimeTypeAudioRed, ClockRate: 48000, Channels: 2, SDPFmtpLine: "111/111"} -var videoRTX = webrtc.RTPCodecCapability{MimeType: videoRTXMimeType, ClockRate: 90000} +var opusCodecCapability = webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", +} +var redCodecCapability = webrtc.RTPCodecCapability{ + MimeType: sfu.MimeTypeAudioRed, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "111/111", +} +var videoRTX = webrtc.RTPCodecCapability{ + MimeType: videoRTXMimeType, + ClockRate: 90000, +} func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { opusCodec := opusCodecCapability @@ -61,32 +74,65 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" for _, codec := range []webrtc.RTPCodecParameters{ { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 96, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 96, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 98, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP9, + ClockRate: 90000, + SDPFmtpLine: "profile-id=0", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 98, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 100, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP9, + ClockRate: 90000, + SDPFmtpLine: "profile-id=1", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 100, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 125, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 125, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 108, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 108, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: h264HighProfileFmtp, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 123, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: h264HighProfileFmtp, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 123, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeAV1, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 35, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeAV1, + ClockRate: 90000, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 35, }, } { if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp { diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index c28c563c3..f575b4e13 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -96,7 +96,6 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { }) t.MediaLossProxy.OnMediaLossUpdate(func(fractionalLoss uint8) { if t.buffer != nil { - // ok to access buffer since receivers are added before subscribers t.buffer.SetLastFractionLostReport(fractionalLoss) } }) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 681eac6a5..9e8b098fc 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -34,7 +34,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/telemetry" ) @@ -161,19 +161,23 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority receivers := slices.Clone(t.receivers) // codec position maybe taken by DummyReceiver, check and upgrade to WebRTCReceiver - var upgradeReceiver bool - for _, r := range receivers { + receiverToAdd := receiver + idx := -1 + for i, r := range receivers { if strings.EqualFold(r.Codec().MimeType, receiver.Codec().MimeType) { - if d, ok := r.TrackReceiver.(*DummyReceiver); ok { - d.Upgrade(receiver) - upgradeReceiver = true - break - } + idx = i + break } } - if !upgradeReceiver { - receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) + if idx != -1 { + if d, ok := receivers[idx].TrackReceiver.(*DummyReceiver); ok { + d.Upgrade(receiver) + receiverToAdd = d + } + // replace receiver + receivers = slices.Delete(receivers, idx, idx+1) } + receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiverToAdd, priority: priority}) sort.Slice(receivers, func(i, j int) bool { return receivers[i].Priority() < receivers[j].Priority() @@ -319,15 +323,21 @@ func (t *MediaTrackReceiver) TryClose() bool { return true } + numActiveReceivers := 0 for _, receiver := range t.receivers { - if dr, _ := receiver.TrackReceiver.(*DummyReceiver); dr != nil && dr.Receiver() != nil { - t.lock.RUnlock() - return false + dr, ok := receiver.TrackReceiver.(*DummyReceiver) + if !ok || dr.Receiver() != nil { + // !ok means real receiver OR + // dummy receiver with a regular receiver attached + numActiveReceivers++ } } t.lock.RUnlock() - t.Close() + if numActiveReceivers != 0 { + return false + } + t.Close() return true } @@ -639,7 +649,7 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { break } - // for client don't use simulcast codecs (old client version or single codec) + // for clients that don't use simulcast codecs (old client version or single codec) if i == 0 { clonedInfo.Layers = ci.Layers } @@ -657,33 +667,39 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { t.updateTrackInfoOfReceivers() } -func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { - t.lock.Lock() - // set video layer ssrc info - for i, ci := range t.trackInfo.Codecs { - originLayers := ci.Layers - ci.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - ci.Layers = append(ci.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - for _, l := range originLayers { - if l.Quality == ci.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - ci.Layers[layerIdx].Ssrc = l.Ssrc - } - break - } - } - } +func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) { + if t.Kind() != livekit.TrackType_AUDIO { + return + } - // for client don't use simulcast codecs (old client version or single codec) - if i == 0 { - t.trackInfo.Layers = ci.Layers + t.lock.Lock() + t.trackInfo.AudioFeatures = update.Features + t.trackInfo.Stereo = false + t.trackInfo.DisableDtx = false + for _, feature := range update.Features { + switch feature { + case livekit.AudioTrackFeature_TF_STEREO: + t.trackInfo.Stereo = true + case livekit.AudioTrackFeature_TF_NO_DTX: + t.trackInfo.DisableDtx = true } } t.lock.Unlock() t.updateTrackInfoOfReceivers() - t.MediaTrackSubscriptions.UpdateVideoLayers() +} + +func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) { + if t.Kind() != livekit.TrackType_VIDEO { + return + } + + t.lock.Lock() + t.trackInfo.Width = update.Width + t.trackInfo.Height = update.Height + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() } func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index ee67987e4..158cafe10 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -129,6 +129,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * downTrack, err := sfu.NewDownTrack(sfu.DowntrackParams{ Codecs: codecs, + Source: t.params.MediaTrack.Source(), Receiver: wr, BufferFactory: sub.GetBufferFactory(), SubID: subscriberID, @@ -322,13 +323,6 @@ func (t *MediaTrackSubscriptions) closeSubscribedTrack(subTrack types.Subscribed if willBeResumed { dt.CloseWithFlush(false) - - // cache transceiver for potential re-use on resume - tr := dt.GetTransceiver() - if tr != nil { - sub := subTrack.Subscriber() - sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState()) - } } else { // flushing blocks, avoid blocking when publisher removes all its subscribers go dt.CloseWithFlush(true) @@ -420,12 +414,27 @@ func (t *MediaTrackSubscriptions) downTrackClosed( willBeResumed bool, ) { subscriberID := sub.ID() - t.subscribedTracksMu.Lock() + t.subscribedTracksMu.RLock() subTrack := t.subscribedTracks[subscriberID] - delete(t.subscribedTracks, subscriberID) - t.subscribedTracksMu.Unlock() + t.subscribedTracksMu.RUnlock() if subTrack != nil { + // Cache transceiver for potential re-use on resume. + // To ensure subscription manager does not re-subscribe before caching, + // delete the subscribed track only after caching. + if willBeResumed { + dt := subTrack.DownTrack() + tr := dt.GetTransceiver() + if tr != nil { + sub := subTrack.Subscriber() + sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState()) + } + } + + t.subscribedTracksMu.Lock() + delete(t.subscribedTracks, subscriberID) + t.subscribedTracksMu.Unlock() + subTrack.Close(willBeResumed) } } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5f8f7e60e..0c3481449 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -75,6 +75,11 @@ type downTrackState struct { downTrack sfu.DownTrackState } +type postRtcpOp struct { + *ParticipantImpl + pkts []rtcp.Packet +} + // --------------------------------------------------------------- type participantUpdateInfo struct { @@ -147,7 +152,8 @@ type ParticipantImpl struct { isClosed atomic.Bool closeReason atomic.Value // types.ParticipantCloseReason - state atomic.Value // livekit.ParticipantInfo_State + state atomic.Value // livekit.ParticipantInfo_State + disconnected chan struct{} resSinkMu sync.Mutex resSink routing.MessageSink @@ -163,7 +169,7 @@ type ParticipantImpl struct { disconnectTimer *time.Timer migrationTimer *time.Timer - pubRTCPQueue *sutils.OpsQueue + pubRTCPQueue *sutils.TypedOpsQueue[postRtcpOp] // hold reference for MediaTrack twcc *twcc.Responder @@ -241,8 +247,9 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { return nil, ErrMissingGrants } p := &ParticipantImpl{ - params: params, - pubRTCPQueue: sutils.NewOpsQueue(sutils.OpsQueueParams{ + params: params, + disconnected: make(chan struct{}), + pubRTCPQueue: sutils.NewTypedOpsQueue[postRtcpOp](sutils.OpsQueueParams{ Name: "pub-rtcp", MinSize: 64, Logger: params.Logger, @@ -325,6 +332,32 @@ func (p *ParticipantImpl) State() livekit.ParticipantInfo_State { return p.state.Load().(livekit.ParticipantInfo_State) } +func (p *ParticipantImpl) Kind() livekit.ParticipantInfo_Kind { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.grants.GetParticipantKind() +} + +func (p *ParticipantImpl) IsRecorder() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.grants.GetParticipantKind() == livekit.ParticipantInfo_EGRESS || p.grants.Video.Recorder +} + +func (p *ParticipantImpl) IsDependent() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + switch p.grants.GetParticipantKind() { + case livekit.ParticipantInfo_AGENT, livekit.ParticipantInfo_EGRESS: + return true + default: + return p.grants.Video.Agent || p.grants.Video.Recorder + } +} + func (p *ParticipantImpl) ProtocolVersion() types.ProtocolVersion { return p.params.ProtocolVersion } @@ -346,6 +379,10 @@ func (p *ParticipantImpl) IsDisconnected() bool { return p.State() == livekit.ParticipantInfo_DISCONNECTED } +func (p *ParticipantImpl) Disconnected() <-chan struct{} { + return p.disconnected +} + func (p *ParticipantImpl) IsIdle() bool { // check if there are any published tracks that are subscribed for _, t := range p.GetPublishedTracks() { @@ -815,6 +852,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.UpTrackManager.Close(isExpectedToResume) p.updateState(livekit.ParticipantInfo_DISCONNECTED) + close(p.disconnected) // ensure this is synchronized p.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) @@ -949,7 +987,6 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { p.TransportManager.ProcessPendingPublisherOffer() case types.MigrateStateComplete: - p.TransportManager.ProcessPendingPublisherDataChannels() } @@ -1090,20 +1127,6 @@ func (p *ParticipantImpl) Hidden() bool { return p.hidden.Load() } -func (p *ParticipantImpl) IsRecorder() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.Recorder -} - -func (p *ParticipantImpl) IsAgent() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.Agent -} - func (p *ParticipantImpl) VerifySubscribeParticipantInfo(pID livekit.ParticipantID, version uint32) { if !p.IsReady() { // we have not sent a JoinResponse yet. metadata would be covered in JoinResponse @@ -1492,44 +1515,47 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt dp.ParticipantIdentity = string(p.params.Identity) } + shouldForward := false // only forward on user payloads switch payload := dp.Value.(type) { case *livekit.DataPacket_User: + u := payload.User + if p.Hidden() { + u.ParticipantSid = "" + u.ParticipantIdentity = "" + } else { + u.ParticipantSid = string(p.params.SID) + u.ParticipantIdentity = string(p.params.Identity) + } + if dp.ParticipantIdentity != "" { + u.ParticipantIdentity = dp.ParticipantIdentity + } else { + dp.ParticipantIdentity = u.ParticipantIdentity + } + if len(dp.DestinationIdentities) != 0 { + u.DestinationIdentities = dp.DestinationIdentities + } else { + dp.DestinationIdentities = u.DestinationIdentities + } + shouldForward = true + case *livekit.DataPacket_SipDtmf: + if p.Kind() == livekit.ParticipantInfo_SIP { + shouldForward = true + } + case *livekit.DataPacket_Transcription: + if p.Kind() == livekit.ParticipantInfo_AGENT { + shouldForward = true + } + default: + p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) + } + if shouldForward { p.lock.RLock() onDataPacket := p.onDataPacket p.lock.RUnlock() if onDataPacket != nil { - u := payload.User - if p.Hidden() { - u.ParticipantSid = "" - u.ParticipantIdentity = "" - } else { - u.ParticipantSid = string(p.params.SID) - u.ParticipantIdentity = string(p.params.Identity) - } - if dp.ParticipantIdentity != "" { - u.ParticipantIdentity = dp.ParticipantIdentity - } else { - dp.ParticipantIdentity = u.ParticipantIdentity - } - if len(dp.DestinationIdentities) != 0 { - u.DestinationIdentities = dp.DestinationIdentities - } else { - dp.DestinationIdentities = u.DestinationIdentities - } onDataPacket(p, kind, dp) } - case *livekit.DataPacket_SipDtmf: - if p.grants.GetParticipantKind() == livekit.ParticipantInfo_SIP { - p.lock.RLock() - onDataPacket := p.onDataPacket - p.lock.RUnlock() - if onDataPacket != nil { - onDataPacket(p, kind, dp) - } - } - default: - p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) } p.setIsPublisher(true) @@ -1792,6 +1818,12 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l Encryption: req.Encryption, Stream: req.Stream, } + if req.Stereo { + ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) + } + if req.DisableDtx { + ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) + } if ti.Stream == "" { ti.Stream = StreamFromTrackSource(ti.Source) } @@ -2341,11 +2373,30 @@ func (p *ParticipantImpl) DebugInfo() map[string]interface{} { } func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { - p.pubRTCPQueue.Enqueue(func() { - if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) { - p.pubLogger.Errorw("could not write RTCP to participant", err) + p.lock.RLock() + migrationTimer := p.migrationTimer + p.lock.RUnlock() + + // Once migration out is active, layers getting added would not be communicated to + // where the publisher is migrating to. Without SSRC, `UnhandleSimulcastInterceptor` + // cannot be set up on the migrating in node. Without that interceptor, simulcast + // probing will fail. + // + // Clients usually send `rid` RTP header extension till they get an RTCP Receiver Report + // from the remote side. So, by curbing RTCP when migration is active, even if a new layer + // get published to this node, client should continue to send `rid` to the new node + // post migration and the new node can do regular simulcast probing (without the + // `UnhandleSimulcastInterceptor`) to fire `OnTrack` on that layer. And when the new node + // sends RTCP Receiver Report back to the client, client will stop `rid`. + if migrationTimer != nil { + return + } + + p.pubRTCPQueue.Enqueue(func(op postRtcpOp) { + if err := op.TransportManager.WritePublisherRTCP(op.pkts); err != nil && !IsEOF(err) { + op.pubLogger.Errorw("could not write RTCP to participant", err) } - }) + }, postRtcpOp{p, pkts}) } func (p *ParticipantImpl) setDowntracksConnected() { @@ -2362,6 +2413,7 @@ func (p *ParticipantImpl) CacheDownTrack(trackID livekit.TrackID, rtpTransceiver p.subLogger.Infow("cached transceiver changed", "trackID", trackID) } p.cachedDownTracks[trackID] = &downTrackState{transceiver: rtpTransceiver, downTrack: downTrack} + p.subLogger.Debugw("caching downtrack", "trackID", trackID) p.lock.Unlock() } @@ -2369,6 +2421,9 @@ func (p *ParticipantImpl) UncacheDownTrack(rtpTransceiver *webrtc.RTPTransceiver p.lock.Lock() for trackID, dts := range p.cachedDownTracks { if dts.transceiver == rtpTransceiver { + if dts := p.cachedDownTracks[trackID]; dts != nil { + p.subLogger.Debugw("uncaching downtrack", "trackID", trackID) + } delete(p.cachedDownTracks, trackID) break } @@ -2380,8 +2435,7 @@ func (p *ParticipantImpl) GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.R p.lock.RLock() defer p.lock.RUnlock() - dts := p.cachedDownTracks[trackID] - if dts != nil { + if dts := p.cachedDownTracks[trackID]; dts != nil { return dts.transceiver, dts.downTrack } diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 2516291d3..d3fae1577 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -22,7 +22,7 @@ import ( "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/livekit" lksdp "github.com/livekit/protocol/sdp" ) diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index d4dbafa35..dc0a2405a 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -22,6 +22,7 @@ import ( "github.com/pion/webrtc/v3" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/routing" @@ -254,6 +255,9 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { func (p *ParticipantImpl) sendICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { trickle := ToProtoTrickle(c.ToJSON()) trickle.Target = target + + p.params.Logger.Debugw("sending ICE candidate", "transport", target, "trickle", logger.Proto(trickle)) + return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Trickle{ Trickle: trickle, diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index e50e84b83..560dc06a1 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -34,9 +34,9 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -103,8 +103,7 @@ type Room struct { trackManager *RoomTrackManager // agents - agentClient AgentClient - publisherAgentsEnabled bool + agentClient agent.Client // map of identity -> Participant participants map[livekit.ParticipantIdentity]types.LocalParticipant @@ -142,7 +141,7 @@ func NewRoom( audioConfig *config.AudioConfig, serverInfo *livekit.ServerInfo, telemetry telemetry.TelemetryService, - agentClient AgentClient, + agentClient agent.Client, egressLauncher EgressLauncher, ) *Room { r := &Room{ @@ -183,21 +182,6 @@ func NewRoom( } r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) - if agentClient != nil { - go func() { - res := r.agentClient.CheckEnabled(context.Background(), &rpc.CheckEnabledRequest{}) - if res.PublisherEnabled { - r.lock.Lock() - r.publisherAgentsEnabled = true - // if there are already published tracks, start the agents - for identity := range r.hasPublished { - r.launchPublisherAgent(r.participants[identity]) - } - r.lock.Unlock() - } - }() - } - go r.audioUpdateWorker() go r.connectionQualityWorker() go r.changeUpdateWorker() @@ -334,10 +318,10 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if r.participants[participant.Identity()] != nil { return ErrAlreadyJoined } - if r.protoRoom.MaxParticipants > 0 && !participant.IsRecorder() { + if r.protoRoom.MaxParticipants > 0 && !participant.IsDependent() { numParticipants := uint32(0) for _, p := range r.participants { - if !p.IsRecorder() { + if !p.IsDependent() { numParticipants++ } } @@ -495,7 +479,14 @@ func (r *Room) GetParticipantRequestSource(identity livekit.ParticipantIdentity) return r.participantRequestSources[identity] } -func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing.MessageSource, responseSink routing.MessageSink, iceServers []*livekit.ICEServer, reason livekit.ReconnectReason) error { +func (r *Room) ResumeParticipant( + p types.LocalParticipant, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + iceConfig *livekit.ICEConfig, + iceServers []*livekit.ICEServer, + reason livekit.ReconnectReason, +) error { r.ReplaceParticipantRequestSource(p.Identity(), requestSource) // close previous sink, and link to new one p.CloseSignalConnection(types.SignallingCloseReasonResume) @@ -537,7 +528,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing } _ = p.SendRoomUpdate(r.ToProto()) - p.ICERestart(nil) + p.ICERestart(iceConfig) // check for simulated signal disconnect on resume r.simulationLock.Lock() @@ -719,10 +710,6 @@ func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, return nil } -func (r *Room) UpdateVideoLayers(participant types.Participant, updateVideoLayers *livekit.UpdateVideoLayers) error { - return participant.UpdateVideoLayers(updateVideoLayers) -} - func (r *Room) ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { res := types.MediaResolverResult{} @@ -766,7 +753,7 @@ func (r *Room) CloseIfEmpty() { } for _, p := range r.participants { - if !p.IsRecorder() { + if !p.IsDependent() { r.lock.Unlock() return } @@ -1013,13 +1000,10 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. r.lock.Lock() hasPublished := r.hasPublished[participant.Identity()] r.hasPublished[participant.Identity()] = true - publisherAgentsEnabled := r.publisherAgentsEnabled r.lock.Unlock() if !hasPublished { - if publisherAgentsEnabled { - r.launchPublisherAgent(participant) - } + r.launchPublisherAgent(participant) if r.internal != nil && r.internal.ParticipantEgress != nil { go func() { if err := StartParticipantEgress( @@ -1245,7 +1229,7 @@ func (r *Room) updateProto() *livekit.Room { room.NumPublishers = 0 room.NumParticipants = 0 for _, p := range r.GetParticipants() { - if !p.IsRecorder() { + if !p.IsDependent() { room.NumParticipants++ } if p.IsPublisher() { @@ -1429,18 +1413,15 @@ func (r *Room) simulationCleanupWorker() { } func (r *Room) launchPublisherAgent(p types.Participant) { - if p == nil || p.IsRecorder() || p.IsAgent() { + if p == nil || p.IsDependent() || r.agentClient == nil { return } - go func() { - r.agentClient.JobRequest(context.Background(), &livekit.Job{ - Id: utils.NewGuid("JP_"), - Type: livekit.JobType_JT_PUBLISHER, - Room: r.ToProto(), - Participant: p.ToProto(), - }) - }() + go r.agentClient.LaunchJob(context.Background(), &agent.JobDescription{ + JobType: livekit.JobType_JT_PUBLISHER, + Room: r.ToProto(), + Participant: p.ToProto(), + }) } func (r *Room) DebugInfo() map[string]interface{} { diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index c73b5fa5c..40a8b30ad 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -38,7 +38,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } const ( diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index e835cf6ba..d7d9c75ad 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -27,8 +27,10 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant switch msg := req.GetMessage().(type) { case *livekit.SignalRequest_Offer: participant.HandleOffer(FromProtoSessionDescription(msg.Offer)) + case *livekit.SignalRequest_Answer: participant.HandleAnswer(FromProtoSessionDescription(msg.Answer)) + case *livekit.SignalRequest_Trickle: candidateInit, err := FromProtoTrickle(msg.Trickle) if err != nil { @@ -36,11 +38,14 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant return nil } participant.AddICECandidate(candidateInit, msg.Trickle.Target) + case *livekit.SignalRequest_AddTrack: pLogger.Debugw("add track request", "trackID", msg.AddTrack.Cid) participant.AddTrack(msg.AddTrack) + case *livekit.SignalRequest_Mute: participant.SetTrackMuted(livekit.TrackID(msg.Mute.Sid), msg.Mute.Muted, false) + case *livekit.SignalRequest_Subscription: // allow participant to indicate their interest in the subscription // permission check happens later in SubscriptionManager @@ -50,32 +55,30 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant msg.Subscription.ParticipantTracks, msg.Subscription.Subscribe, ) + case *livekit.SignalRequest_TrackSetting: for _, sid := range livekit.StringsAsIDs[livekit.TrackID](msg.TrackSetting.TrackSids) { participant.UpdateSubscribedTrackSettings(sid, msg.TrackSetting) } + case *livekit.SignalRequest_Leave: pLogger.Debugw("client leaving room") room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonClientRequestLeave) - case *livekit.SignalRequest_UpdateLayers: - err := room.UpdateVideoLayers(participant, msg.UpdateLayers) - if err != nil { - pLogger.Warnw("could not update video layers", err, - "update", msg.UpdateLayers) - return nil - } + case *livekit.SignalRequest_SubscriptionPermission: err := room.UpdateSubscriptionPermission(participant, msg.SubscriptionPermission) if err != nil { pLogger.Warnw("could not update subscription permission", err, "permissions", msg.SubscriptionPermission) } + case *livekit.SignalRequest_SyncState: err := room.SyncState(participant, msg.SyncState) if err != nil { pLogger.Warnw("could not sync state", err, "state", msg.SyncState) } + case *livekit.SignalRequest_Simulate: err := room.SimulateScenario(participant, msg.Simulate) if err != nil { @@ -92,6 +95,17 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { room.UpdateParticipantMetadata(participant, msg.UpdateMetadata.Name, msg.UpdateMetadata.Metadata) } + + case *livekit.SignalRequest_UpdateAudioTrack: + if err := participant.UpdateAudioTrack(msg.UpdateAudioTrack); err != nil { + pLogger.Warnw("could not update audio track", err, "update", msg.UpdateAudioTrack) + } + + case *livekit.SignalRequest_UpdateVideoTrack: + if err := participant.UpdateVideoTrack(msg.UpdateVideoTrack); err != nil { + pLogger.Warnw("could not update video track", err, "update", msg.UpdateVideoTrack) + } } + return nil } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 8232c7527..a7dfa6608 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -497,8 +497,6 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { s.setPublisher(res.PublisherIdentity, res.PublisherID) - // since hasPermission defaults to true, we will want to send a message to the client the first time - // that we discover permissions were denied permChanged := s.setHasPermission(res.HasPermission) if permChanged { m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), trackID, res.HasPermission) @@ -722,19 +720,20 @@ type trackSubscription struct { trackID livekit.TrackID logger logger.Logger - lock sync.RWMutex - desired bool - publisherID livekit.ParticipantID - publisherIdentity livekit.ParticipantIdentity - settings *livekit.UpdateTrackSettings - changedNotifier types.ChangeNotifier - removedNotifier types.ChangeNotifier - hasPermission bool - subscribedTrack types.SubscribedTrack - eventSent atomic.Bool - numAttempts atomic.Int32 - bound bool - kind atomic.Pointer[livekit.TrackType] + lock sync.RWMutex + desired bool + publisherID livekit.ParticipantID + publisherIdentity livekit.ParticipantIdentity + settings *livekit.UpdateTrackSettings + changedNotifier types.ChangeNotifier + removedNotifier types.ChangeNotifier + hasPermissionInitialized bool + hasPermission bool + subscribedTrack types.SubscribedTrack + eventSent atomic.Bool + numAttempts atomic.Int32 + bound bool + kind atomic.Pointer[livekit.TrackType] // the later of when subscription was requested OR when the first failure was encountered OR when permission is granted // this timestamp determines when failures are reported @@ -746,8 +745,6 @@ func newTrackSubscription(subscriberID livekit.ParticipantID, trackID livekit.Tr subscriberID: subscriberID, trackID: trackID, logger: l, - // default allow - hasPermission: true, } } @@ -796,9 +793,11 @@ func (s *trackSubscription) setDesired(desired bool) bool { func (s *trackSubscription) setHasPermission(perm bool) bool { s.lock.Lock() defer s.lock.Unlock() - if s.hasPermission == perm { + if s.hasPermissionInitialized && s.hasPermission == perm { return false } + + s.hasPermissionInitialized = true s.hasPermission = perm if s.hasPermission { // when permission is granted, reset the timer so it has sufficient time to reconcile diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 1e0cd1a93..8e062f6d9 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -24,6 +24,7 @@ import ( "github.com/bep/debounce" "github.com/pion/dtls/v2/pkg/crypto/elliptic" + "github.com/pion/ice/v2" "github.com/pion/interceptor" "github.com/pion/interceptor/pkg/cc" "github.com/pion/interceptor/pkg/gcc" @@ -47,12 +48,11 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" - sutils "github.com/livekit/livekit-server/pkg/utils" ) const ( @@ -74,8 +74,6 @@ const ( minConnectTimeoutAfterICE = 10 * time.Second maxConnectTimeoutAfterICE = 20 * time.Second // max duration for waiting pc to connect after ICE is connected - maxICECandidates = 20 - shortConnectionThreshold = 90 * time.Second ) @@ -122,6 +120,7 @@ func (s signal) String() string { // ------------------------------------------------------- type event struct { + *PCTransport signal signal data interface{} } @@ -150,10 +149,12 @@ type PCTransport struct { lock sync.RWMutex - reliableDC *webrtc.DataChannel - reliableDCOpened bool - lossyDC *webrtc.DataChannel - lossyDCOpened bool + firstOfferReceived bool + firstOfferNoDataChannel bool + reliableDC *webrtc.DataChannel + reliableDCOpened bool + lossyDC *webrtc.DataChannel + lossyDCOpened bool iceStartedAt time.Time iceConnectedAt time.Time @@ -183,7 +184,7 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool - eventsQueue *sutils.OpsQueue + eventsQueue *utils.TypedOpsQueue[event] // the following should be accessed only in event processing go routine cacheLocalCandidates bool @@ -222,9 +223,8 @@ type TransportParams struct { func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimator cc.BandwidthEstimator)) (*webrtc.PeerConnection, *webrtc.MediaEngine, error) { directionConfig := params.DirectionConfig - if params.AllowPlayoutDelay { - directionConfig.RTPHeaderExtension.Video = append(directionConfig.RTPHeaderExtension.Video, rtpextension.PlayoutDelayURI) + directionConfig.RTPHeaderExtension.Video = append(directionConfig.RTPHeaderExtension.Video, pd.PlayoutDelayURI) } // Some of the browser clients do not handle H.264 High Profile in signalling properly. @@ -304,6 +304,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat ir := &interceptor.Registry{} if params.IsSendSide { + se.DetachDataChannels() if params.CongestionControlConfig.UseSendSideBWE { gf, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) { return gcc.NewSendSideBWE( @@ -389,7 +390,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params: params, debouncedNegotiate: debounce.New(negotiationFrequency), negotiationState: transport.NegotiationStateNone, - eventsQueue: sutils.NewOpsQueue(utils.OpsQueueParams{ + eventsQueue: utils.NewTypedOpsQueue[event](utils.OpsQueueParams{ Name: "transport", MinSize: 64, Logger: params.Logger, @@ -401,7 +402,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { if params.IsSendSide { t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ Config: params.CongestionControlConfig, - Logger: params.Logger.WithComponent(sutils.ComponentCongestionControl), + Logger: params.Logger.WithComponent(utils.ComponentCongestionControl), }) t.streamAllocator.OnStreamStateChange(params.Handler.OnStreamStateChange) t.streamAllocator.Start() @@ -695,7 +696,9 @@ func (t *PCTransport) isFullyEstablished() bool { t.lock.RLock() defer t.lock.RUnlock() - return t.reliableDCOpened && t.lossyDCOpened && !t.connectedAt.IsZero() + dataChannelReady := t.firstOfferNoDataChannel || (t.reliableDCOpened && t.lossyDCOpened) + + return dataChannelReady && !t.connectedAt.IsZero() } func (t *PCTransport) SetPreferTCP(preferTCP bool) { @@ -703,6 +706,19 @@ func (t *PCTransport) SetPreferTCP(preferTCP bool) { } func (t *PCTransport) AddICECandidate(candidate webrtc.ICECandidateInit) { + if !t.params.Config.UseMDNS { + candidateValue := strings.TrimPrefix(candidate.Candidate, "candidate:") + if candidateValue != "" { + candidate, err := ice.UnmarshalCandidate(candidateValue) + if err != nil { + t.params.Logger.Errorw("failed to parse ice candidate", err) + } else if strings.HasSuffix(candidate.Address(), ".local") { + t.params.Logger.Debugw("ignoring mDNS candidate", "candidate", candidateValue) + return + } + } + } + t.postEvent(event{ signal: signalRemoteICECandidate, data: &candidate, @@ -841,8 +857,22 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni defer t.lock.Unlock() *dcPtr = dc if t.params.DirectionConfig.StrictACKs { - dc.OnOpen(dcReadyHandler) + dc.OnOpen(func() { + if t.params.IsSendSide { + if _, err := dc.Detach(); err != nil { + t.params.Logger.Warnw("failed to detach data channel", err) + } + } + dcReadyHandler() + }) } else { + dc.OnOpen(func() { + if t.params.IsSendSide { + if _, err := dc.Detach(); err != nil { + t.params.Logger.Warnw("failed to detach data channel", err) + } + } + }) dc.OnDial(dcReadyHandler) } dc.OnClose(dcCloseHandler) @@ -1241,38 +1271,34 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map return nil } -func (t *PCTransport) postEvent(event event) { - t.eventsQueue.Enqueue(func() { - err := t.handleEvent(&event) +func (t *PCTransport) postEvent(e event) { + e.PCTransport = t + t.eventsQueue.Enqueue(func(e event) { + var err error + switch e.signal { + case signalICEGatheringComplete: + err = e.handleICEGatheringComplete(e) + case signalLocalICECandidate: + err = e.handleLocalICECandidate(e) + case signalRemoteICECandidate: + err = e.handleRemoteICECandidate(e) + case signalSendOffer: + err = e.handleSendOffer(e) + case signalRemoteDescriptionReceived: + err = e.handleRemoteDescriptionReceived(e) + case signalICERestart: + err = e.handleICERestart(e) + } if err != nil { - if !t.isClosed.Load() { - t.params.Logger.Warnw("error handling event", err, "event", event.String()) - t.params.Handler.OnNegotiationFailed() + if !e.isClosed.Load() { + e.params.Logger.Warnw("error handling event", err, "event", e.String()) + e.params.Handler.OnNegotiationFailed() } } - }) + }, e) } -func (t *PCTransport) handleEvent(e *event) error { - switch e.signal { - case signalICEGatheringComplete: - return t.handleICEGatheringComplete(e) - case signalLocalICECandidate: - return t.handleLocalICECandidate(e) - case signalRemoteICECandidate: - return t.handleRemoteICECandidate(e) - case signalSendOffer: - return t.handleSendOffer(e) - case signalRemoteDescriptionReceived: - return t.handleRemoteDescriptionReceived(e) - case signalICERestart: - return t.handleICERestart(e) - } - - return nil -} - -func (t *PCTransport) handleICEGatheringComplete(_ *event) error { +func (t *PCTransport) handleICEGatheringComplete(_ event) error { if t.params.IsOfferer { return t.handleICEGatheringCompleteOfferer() } else { @@ -1318,6 +1344,7 @@ func (t *PCTransport) localDescriptionSent() error { for _, c := range cachedLocalCandidates { if err := t.params.Handler.OnICECandidate(c, t.params.Transport); err != nil { + t.params.Logger.Warnw("failed to send cached ICE candidate", err, "candidate", c) return err } } @@ -1330,16 +1357,13 @@ func (t *PCTransport) clearLocalDescriptionSent() { t.connectionDetails.Clear() } -func (t *PCTransport) handleLocalICECandidate(e *event) error { +func (t *PCTransport) handleLocalICECandidate(e event) error { c := e.data.(*webrtc.ICECandidate) filtered := false if c != nil { if t.preferTCP.Load() && c.Protocol != webrtc.ICEProtocolTCP { - t.params.Logger.Debugw("filtering out local candidate", - "candidate", func() interface{} { - return c.String() - }) + t.params.Logger.Debugw("filtering out local candidate", "candidate", c.String()) filtered = true } t.connectionDetails.AddLocalCandidate(c, filtered) @@ -1354,10 +1378,15 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { return nil } - return t.params.Handler.OnICECandidate(c, t.params.Transport) + if err := t.params.Handler.OnICECandidate(c, t.params.Transport); err != nil { + t.params.Logger.Warnw("failed to send ICE candidate", err, "candidate", c) + return err + } + + return nil } -func (t *PCTransport) handleRemoteICECandidate(e *event) error { +func (t *PCTransport) handleRemoteICECandidate(e event) error { c := e.data.(*webrtc.ICECandidateInit) filtered := false @@ -1377,7 +1406,10 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { } if err := t.pc.AddICECandidate(*c); err != nil { + t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") + } else { + t.params.Logger.Debugw("added cached ICE candidate", "candidate", c) } return nil @@ -1553,11 +1585,11 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { return t.localDescriptionSent() } -func (t *PCTransport) handleSendOffer(_ *event) error { +func (t *PCTransport) handleSendOffer(_ event) error { return t.createAndSendOffer(nil) } -func (t *PCTransport) handleRemoteDescriptionReceived(e *event) error { +func (t *PCTransport) handleRemoteDescriptionReceived(e event) error { sd := e.data.(*webrtc.SessionDescription) if sd.Type == webrtc.SDPTypeOffer { return t.handleRemoteOfferReceived(sd) @@ -1612,7 +1644,10 @@ func (t *PCTransport) setRemoteDescription(sd webrtc.SessionDescription) error { for _, c := range t.pendingRemoteCandidates { if err := t.pc.AddICECandidate(*c); err != nil { + t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") + } else { + t.params.Logger.Debugw("added cached ICE candidate", "candidate", c) } } t.pendingRemoteCandidates = nil @@ -1667,6 +1702,21 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e if err != nil { return nil } + + t.lock.Lock() + if !t.firstOfferReceived { + t.firstOfferReceived = true + var dataChannelFound bool + for _, media := range parsed.MediaDescriptions { + if strings.EqualFold(media.MediaName.Media, "application") { + dataChannelFound = true + break + } + } + t.firstOfferNoDataChannel = !dataChannelFound + } + t.lock.Unlock() + iceCredential, offerRestartICE, err := t.isRemoteOfferRestartICE(parsed) if err != nil { return errors.Wrap(err, "check remote offer restart ice failed") @@ -1689,7 +1739,7 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e if err := t.setRemoteDescription(*sd); err != nil { return err } - rtxRepairs := rtxRepairsFromSDP(parsed, t.params.Logger) + rtxRepairs := nonSimulcastRTXRepairsFromSDP(parsed, t.params.Logger) if len(rtxRepairs) > 0 { t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs) for repair, base := range rtxRepairs { @@ -1782,7 +1832,7 @@ func (t *PCTransport) doICERestart() error { } } -func (t *PCTransport) handleICERestart(_ *event) error { +func (t *PCTransport) handleICERestart(_ event) error { return t.doICERestart() } @@ -1820,11 +1870,19 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool tr.SetCodecPreferences(configCodecs) } -func rtxRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 { +func nonSimulcastRTXRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 { rtxRepairFlows := map[uint32]uint32{} for _, media := range s.MediaDescriptions { + // extract rtx repair flows from the media section for non-simulcast stream, + // pion will handle simulcast streams by rid probe, don't need handle it here. + var ridFound bool + rtxPairs := make(map[uint32]uint32) + findRTX: for _, attr := range media.Attributes { switch attr.Key { + case "rid": + ridFound = true + break findRTX case sdp.AttrKeySSRCGroup: split := strings.Split(attr.Value, " ") if split[0] == sdp.SemanticTokenFlowIdentification { @@ -1842,11 +1900,16 @@ func rtxRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint logger.Warnw("Failed to parse SSRC", err, "ssrc", split[2]) continue } - rtxRepairFlows[uint32(rtxRepairFlow)] = uint32(baseSsrc) + rtxPairs[uint32(rtxRepairFlow)] = uint32(baseSsrc) } } } } + if !ridFound { + for rtx, base := range rtxPairs { + rtxRepairFlows[rtx] = base + } + } } return rtxRepairFlows diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 049b76a5e..64569f196 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -16,11 +16,9 @@ package rtc import ( "math/bits" - "strings" "sync" "time" - "github.com/pion/ice/v2" "github.com/pion/rtcp" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" @@ -372,19 +370,6 @@ func (t *TransportManager) HandleAnswer(answer webrtc.SessionDescription) { // AddICECandidate adds candidates for remote peer func (t *TransportManager) AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget) { - if !t.params.Config.UseMDNS { - candidateValue := strings.TrimPrefix(candidate.Candidate, "candidate:") - if candidateValue != "" { - candidate, err := ice.UnmarshalCandidate(candidateValue) - if err != nil { - t.params.Logger.Errorw("failed to parse ice candidate", err) - } else if strings.HasSuffix(candidate.Address(), ".local") { - t.params.Logger.Debugw("ignoring mDNS candidate", "candidate", candidateValue, "target", target) - return - } - } - } - switch target { case livekit.SignalTarget_PUBLISHER: t.publisher.AddICECandidate(candidate) @@ -431,9 +416,7 @@ func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) } func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) error { - if iceConfig != nil { - t.SetICEConfig(iceConfig) - } + t.SetICEConfig(iceConfig) return t.subscriber.ICERestart() } @@ -445,7 +428,9 @@ func (t *TransportManager) OnICEConfigChanged(f func(iceConfig *livekit.ICEConfi } func (t *TransportManager) SetICEConfig(iceConfig *livekit.ICEConfig) { - t.configureICE(iceConfig, true) + if iceConfig != nil { + t.configureICE(iceConfig, true) + } } func (t *TransportManager) resetTransportConfigureLocked(reconfigured bool) { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index b0943d4f4..3b230b0ee 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -245,6 +245,9 @@ type Participant interface { Identity() livekit.ParticipantIdentity State() livekit.ParticipantInfo_State CloseReason() ParticipantCloseReason + Kind() livekit.ParticipantInfo_Kind + IsRecorder() bool + IsDependent() bool CanSkipBroadcast() bool ToProto() *livekit.ParticipantInfo @@ -265,8 +268,6 @@ type Participant interface { // permissions Hidden() bool - IsRecorder() bool - IsAgent() bool Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error @@ -278,7 +279,8 @@ type Participant interface { timedVersion utils.TimedVersion, resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error - UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error + UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error + UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error DebugInfo() map[string]interface{} } @@ -307,6 +309,7 @@ type LocalParticipant interface { IsClosed() bool IsReady() bool IsDisconnected() bool + Disconnected() <-chan struct{} IsIdle() bool SubscriberAsPrimary() bool GetClientInfo() *livekit.ClientInfo @@ -432,7 +435,6 @@ type Room interface { UpdateSubscriptionPermission(participant LocalParticipant, permissions *livekit.SubscriptionPermission) error SyncState(participant LocalParticipant, state *livekit.SyncState) error SimulateScenario(participant LocalParticipant, scenario *livekit.SimulateScenario) error - UpdateVideoLayers(participant Participant, updateVideoLayers *livekit.UpdateVideoLayers) error ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) MediaResolverResult GetLocalParticipants() []LocalParticipant UpdateParticipantMetadata(participant LocalParticipant, name string, metadata string) @@ -449,6 +451,8 @@ type MediaTrack interface { Stream() string UpdateTrackInfo(ti *livekit.TrackInfo) + UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) + UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) ToProto() *livekit.TrackInfo PublisherID() livekit.ParticipantID @@ -458,7 +462,6 @@ type MediaTrack interface { IsMuted() bool SetMuted(muted bool) - UpdateVideoLayers(layers []*livekit.VideoLayer) IsSimulcast() bool GetAudioLevel() (level float64, active bool) diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index be86e74c4..3545f9187 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -332,15 +332,20 @@ type FakeLocalMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } UpdateTrackInfoStub func(*livekit.TrackInfo) updateTrackInfoMutex sync.RWMutex updateTrackInfoArgsForCall []struct { arg1 *livekit.TrackInfo } - UpdateVideoLayersStub func([]*livekit.VideoLayer) - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 []*livekit.VideoLayer + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack } invocations map[string][][]interface{} invocationsMutex sync.RWMutex @@ -2077,6 +2082,38 @@ func (fake *FakeLocalMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.Tr }{result1} } +func (fake *FakeLocalMediaTrack) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) { + fake.updateAudioTrackMutex.Lock() + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + fake.UpdateAudioTrackStub(arg1) + } +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack)) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { fake.updateTrackInfoMutex.Lock() fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { @@ -2109,40 +2146,35 @@ func (fake *FakeLocalMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.Trac return argsForCall.arg1 } -func (fake *FakeLocalMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { - var arg1Copy []*livekit.VideoLayer - if arg1 != nil { - arg1Copy = make([]*livekit.VideoLayer, len(arg1)) - copy(arg1Copy, arg1) - } - fake.updateVideoLayersMutex.Lock() - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 []*livekit.VideoLayer - }{arg1Copy}) - stub := fake.UpdateVideoLayersStub - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1Copy}) - fake.updateVideoLayersMutex.Unlock() +func (fake *FakeLocalMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { + fake.updateVideoTrackMutex.Lock() + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() if stub != nil { - fake.UpdateVideoLayersStub(arg1) + fake.UpdateVideoTrackStub(arg1) } } -func (fake *FakeLocalMediaTrack) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) +func (fake *FakeLocalMediaTrack) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) } -func (fake *FakeLocalMediaTrack) UpdateVideoLayersCalls(stub func([]*livekit.VideoLayer)) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub +func (fake *FakeLocalMediaTrack) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack)) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub } -func (fake *FakeLocalMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.VideoLayer { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] +func (fake *FakeLocalMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] return argsForCall.arg1 } @@ -2219,10 +2251,12 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 37c03ef10..7705cda47 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -168,6 +168,16 @@ type FakeLocalParticipant struct { debugInfoReturnsOnCall map[int]struct { result1 map[string]interface{} } + DisconnectedStub func() <-chan struct{} + disconnectedMutex sync.RWMutex + disconnectedArgsForCall []struct { + } + disconnectedReturns struct { + result1 <-chan struct{} + } + disconnectedReturnsOnCall map[int]struct { + result1 <-chan struct{} + } GetAdaptiveStreamStub func() bool getAdaptiveStreamMutex sync.RWMutex getAdaptiveStreamArgsForCall []struct { @@ -444,16 +454,6 @@ type FakeLocalParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } - IsAgentStub func() bool - isAgentMutex sync.RWMutex - isAgentArgsForCall []struct { - } - isAgentReturns struct { - result1 bool - } - isAgentReturnsOnCall map[int]struct { - result1 bool - } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -464,6 +464,16 @@ type FakeLocalParticipant struct { isClosedReturnsOnCall map[int]struct { result1 bool } + IsDependentStub func() bool + isDependentMutex sync.RWMutex + isDependentArgsForCall []struct { + } + isDependentReturns struct { + result1 bool + } + isDependentReturnsOnCall map[int]struct { + result1 bool + } IsDisconnectedStub func() bool isDisconnectedMutex sync.RWMutex isDisconnectedArgsForCall []struct { @@ -530,6 +540,16 @@ type FakeLocalParticipant struct { issueFullReconnectArgsForCall []struct { arg1 types.ParticipantCloseReason } + KindStub func() livekit.ParticipantInfo_Kind + kindMutex sync.RWMutex + kindArgsForCall []struct { + } + kindReturns struct { + result1 livekit.ParticipantInfo_Kind + } + kindReturnsOnCall map[int]struct { + result1 livekit.ParticipantInfo_Kind + } MaybeStartMigrationStub func(bool, func()) bool maybeStartMigrationMutex sync.RWMutex maybeStartMigrationArgsForCall []struct { @@ -896,6 +916,17 @@ type FakeLocalParticipant struct { unsubscribeFromTrackArgsForCall []struct { arg1 livekit.TrackID } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) error + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } + updateAudioTrackReturns struct { + result1 error + } + updateAudioTrackReturnsOnCall map[int]struct { + result1 error + } UpdateLastSeenSignalStub func() updateLastSeenSignalMutex sync.RWMutex updateLastSeenSignalArgsForCall []struct { @@ -955,15 +986,15 @@ type FakeLocalParticipant struct { updateSubscriptionPermissionReturnsOnCall map[int]struct { result1 error } - UpdateVideoLayersStub func(*livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 *livekit.UpdateVideoLayers + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack } - updateVideoLayersReturns struct { + updateVideoTrackReturns struct { result1 error } - updateVideoLayersReturnsOnCall map[int]struct { + updateVideoTrackReturnsOnCall map[int]struct { result1 error } VerifySubscribeParticipantInfoStub func(livekit.ParticipantID, uint32) @@ -1760,6 +1791,59 @@ func (fake *FakeLocalParticipant) DebugInfoReturnsOnCall(i int, result1 map[stri }{result1} } +func (fake *FakeLocalParticipant) Disconnected() <-chan struct{} { + fake.disconnectedMutex.Lock() + ret, specificReturn := fake.disconnectedReturnsOnCall[len(fake.disconnectedArgsForCall)] + fake.disconnectedArgsForCall = append(fake.disconnectedArgsForCall, struct { + }{}) + stub := fake.DisconnectedStub + fakeReturns := fake.disconnectedReturns + fake.recordInvocation("Disconnected", []interface{}{}) + fake.disconnectedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) DisconnectedCallCount() int { + fake.disconnectedMutex.RLock() + defer fake.disconnectedMutex.RUnlock() + return len(fake.disconnectedArgsForCall) +} + +func (fake *FakeLocalParticipant) DisconnectedCalls(stub func() <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = stub +} + +func (fake *FakeLocalParticipant) DisconnectedReturns(result1 <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = nil + fake.disconnectedReturns = struct { + result1 <-chan struct{} + }{result1} +} + +func (fake *FakeLocalParticipant) DisconnectedReturnsOnCall(i int, result1 <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = nil + if fake.disconnectedReturnsOnCall == nil { + fake.disconnectedReturnsOnCall = make(map[int]struct { + result1 <-chan struct{} + }) + } + fake.disconnectedReturnsOnCall[i] = struct { + result1 <-chan struct{} + }{result1} +} + func (fake *FakeLocalParticipant) GetAdaptiveStream() bool { fake.getAdaptiveStreamMutex.Lock() ret, specificReturn := fake.getAdaptiveStreamReturnsOnCall[len(fake.getAdaptiveStreamArgsForCall)] @@ -3233,59 +3317,6 @@ func (fake *FakeLocalParticipant) IdentityReturnsOnCall(i int, result1 livekit.P }{result1} } -func (fake *FakeLocalParticipant) IsAgent() bool { - fake.isAgentMutex.Lock() - ret, specificReturn := fake.isAgentReturnsOnCall[len(fake.isAgentArgsForCall)] - fake.isAgentArgsForCall = append(fake.isAgentArgsForCall, struct { - }{}) - stub := fake.IsAgentStub - fakeReturns := fake.isAgentReturns - fake.recordInvocation("IsAgent", []interface{}{}) - fake.isAgentMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) IsAgentCallCount() int { - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - return len(fake.isAgentArgsForCall) -} - -func (fake *FakeLocalParticipant) IsAgentCalls(stub func() bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = stub -} - -func (fake *FakeLocalParticipant) IsAgentReturns(result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - fake.isAgentReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) IsAgentReturnsOnCall(i int, result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - if fake.isAgentReturnsOnCall == nil { - fake.isAgentReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isAgentReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -3339,6 +3370,59 @@ func (fake *FakeLocalParticipant) IsClosedReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeLocalParticipant) IsDependent() bool { + fake.isDependentMutex.Lock() + ret, specificReturn := fake.isDependentReturnsOnCall[len(fake.isDependentArgsForCall)] + fake.isDependentArgsForCall = append(fake.isDependentArgsForCall, struct { + }{}) + stub := fake.IsDependentStub + fakeReturns := fake.isDependentReturns + fake.recordInvocation("IsDependent", []interface{}{}) + fake.isDependentMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) IsDependentCallCount() int { + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() + return len(fake.isDependentArgsForCall) +} + +func (fake *FakeLocalParticipant) IsDependentCalls(stub func() bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = stub +} + +func (fake *FakeLocalParticipant) IsDependentReturns(result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + fake.isDependentReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) IsDependentReturnsOnCall(i int, result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + if fake.isDependentReturnsOnCall == nil { + fake.isDependentReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isDependentReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) IsDisconnected() bool { fake.isDisconnectedMutex.Lock() ret, specificReturn := fake.isDisconnectedReturnsOnCall[len(fake.isDisconnectedArgsForCall)] @@ -3697,6 +3781,59 @@ func (fake *FakeLocalParticipant) IssueFullReconnectArgsForCall(i int) types.Par return argsForCall.arg1 } +func (fake *FakeLocalParticipant) Kind() livekit.ParticipantInfo_Kind { + fake.kindMutex.Lock() + ret, specificReturn := fake.kindReturnsOnCall[len(fake.kindArgsForCall)] + fake.kindArgsForCall = append(fake.kindArgsForCall, struct { + }{}) + stub := fake.KindStub + fakeReturns := fake.kindReturns + fake.recordInvocation("Kind", []interface{}{}) + fake.kindMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) KindCallCount() int { + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() + return len(fake.kindArgsForCall) +} + +func (fake *FakeLocalParticipant) KindCalls(stub func() livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = stub +} + +func (fake *FakeLocalParticipant) KindReturns(result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + fake.kindReturns = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + +func (fake *FakeLocalParticipant) KindReturnsOnCall(i int, result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + if fake.kindReturnsOnCall == nil { + fake.kindReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantInfo_Kind + }) + } + fake.kindReturnsOnCall[i] = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + func (fake *FakeLocalParticipant) MaybeStartMigration(arg1 bool, arg2 func()) bool { fake.maybeStartMigrationMutex.Lock() ret, specificReturn := fake.maybeStartMigrationReturnsOnCall[len(fake.maybeStartMigrationArgsForCall)] @@ -5776,6 +5913,67 @@ func (fake *FakeLocalParticipant) UnsubscribeFromTrackArgsForCall(i int) livekit return argsForCall.arg1 } +func (fake *FakeLocalParticipant) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) error { + fake.updateAudioTrackMutex.Lock() + ret, specificReturn := fake.updateAudioTrackReturnsOnCall[len(fake.updateAudioTrackArgsForCall)] + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fakeReturns := fake.updateAudioTrackReturns + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack) error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackReturns(result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + fake.updateAudioTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackReturnsOnCall(i int, result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + if fake.updateAudioTrackReturnsOnCall == nil { + fake.updateAudioTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateAudioTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) UpdateLastSeenSignal() { fake.updateLastSeenSignalMutex.Lock() fake.updateLastSeenSignalArgsForCall = append(fake.updateLastSeenSignalArgsForCall, struct { @@ -6091,16 +6289,16 @@ func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionReturnsOnCall(i in }{result1} } -func (fake *FakeLocalParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 *livekit.UpdateVideoLayers +func (fake *FakeLocalParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { + fake.updateVideoTrackMutex.Lock() + ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack }{arg1}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1}) - fake.updateVideoLayersMutex.Unlock() + stub := fake.UpdateVideoTrackStub + fakeReturns := fake.updateVideoTrackReturns + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() if stub != nil { return stub(arg1) } @@ -6110,44 +6308,44 @@ func (fake *FakeLocalParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLay return fakeReturns.result1 } -func (fake *FakeLocalParticipant) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) +func (fake *FakeLocalParticipant) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) } -func (fake *FakeLocalParticipant) UpdateVideoLayersCalls(stub func(*livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub +func (fake *FakeLocalParticipant) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack) error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub } -func (fake *FakeLocalParticipant) UpdateVideoLayersArgsForCall(i int) *livekit.UpdateVideoLayers { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] +func (fake *FakeLocalParticipant) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] return argsForCall.arg1 } -func (fake *FakeLocalParticipant) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { +func (fake *FakeLocalParticipant) UpdateVideoTrackReturns(result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + fake.updateVideoTrackReturns = struct { result1 error }{result1} } -func (fake *FakeLocalParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { +func (fake *FakeLocalParticipant) UpdateVideoTrackReturnsOnCall(i int, result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + if fake.updateVideoTrackReturnsOnCall == nil { + fake.updateVideoTrackReturnsOnCall = make(map[int]struct { result1 error }) } - fake.updateVideoLayersReturnsOnCall[i] = struct { + fake.updateVideoTrackReturnsOnCall[i] = struct { result1 error }{result1} } @@ -6345,6 +6543,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.connectedAtMutex.RUnlock() fake.debugInfoMutex.RLock() defer fake.debugInfoMutex.RUnlock() + fake.disconnectedMutex.RLock() + defer fake.disconnectedMutex.RUnlock() fake.getAdaptiveStreamMutex.RLock() defer fake.getAdaptiveStreamMutex.RUnlock() fake.getAudioLevelMutex.RLock() @@ -6403,10 +6603,10 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() fake.isDisconnectedMutex.RLock() defer fake.isDisconnectedMutex.RUnlock() fake.isIdleMutex.RLock() @@ -6421,6 +6621,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.isSubscribedToMutex.RUnlock() fake.issueFullReconnectMutex.RLock() defer fake.issueFullReconnectMutex.RUnlock() + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() fake.maybeStartMigrationMutex.RLock() defer fake.maybeStartMigrationMutex.RUnlock() fake.migrateStateMutex.RLock() @@ -6517,6 +6719,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.uncacheDownTrackMutex.RUnlock() fake.unsubscribeFromTrackMutex.RLock() defer fake.unsubscribeFromTrackMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateLastSeenSignalMutex.RLock() defer fake.updateLastSeenSignalMutex.RUnlock() fake.updateMediaLossMutex.RLock() @@ -6531,8 +6735,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.updateSubscribedTrackSettingsMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() fake.verifySubscribeParticipantInfoMutex.RLock() defer fake.verifySubscribeParticipantInfoMutex.RUnlock() fake.waitUntilSubscribedMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index d4bdfc17e..174d49b82 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -268,15 +268,20 @@ type FakeMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } UpdateTrackInfoStub func(*livekit.TrackInfo) updateTrackInfoMutex sync.RWMutex updateTrackInfoArgsForCall []struct { arg1 *livekit.TrackInfo } - UpdateVideoLayersStub func([]*livekit.VideoLayer) - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 []*livekit.VideoLayer + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack } invocations map[string][][]interface{} invocationsMutex sync.RWMutex @@ -1663,6 +1668,38 @@ func (fake *FakeMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.TrackIn }{result1} } +func (fake *FakeMediaTrack) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) { + fake.updateAudioTrackMutex.Lock() + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + fake.UpdateAudioTrackStub(arg1) + } +} + +func (fake *FakeMediaTrack) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeMediaTrack) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack)) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeMediaTrack) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { fake.updateTrackInfoMutex.Lock() fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { @@ -1695,40 +1732,35 @@ func (fake *FakeMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.TrackInfo return argsForCall.arg1 } -func (fake *FakeMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { - var arg1Copy []*livekit.VideoLayer - if arg1 != nil { - arg1Copy = make([]*livekit.VideoLayer, len(arg1)) - copy(arg1Copy, arg1) - } - fake.updateVideoLayersMutex.Lock() - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 []*livekit.VideoLayer - }{arg1Copy}) - stub := fake.UpdateVideoLayersStub - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1Copy}) - fake.updateVideoLayersMutex.Unlock() +func (fake *FakeMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { + fake.updateVideoTrackMutex.Lock() + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() if stub != nil { - fake.UpdateVideoLayersStub(arg1) + fake.UpdateVideoTrackStub(arg1) } } -func (fake *FakeMediaTrack) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) +func (fake *FakeMediaTrack) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) } -func (fake *FakeMediaTrack) UpdateVideoLayersCalls(stub func([]*livekit.VideoLayer)) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub +func (fake *FakeMediaTrack) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack)) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub } -func (fake *FakeMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.VideoLayer { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] +func (fake *FakeMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] return argsForCall.arg1 } @@ -1789,10 +1821,12 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 9f6f97e14..9494b87e6 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -128,14 +128,14 @@ type FakeParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } - IsAgentStub func() bool - isAgentMutex sync.RWMutex - isAgentArgsForCall []struct { + IsDependentStub func() bool + isDependentMutex sync.RWMutex + isDependentArgsForCall []struct { } - isAgentReturns struct { + isDependentReturns struct { result1 bool } - isAgentReturnsOnCall map[int]struct { + isDependentReturnsOnCall map[int]struct { result1 bool } IsPublisherStub func() bool @@ -158,6 +158,16 @@ type FakeParticipant struct { isRecorderReturnsOnCall map[int]struct { result1 bool } + KindStub func() livekit.ParticipantInfo_Kind + kindMutex sync.RWMutex + kindArgsForCall []struct { + } + kindReturns struct { + result1 livekit.ParticipantInfo_Kind + } + kindReturnsOnCall map[int]struct { + result1 livekit.ParticipantInfo_Kind + } RemovePublishedTrackStub func(types.MediaTrack, bool, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { @@ -207,6 +217,17 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) error + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } + updateAudioTrackReturns struct { + result1 error + } + updateAudioTrackReturnsOnCall map[int]struct { + result1 error + } UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { @@ -220,15 +241,15 @@ type FakeParticipant struct { updateSubscriptionPermissionReturnsOnCall map[int]struct { result1 error } - UpdateVideoLayersStub func(*livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 *livekit.UpdateVideoLayers + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack } - updateVideoLayersReturns struct { + updateVideoTrackReturns struct { result1 error } - updateVideoLayersReturnsOnCall map[int]struct { + updateVideoTrackReturnsOnCall map[int]struct { result1 error } invocations map[string][][]interface{} @@ -848,15 +869,15 @@ func (fake *FakeParticipant) IdentityReturnsOnCall(i int, result1 livekit.Partic }{result1} } -func (fake *FakeParticipant) IsAgent() bool { - fake.isAgentMutex.Lock() - ret, specificReturn := fake.isAgentReturnsOnCall[len(fake.isAgentArgsForCall)] - fake.isAgentArgsForCall = append(fake.isAgentArgsForCall, struct { +func (fake *FakeParticipant) IsDependent() bool { + fake.isDependentMutex.Lock() + ret, specificReturn := fake.isDependentReturnsOnCall[len(fake.isDependentArgsForCall)] + fake.isDependentArgsForCall = append(fake.isDependentArgsForCall, struct { }{}) - stub := fake.IsAgentStub - fakeReturns := fake.isAgentReturns - fake.recordInvocation("IsAgent", []interface{}{}) - fake.isAgentMutex.Unlock() + stub := fake.IsDependentStub + fakeReturns := fake.isDependentReturns + fake.recordInvocation("IsDependent", []interface{}{}) + fake.isDependentMutex.Unlock() if stub != nil { return stub() } @@ -866,37 +887,37 @@ func (fake *FakeParticipant) IsAgent() bool { return fakeReturns.result1 } -func (fake *FakeParticipant) IsAgentCallCount() int { - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - return len(fake.isAgentArgsForCall) +func (fake *FakeParticipant) IsDependentCallCount() int { + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() + return len(fake.isDependentArgsForCall) } -func (fake *FakeParticipant) IsAgentCalls(stub func() bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = stub +func (fake *FakeParticipant) IsDependentCalls(stub func() bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = stub } -func (fake *FakeParticipant) IsAgentReturns(result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - fake.isAgentReturns = struct { +func (fake *FakeParticipant) IsDependentReturns(result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + fake.isDependentReturns = struct { result1 bool }{result1} } -func (fake *FakeParticipant) IsAgentReturnsOnCall(i int, result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - if fake.isAgentReturnsOnCall == nil { - fake.isAgentReturnsOnCall = make(map[int]struct { +func (fake *FakeParticipant) IsDependentReturnsOnCall(i int, result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + if fake.isDependentReturnsOnCall == nil { + fake.isDependentReturnsOnCall = make(map[int]struct { result1 bool }) } - fake.isAgentReturnsOnCall[i] = struct { + fake.isDependentReturnsOnCall[i] = struct { result1 bool }{result1} } @@ -1007,6 +1028,59 @@ func (fake *FakeParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeParticipant) Kind() livekit.ParticipantInfo_Kind { + fake.kindMutex.Lock() + ret, specificReturn := fake.kindReturnsOnCall[len(fake.kindArgsForCall)] + fake.kindArgsForCall = append(fake.kindArgsForCall, struct { + }{}) + stub := fake.KindStub + fakeReturns := fake.kindReturns + fake.recordInvocation("Kind", []interface{}{}) + fake.kindMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) KindCallCount() int { + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() + return len(fake.kindArgsForCall) +} + +func (fake *FakeParticipant) KindCalls(stub func() livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = stub +} + +func (fake *FakeParticipant) KindReturns(result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + fake.kindReturns = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + +func (fake *FakeParticipant) KindReturnsOnCall(i int, result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + if fake.kindReturnsOnCall == nil { + fake.kindReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantInfo_Kind + }) + } + fake.kindReturnsOnCall[i] = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { @@ -1267,6 +1341,67 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } +func (fake *FakeParticipant) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) error { + fake.updateAudioTrackMutex.Lock() + ret, specificReturn := fake.updateAudioTrackReturnsOnCall[len(fake.updateAudioTrackArgsForCall)] + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fakeReturns := fake.updateAudioTrackReturns + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeParticipant) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack) error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeParticipant) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeParticipant) UpdateAudioTrackReturns(result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + fake.updateAudioTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeParticipant) UpdateAudioTrackReturnsOnCall(i int, result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + if fake.updateAudioTrackReturnsOnCall == nil { + fake.updateAudioTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateAudioTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] @@ -1330,16 +1465,16 @@ func (fake *FakeParticipant) UpdateSubscriptionPermissionReturnsOnCall(i int, re }{result1} } -func (fake *FakeParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 *livekit.UpdateVideoLayers +func (fake *FakeParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { + fake.updateVideoTrackMutex.Lock() + ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack }{arg1}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1}) - fake.updateVideoLayersMutex.Unlock() + stub := fake.UpdateVideoTrackStub + fakeReturns := fake.updateVideoTrackReturns + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() if stub != nil { return stub(arg1) } @@ -1349,44 +1484,44 @@ func (fake *FakeParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLayers) return fakeReturns.result1 } -func (fake *FakeParticipant) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) +func (fake *FakeParticipant) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) } -func (fake *FakeParticipant) UpdateVideoLayersCalls(stub func(*livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub +func (fake *FakeParticipant) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack) error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub } -func (fake *FakeParticipant) UpdateVideoLayersArgsForCall(i int) *livekit.UpdateVideoLayers { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] +func (fake *FakeParticipant) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] return argsForCall.arg1 } -func (fake *FakeParticipant) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { +func (fake *FakeParticipant) UpdateVideoTrackReturns(result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + fake.updateVideoTrackReturns = struct { result1 error }{result1} } -func (fake *FakeParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { +func (fake *FakeParticipant) UpdateVideoTrackReturnsOnCall(i int, result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + if fake.updateVideoTrackReturnsOnCall == nil { + fake.updateVideoTrackReturnsOnCall = make(map[int]struct { result1 error }) } - fake.updateVideoLayersReturnsOnCall[i] = struct { + fake.updateVideoTrackReturnsOnCall[i] = struct { result1 error }{result1} } @@ -1416,12 +1551,14 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() fake.isPublisherMutex.RLock() defer fake.isPublisherMutex.RUnlock() fake.isRecorderMutex.RLock() defer fake.isRecorderMutex.RUnlock() + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() fake.setMetadataMutex.RLock() @@ -1434,10 +1571,12 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.subscriptionPermissionMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_room.go b/pkg/rtc/types/typesfakes/fake_room.go index 67d68e8ca..0abe91d69 100644 --- a/pkg/rtc/types/typesfakes/fake_room.go +++ b/pkg/rtc/types/typesfakes/fake_room.go @@ -109,18 +109,6 @@ type FakeRoom struct { arg3 []*livekit.ParticipantTracks arg4 bool } - UpdateVideoLayersStub func(types.Participant, *livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 types.Participant - arg2 *livekit.UpdateVideoLayers - } - updateVideoLayersReturns struct { - result1 error - } - updateVideoLayersReturnsOnCall map[int]struct { - result1 error - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -645,68 +633,6 @@ func (fake *FakeRoom) UpdateSubscriptionsArgsForCall(i int) (types.LocalParticip return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 } -func (fake *FakeRoom) UpdateVideoLayers(arg1 types.Participant, arg2 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 types.Participant - arg2 *livekit.UpdateVideoLayers - }{arg1, arg2}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1, arg2}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRoom) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeRoom) UpdateVideoLayersCalls(stub func(types.Participant, *livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeRoom) UpdateVideoLayersArgsForCall(i int) (types.Participant, *livekit.UpdateVideoLayers) { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRoom) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.updateVideoLayersReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeRoom) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -730,8 +656,6 @@ func (fake *FakeRoom) Invocations() map[string][][]interface{} { defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateSubscriptionsMutex.RLock() defer fake.updateSubscriptionsMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 57cf372ac..a0674324e 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -239,14 +239,29 @@ func (u *UpTrackManager) HasPermission(trackID livekit.TrackID, subIdentity live return u.hasPermissionLocked(trackID, subIdentity) } -func (u *UpTrackManager) UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error { - track := u.GetPublishedTrack(livekit.TrackID(updateVideoLayers.TrackSid)) +func (u *UpTrackManager) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error { + track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid)) if track == nil { - u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(updateVideoLayers.TrackSid)) + u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(update.TrackSid)) return errors.New("could not find published track") } - track.UpdateVideoLayers(updateVideoLayers.Layers) + track.UpdateAudioTrack(update) + if u.onTrackUpdated != nil { + u.onTrackUpdated(track) + } + + return nil +} + +func (u *UpTrackManager) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error { + track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid)) + if track == nil { + u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(update.TrackSid)) + return errors.New("could not find published track") + } + + track.UpdateVideoTrack(update) if u.onTrackUpdated != nil { u.onTrackUpdated(track) } diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 8f9f91d1d..57572d530 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -26,7 +26,6 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -324,20 +323,6 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) - } - return 0, errors.New("receiver not available") -} - -func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetRTCPSenderReportData(layer) - } - return nil -} - func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetTrackStats() diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index 20636bc60..9a7106438 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -1,4 +1,4 @@ -// Copyright 2023 LiveKit, Inc. +// Copyright 2024 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "io" "math/rand" "net/http" + "strconv" "strings" "sync" "time" @@ -27,16 +28,19 @@ import ( "github.com/gorilla/websocket" "google.golang.org/protobuf/types/known/emptypb" + "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" - "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/version" + "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" ) -const AgentServiceVersion = "0.1.0" - type AgentService struct { upgrader websocket.Upgrader @@ -44,38 +48,31 @@ type AgentService struct { } type AgentHandler struct { - agentServer rpc.AgentInternalServer - roomTopic string - publisherTopic string + agentServer rpc.AgentInternalServer + mu sync.Mutex + logger logger.Logger - mu sync.Mutex - availability map[string]chan *availability - unregistered map[*websocket.Conn]*worker - roomRegistered bool - roomWorkers map[string]*worker - publisherRegistered bool - publisherWorkers map[string]*worker - onWorkerRegistered func(handler *AgentHandler) + serverInfo *livekit.ServerInfo + workers map[string]*agent.Worker + keyProvider auth.KeyProvider + + namespaces map[string]*namespaceInfo + publisherEnabled bool + roomEnabled bool + roomTopic string + publisherTopic string } -type worker struct { - mu sync.Mutex - conn *websocket.Conn - sigConn *WSSignalConnection - - id string - jobType livekit.JobType - status livekit.WorkerStatus - activeJobs int - logger logger.Logger +type namespaceInfo struct { + numPublishers int32 + numRooms int32 } -type availability struct { - workerID string - available bool -} - -func NewAgentService(bus psrpc.MessageBus) (*AgentService, error) { +func NewAgentService(conf *config.Config, + currentNode routing.LocalNode, + bus psrpc.MessageBus, + keyProvider auth.KeyProvider, +) (*AgentService, error) { s := &AgentService{ upgrader: websocket.Upgrader{}, } @@ -86,12 +83,27 @@ func NewAgentService(bus psrpc.MessageBus) (*AgentService, error) { return true } + serverInfo := &livekit.ServerInfo{ + Edition: livekit.ServerInfo_Standard, + Version: version.Version, + Protocol: types.CurrentProtocol, + AgentProtocol: agent.CurrentProtocol, + Region: conf.Region, + NodeId: currentNode.Id, + } + agentServer, err := rpc.NewAgentInternalServer(s, bus) if err != nil { return nil, err } - s.AgentHandler = NewAgentHandler(agentServer, rtc.RoomAgentTopic, rtc.PublisherAgentTopic) - + s.AgentHandler = NewAgentHandler( + agentServer, + keyProvider, + logger.GetLogger(), + serverInfo, + agent.RoomAgentTopic, + agent.PublisherAgentTopic, + ) return s, nil } @@ -116,64 +128,64 @@ func (s *AgentService) ServeHTTP(writer http.ResponseWriter, r *http.Request) { return } - s.HandleConnection(r.Context(), conn) + s.HandleConnection(r, conn, nil) } -func NewAgentHandler(agentServer rpc.AgentInternalServer, roomTopic, publisherTopic string) *AgentHandler { +func NewAgentHandler( + agentServer rpc.AgentInternalServer, + keyProvider auth.KeyProvider, + logger logger.Logger, + serverInfo *livekit.ServerInfo, + roomTopic string, + publisherTopic string, +) *AgentHandler { return &AgentHandler{ - agentServer: agentServer, - roomTopic: roomTopic, - publisherTopic: publisherTopic, - availability: make(map[string]chan *availability), - unregistered: make(map[*websocket.Conn]*worker), - roomWorkers: make(map[string]*worker), - publisherWorkers: make(map[string]*worker), + agentServer: agentServer, + logger: logger, + workers: make(map[string]*agent.Worker), + namespaces: make(map[string]*namespaceInfo), + serverInfo: serverInfo, + keyProvider: keyProvider, + roomTopic: roomTopic, + publisherTopic: publisherTopic, } } -// OnWorkerRegistered registers a callback to be called when the first worker of each type is registered -func (s *AgentHandler) OnWorkerRegistered(handler func(handler *AgentHandler)) { - s.mu.Lock() - defer s.mu.Unlock() - s.onWorkerRegistered = handler -} +func (h *AgentHandler) HandleConnection(r *http.Request, conn *websocket.Conn, onIdle func()) { + var protocol agent.WorkerProtocolVersion + if pv, err := strconv.Atoi(r.FormValue("protocol")); err == nil { + protocol = agent.WorkerProtocolVersion(pv) + } -func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Conn) { sigConn := NewWSSignalConnection(conn) - w := &worker{ - conn: conn, - sigConn: sigConn, - logger: utils.GetLogger(ctx), - } - s.mu.Lock() - s.unregistered[conn] = w - s.mu.Unlock() + apiKey := GetAPIKey(r.Context()) + apiSecret := h.keyProvider.GetSecret(apiKey) + + worker := agent.NewWorker(protocol, apiKey, apiSecret, h.serverInfo, conn, sigConn, h.logger) + worker.OnWorkerRegistered(h.handleWorkerRegister) + + h.mu.Lock() + h.workers[worker.ID()] = worker + h.mu.Unlock() defer func() { - s.mu.Lock() - if w.id == "" { - delete(s.unregistered, conn) - } else { - switch w.jobType { - case livekit.JobType_JT_ROOM: - delete(s.roomWorkers, w.id) - if s.roomRegistered && !s.roomAvailableLocked() { - s.roomRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.roomTopic) - } - case livekit.JobType_JT_PUBLISHER: - delete(s.publisherWorkers, w.id) - if s.publisherRegistered && !s.publisherAvailableLocked() { - s.publisherRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.publisherTopic) - } - } + worker.Close() + + h.mu.Lock() + delete(h.workers, worker.ID()) + numWorkers := len(h.workers) + h.mu.Unlock() + + if worker.Registered() { + h.handleWorkerDeregister(worker) + } + + if numWorkers == 0 && onIdle != nil { + onIdle() } - s.mu.Unlock() }() - // handle incoming requests from websocket for { req, _, err := sigConn.ReadWorkerMessage() if err != nil { @@ -188,321 +200,239 @@ func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Con websocket.CloseNormalClosure, websocket.CloseNoStatusReceived, ) { - w.logger.Infow("Agent worker closed WS connection", "wsError", err) + worker.Logger.Infow("worker closed WS connection", "wsError", err) } else { - w.logger.Errorw("error reading from websocket", err) + worker.Logger.Errorw("error reading from websocket", err) } return } - switch m := req.Message.(type) { - case *livekit.WorkerMessage_Register: - go s.handleRegister(w, m.Register) - case *livekit.WorkerMessage_Availability: - go s.handleAvailability(w, m.Availability) - case *livekit.WorkerMessage_JobUpdate: - go s.handleJobUpdate(w, m.JobUpdate) - case *livekit.WorkerMessage_Status: - go s.handleStatus(w, m.Status) - } + worker.HandleMessage(req) } } -func (s *AgentHandler) handleRegister(worker *worker, msg *livekit.RegisterWorkerRequest) { - if err := s.doHandleRegister(worker, msg); err != nil { - worker.logger.Errorw("failed to register worker", err, "workerID", msg.WorkerId, "jobType", msg.Type) - worker.conn.Close() - } -} - -func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWorkerRequest) error { - if msg.WorkerId == "" { - return errors.New("invalid worker id") - } - - s.mu.Lock() - if worker.id != "" { - s.mu.Unlock() - return errors.New("worker already registered") - } - - onRegistered := s.onWorkerRegistered - firstWorker := false - - switch msg.Type { - case livekit.JobType_JT_ROOM: - worker.id = msg.WorkerId - worker.jobType = msg.Type - delete(s.unregistered, worker.conn) - s.roomWorkers[worker.id] = worker - - if !s.roomRegistered { - err := s.agentServer.RegisterJobRequestTopic(s.roomTopic) - if err != nil { - worker.logger.Errorw("failed to register room agents", err) - } else { - s.roomRegistered = true - firstWorker = true - } - } - - case livekit.JobType_JT_PUBLISHER: - worker.id = msg.WorkerId - worker.jobType = msg.Type - delete(s.unregistered, worker.conn) - s.publisherWorkers[worker.id] = worker - - if !s.publisherRegistered { - err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic) - if err != nil { - worker.logger.Errorw("failed to register publisher agents", err) - } else { - s.publisherRegistered = true - firstWorker = true - } - } - default: - s.mu.Unlock() - return errors.New("invalid job type") - } - s.mu.Unlock() - - _, err := worker.sigConn.WriteServerMessage(&livekit.ServerMessage{ - Message: &livekit.ServerMessage_Register{ - Register: &livekit.RegisterWorkerResponse{ - WorkerId: worker.id, - ServerVersion: AgentServiceVersion, - }, - }, - }) - if err != nil { - worker.logger.Errorw("failed to write server message", err) - } - - if firstWorker && onRegistered != nil { - onRegistered(s) - } - return nil -} - -func (s *AgentHandler) handleAvailability(w *worker, msg *livekit.AvailabilityResponse) { - s.mu.Lock() - availabilityChan, ok := s.availability[msg.JobId] - s.mu.Unlock() +func (h *AgentHandler) handleWorkerRegister(w *agent.Worker) { + h.mu.Lock() + info, ok := h.namespaces[w.Namespace()] + numPublishers := int32(0) + numRooms := int32(0) if ok { - availabilityChan <- &availability{ - workerID: w.id, - available: msg.Available, + numPublishers = info.numPublishers + numRooms = info.numRooms + } + + shouldNotify := false + var err error + if w.JobType() == livekit.JobType_JT_PUBLISHER { + numPublishers++ + if numPublishers == 1 { + shouldNotify = true + err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.publisherTopic) + } + + } else if w.JobType() == livekit.JobType_JT_ROOM { + numRooms++ + if numRooms == 1 { + shouldNotify = true + err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.roomTopic) + } + } + + if err != nil { + w.Logger.Errorw("failed to register job request topic", err) + h.mu.Unlock() + w.Close() // Close the worker + return + } + + h.namespaces[w.Namespace()] = &namespaceInfo{ + numPublishers: numPublishers, + numRooms: numRooms, + } + + h.roomEnabled = h.roomAvailableLocked() + h.publisherEnabled = h.publisherAvailableLocked() + h.mu.Unlock() + + if shouldNotify { + h.logger.Infow("initial worker registered", "namespace", w.Namespace(), "jobType", w.JobType()) + err = h.agentServer.PublishWorkerRegistered(context.Background(), agent.DefaultHandlerNamespace, &emptypb.Empty{}) + if err != nil { + w.Logger.Errorw("failed to publish worker registered", err) } } } -func (s *AgentHandler) handleJobUpdate(w *worker, msg *livekit.JobStatusUpdate) { - switch msg.Status { - case livekit.JobStatus_JS_SUCCESS: - w.logger.Debugw("job complete", "jobID", msg.JobId) - case livekit.JobStatus_JS_FAILED: - w.logger.Warnw("job failed", errors.New(msg.Error), "jobID", msg.JobId) +func (h *AgentHandler) handleWorkerDeregister(worker *agent.Worker) { + h.mu.Lock() + defer h.mu.Unlock() + + info, ok := h.namespaces[worker.Namespace()] + if !ok { + return } - w.mu.Lock() - w.activeJobs-- - w.mu.Unlock() -} - -func (s *AgentHandler) handleStatus(w *worker, msg *livekit.UpdateWorkerStatus) { - s.mu.Lock() - defer s.mu.Unlock() - - w.mu.Lock() - w.status = msg.Status - w.mu.Unlock() - - switch w.jobType { - case livekit.JobType_JT_ROOM: - if s.roomRegistered && !s.roomAvailableLocked() { - s.roomRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.roomTopic) - } else if !s.roomRegistered && s.roomAvailableLocked() { - if err := s.agentServer.RegisterJobRequestTopic(s.roomTopic); err != nil { - w.logger.Errorw("failed to register room agents", err) - } else { - s.roomRegistered = true - } + if worker.JobType() == livekit.JobType_JT_PUBLISHER { + info.numPublishers-- + if info.numPublishers == 0 { + h.agentServer.DeregisterJobRequestTopic(worker.Namespace(), h.publisherTopic) } - case livekit.JobType_JT_PUBLISHER: - if s.publisherRegistered && !s.publisherAvailableLocked() { - s.publisherRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.publisherTopic) - } else if !s.publisherRegistered && s.publisherAvailableLocked() { - if err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic); err != nil { - w.logger.Errorw("failed to register publisher agents", err) - } else { - s.publisherRegistered = true - } + } else if worker.JobType() == livekit.JobType_JT_ROOM { + info.numRooms-- + if info.numRooms == 0 { + h.agentServer.DeregisterJobRequestTopic(worker.Namespace(), h.roomTopic) } } -} -func (s *AgentHandler) CheckEnabled(_ context.Context, _ *rpc.CheckEnabledRequest) (*rpc.CheckEnabledResponse, error) { - s.mu.Lock() - res := &rpc.CheckEnabledResponse{ - RoomEnabled: len(s.roomWorkers) > 0, - PublisherEnabled: len(s.publisherWorkers) > 0, - } - s.mu.Unlock() - return res, nil -} - -func (s *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*emptypb.Empty, error) { - s.mu.Lock() - ac := make(chan *availability, 100) - s.availability[job.Id] = ac - s.mu.Unlock() - - defer func() { - s.mu.Lock() - delete(s.availability, job.Id) - s.mu.Unlock() - }() - - var pool map[string]*worker - switch job.Type { - case livekit.JobType_JT_ROOM: - pool = s.roomWorkers - case livekit.JobType_JT_PUBLISHER: - pool = s.publisherWorkers + if info.numPublishers == 0 && info.numRooms == 0 { + h.logger.Debugw("last worker deregistered") + delete(h.namespaces, worker.Namespace()) } + h.roomEnabled = h.roomAvailableLocked() + h.publisherEnabled = h.publisherAvailableLocked() +} + +func (h *AgentHandler) roomAvailableLocked() bool { + for _, w := range h.workers { + if w.JobType() == livekit.JobType_JT_ROOM { + return true + } + } + return false + +} + +func (h *AgentHandler) publisherAvailableLocked() bool { + for _, w := range h.workers { + if w.JobType() == livekit.JobType_JT_PUBLISHER { + return true + } + } + + return false +} + +func (h *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*emptypb.Empty, error) { attempted := make(map[string]bool) for { - select { - case <-ctx.Done(): - return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "request timed out") - default: - s.mu.Lock() - var selected *worker - for _, w := range pool { - if attempted[w.id] { - continue - } - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - if w.activeJobs > 0 { - selected = w - break - } else if selected == nil { - selected = w - } - } - } - s.mu.Unlock() - - if selected == nil { - return nil, psrpc.NewErrorf(psrpc.Unavailable, "no workers available") + h.mu.Lock() + var selected *agent.Worker + var maxLoad float32 + for _, w := range h.workers { + if w.Namespace() != job.Namespace || w.JobType() != job.Type { + continue } - attempted[selected.id] = true - _, err := selected.sigConn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ - Availability: &livekit.AvailabilityRequest{Job: job}, - }}) - if err != nil { - selected.logger.Errorw("failed to send availability request", err, "workerID", selected.id) + _, ok := attempted[w.ID()] + if ok { + continue } - select { - case <-ctx.Done(): - return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "request timed out") - case res := <-ac: - if res.available { - _, err = selected.sigConn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Assignment{ - Assignment: &livekit.JobAssignment{Job: job}, - }}) - if err != nil { - selected.logger.Errorw("failed to assign job", err, "workerID", selected.id) - } else { - selected.mu.Lock() - selected.activeJobs++ - selected.mu.Unlock() - return &emptypb.Empty{}, nil - } + if w.Status() == livekit.WorkerStatus_WS_AVAILABLE { + load := w.Load() + if len(w.RunningJobs()) > 0 && load > maxLoad { + maxLoad = load + selected = w + } else if selected == nil { + selected = w } } + } + h.mu.Unlock() + + if selected == nil { + return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "no workers available") + } + + attempted[selected.ID()] = true + + values := []interface{}{ + "jobID", job.Id, + "namespace", job.Namespace, + "workerID", selected.ID(), + } + if job.Room != nil { + values = append(values, "room", job.Room.Name, "roomID", job.Room.Sid) + } + if job.Participant != nil { + values = append(values, "participant", job.Participant.Identity) + } + logger.Debugw("assigning job", values...) + err := selected.AssignJob(ctx, job) + if err != nil { + if errors.Is(err, agent.ErrWorkerNotAvailable) { + continue // Try another worker + } + return nil, err + } + + return &emptypb.Empty{}, nil } + } -func (s *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) float32 { - s.mu.Lock() - defer s.mu.Unlock() - - var pool map[string]*worker - switch job.Type { - case livekit.JobType_JT_ROOM: - pool = s.roomWorkers - case livekit.JobType_JT_PUBLISHER: - pool = s.publisherWorkers - } +func (h *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) float32 { + h.mu.Lock() + defer h.mu.Unlock() var affinity float32 - for _, w := range pool { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - if w.activeJobs > 0 { - return 1 - } else { + var maxLoad float32 + for _, w := range h.workers { + if w.Namespace() != job.Namespace || w.JobType() != job.Type { + continue + } + + if w.Status() == livekit.WorkerStatus_WS_AVAILABLE { + load := w.Load() + if len(w.RunningJobs()) > 0 && load > maxLoad { + maxLoad = load + affinity = 0.5 + load/2 + } else if affinity == 0 { affinity = 0.5 } } + } return affinity } -func (s *AgentHandler) NumConnections() int { - s.mu.Lock() - defer s.mu.Unlock() +func (h *AgentHandler) CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) (*rpc.CheckEnabledResponse, error) { + h.mu.Lock() + defer h.mu.Unlock() + namespaces := make([]string, 0, len(h.namespaces)) + for ns := range h.namespaces { + namespaces = append(namespaces, ns) + } - return len(s.unregistered) + len(s.roomWorkers) + len(s.publisherWorkers) + return &rpc.CheckEnabledResponse{ + Namespaces: namespaces, + RoomEnabled: h.roomEnabled, + PublisherEnabled: h.publisherEnabled, + }, nil } -func (s *AgentHandler) DrainConnections(interval time.Duration) { +func (h *AgentHandler) NumConnections() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.workers) +} + +func (h *AgentHandler) DrainConnections(interval time.Duration) { // jitter drain start time.Sleep(time.Duration(rand.Int63n(int64(interval)))) t := time.NewTicker(interval) defer t.Stop() - s.mu.Lock() - defer s.mu.Unlock() + h.mu.Lock() + defer h.mu.Unlock() - for conn := range s.unregistered { - _ = conn.Close() - <-t.C - } - for _, w := range s.roomWorkers { - _ = w.conn.Close() - <-t.C - } - for _, w := range s.publisherWorkers { - _ = w.conn.Close() + for _, w := range h.workers { + w.Close() <-t.C } } - -func (s *AgentHandler) roomAvailableLocked() bool { - for _, w := range s.roomWorkers { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - return true - } - } - return false -} - -func (s *AgentHandler) publisherAvailableLocked() bool { - for _, w := range s.publisherWorkers { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - return true - } - } - return false -} diff --git a/pkg/service/auth.go b/pkg/service/auth.go index 5633b2889..c319eceff 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -34,6 +34,11 @@ const ( type grantsKey struct{} +type grantsValue struct { + claims *auth.ClaimGrants + apiKey string +} + var ( ErrPermissionDenied = errors.New("permissions denied") ErrMissingAuthorization = errors.New("invalid authorization header. Must start with " + bearerPrefix) @@ -93,7 +98,10 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, // set grants in context ctx := r.Context() - r = r.WithContext(context.WithValue(ctx, grantsKey{}, grants)) + r = r.WithContext(context.WithValue(ctx, grantsKey{}, &grantsValue{ + claims: grants, + apiKey: v.APIKey(), + })) } next.ServeHTTP(w, r) @@ -101,15 +109,27 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, func GetGrants(ctx context.Context) *auth.ClaimGrants { val := ctx.Value(grantsKey{}) - claims, ok := val.(*auth.ClaimGrants) + v, ok := val.(*grantsValue) if !ok { return nil } - return claims + return v.claims } -func WithGrants(ctx context.Context, grants *auth.ClaimGrants) context.Context { - return context.WithValue(ctx, grantsKey{}, grants) +func GetAPIKey(ctx context.Context) string { + val := ctx.Value(grantsKey{}) + v, ok := val.(*grantsValue) + if !ok { + return "" + } + return v.apiKey +} + +func WithGrants(ctx context.Context, grants *auth.ClaimGrants, apiKey string) context.Context { + return context.WithValue(ctx, grantsKey{}, &grantsValue{ + claims: grants, + apiKey: apiKey, + }) } func SetAuthorizationToken(r *http.Request, token string) { diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 91b437f40..67a573874 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -608,6 +608,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Use a "transaction" to remove the old room association if it changed txf := func(tx *redis.Tx) error { var oldStartedAt int64 + var oldUpdatedAt int64 oldState, err := s.loadIngressState(tx, ingressId) switch err { @@ -615,6 +616,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Ingress state doesn't exist yet case nil: oldStartedAt = oldState.StartedAt + oldUpdatedAt = oldState.UpdatedAt default: return err } @@ -625,6 +627,12 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat return ingress.ErrIngressOutOfDate } + if state.StartedAt == oldStartedAt && state.UpdatedAt < oldUpdatedAt { + // Do not overwrite with an old state in case RPCs were delivered out of order. + // All RPCs come from the same ingress server and should thus be on the same clock. + return nil + } + p.Set(s.ctx, IngressStatePrefix+ingressId, data, 0) return nil diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 04543a59d..54043e6a6 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -16,6 +16,7 @@ package service import ( "context" + "errors" "time" "github.com/livekit/protocol/livekit" @@ -62,7 +63,7 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre // find existing room and update it var created bool rm, internal, err := r.roomStore.LoadRoom(ctx, livekit.RoomName(req.Name), true) - if err == ErrRoomNotFound { + if errors.Is(err, ErrRoomNotFound) { created = true rm = &livekit.Room{ Sid: utils.NewGuid(utils.RoomPrefix), @@ -88,8 +89,13 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre if req.Metadata != "" { rm.Metadata = req.Metadata } - if req.Egress != nil && req.Egress.Tracks != nil { - internal.TrackEgress = req.Egress.Tracks + if req.Egress != nil { + if req.Egress.Participant != nil { + internal.ParticipantEgress = req.Egress.Participant + } + if req.Egress.Tracks != nil { + internal.TrackEgress = req.Egress.Tracks + } } if req.MinPlayoutDelay > 0 || req.MaxPlayoutDelay > 0 { internal.PlayoutDelay = &livekit.PlayoutDelay{ @@ -108,7 +114,7 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre // check if room already assigned existing, err := r.router.GetNodeForRoom(ctx, livekit.RoomName(rm.Name)) - if err != routing.ErrNotFound && err != nil { + if !errors.Is(err, routing.ErrNotFound) && err != nil { return nil, false, err } diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 96dd7036c..3dfa146d5 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -24,6 +24,8 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/maps" + "github.com/livekit/livekit-server/pkg/agent" + sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -47,14 +49,13 @@ const ( roomPurgeSeconds = 24 * 60 * 60 tokenRefreshInterval = 5 * time.Minute tokenDefaultTTL = 10 * time.Minute - iceConfigTTL = 5 * time.Minute ) var affinityEpoch = time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC) -type iceConfigCacheEntry struct { - iceConfig *livekit.ICEConfig - modifiedAt time.Time +type iceConfigCacheKey struct { + roomName livekit.RoomName + participantIdentity livekit.ParticipantIdentity } // RoomManager manages rooms and its interaction with participants. @@ -70,7 +71,7 @@ type RoomManager struct { roomStore ObjectStore telemetry telemetry.TelemetryService clientConfManager clientconfiguration.ClientConfigurationManager - agentClient rtc.AgentClient + agentClient agent.Client egressLauncher rtc.EgressLauncher versionGenerator utils.TimedVersionGenerator turnAuthHandler *TURNAuthHandler @@ -81,7 +82,7 @@ type RoomManager struct { roomServers utils.MultitonService[rpc.RoomTopic] participantServers utils.MultitonService[rpc.ParticipantTopic] - iceConfigCache map[livekit.ParticipantIdentity]*iceConfigCacheEntry + iceConfigCache *sutils.IceConfigCache[iceConfigCacheKey] } func NewLocalRoomManager( @@ -91,7 +92,7 @@ func NewLocalRoomManager( router routing.Router, telemetry telemetry.TelemetryService, clientConfManager clientconfiguration.ClientConfigurationManager, - agentClient rtc.AgentClient, + agentClient agent.Client, egressLauncher rtc.EgressLauncher, versionGenerator utils.TimedVersionGenerator, turnAuthHandler *TURNAuthHandler, @@ -118,14 +119,15 @@ func NewLocalRoomManager( rooms: make(map[livekit.RoomName]*rtc.Room), - iceConfigCache: make(map[livekit.ParticipantIdentity]*iceConfigCacheEntry), + iceConfigCache: sutils.NewIceConfigCache[iceConfigCacheKey](0), serverInfo: &livekit.ServerInfo{ - Edition: livekit.ServerInfo_Standard, - Version: version.Version, - Protocol: types.CurrentProtocol, - Region: conf.Region, - NodeId: currentNode.Id, + Edition: livekit.ServerInfo_Standard, + Version: version.Version, + Protocol: types.CurrentProtocol, + AgentProtocol: agent.CurrentProtocol, + Region: conf.Region, + NodeId: currentNode.Id, }, }, nil } @@ -228,6 +230,8 @@ func (r *RoomManager) Stop() { _ = r.rtcConfig.TCPMuxListener.Close() } } + + r.iceConfigCache.Stop() } // StartSession starts WebRTC session when a new participant is connected, takes place on RTC node @@ -301,14 +305,12 @@ func (r *RoomManager) StartSession( "reason", pi.ReconnectReason, "numParticipants", room.GetParticipantCount(), ) - iceConfig := r.getIceConfig(participant) - if iceConfig == nil { - iceConfig = &livekit.ICEConfig{} - } + iceConfig := r.getIceConfig(roomName, participant) if err = room.ResumeParticipant( participant, requestSource, responseSink, + iceConfig, r.iceServersForParticipant( apiKey, participant, @@ -442,7 +444,7 @@ func (r *RoomManager) StartSession( if err != nil { return err } - iceConfig := r.setIceConfig(participant) + iceConfig := r.setIceConfig(roomName, participant) // join room opts := rtc.ParticipantOptions{ @@ -502,12 +504,7 @@ func (r *RoomManager) StartSession( } }) participant.OnICEConfigChanged(func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) { - r.lock.Lock() - r.iceConfigCache[participant.Identity()] = &iceConfigCacheEntry{ - iceConfig: iceConfig, - modifiedAt: time.Now(), - } - r.lock.Unlock() + r.iceConfigCache.Put(iceConfigCacheKey{roomName, participant.Identity()}, iceConfig) }) go r.rtcSessionWorker(room, participant, requestSource) @@ -618,15 +615,10 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa _ = r.refreshToken(participant) tokenTicker := time.NewTicker(tokenRefreshInterval) defer tokenTicker.Stop() - stateCheckTicker := time.NewTicker(time.Millisecond * 500) - defer stateCheckTicker.Stop() for { select { - case <-stateCheckTicker.C: - // periodic check to ensure participant didn't become disconnected - if participant.IsDisconnected() { - return - } + case <-participant.Disconnected(): + return case <-tokenTicker.C: // refresh token with the first API Key/secret pair if err := r.refreshToken(participant); err != nil { @@ -874,24 +866,14 @@ func (r *RoomManager) refreshToken(participant types.LocalParticipant) error { return nil } -func (r *RoomManager) setIceConfig(participant types.LocalParticipant) *livekit.ICEConfig { - iceConfig := r.getIceConfig(participant) - if iceConfig == nil { - return &livekit.ICEConfig{} - } +func (r *RoomManager) setIceConfig(roomName livekit.RoomName, participant types.LocalParticipant) *livekit.ICEConfig { + iceConfig := r.getIceConfig(roomName, participant) participant.SetICEConfig(iceConfig) return iceConfig } -func (r *RoomManager) getIceConfig(participant types.LocalParticipant) *livekit.ICEConfig { - r.lock.Lock() - defer r.lock.Unlock() - iceConfigCacheEntry, ok := r.iceConfigCache[participant.Identity()] - if !ok || time.Since(iceConfigCacheEntry.modifiedAt) > iceConfigTTL { - delete(r.iceConfigCache, participant.Identity()) - return nil - } - return iceConfigCacheEntry.iceConfig +func (r *RoomManager) getIceConfig(roomName livekit.RoomName, participant types.LocalParticipant) *livekit.ICEConfig { + return r.iceConfigCache.Get(iceConfigCacheKey{roomName, participant.Identity()}) } func (r *RoomManager) getFirstKeyPair() (string, string, error) { diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 3da38851c..319e06cb5 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -22,12 +22,12 @@ import ( "github.com/pkg/errors" "github.com/twitchtv/twirp" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" - "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" ) @@ -39,7 +39,7 @@ type RoomService struct { router routing.MessageRouter roomAllocator RoomAllocator roomStore ServiceStore - agentClient rtc.AgentClient + agentClient agent.Client egressLauncher rtc.EgressLauncher topicFormatter rpc.TopicFormatter roomClient rpc.TypedRoomClient @@ -53,7 +53,7 @@ func NewRoomService( router routing.MessageRouter, roomAllocator RoomAllocator, serviceStore ServiceStore, - agentClient rtc.AgentClient, + agentClient agent.Client, egressLauncher rtc.EgressLauncher, topicFormatter rpc.TopicFormatter, roomClient rpc.TypedRoomClient, @@ -101,15 +101,14 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq defer res.ResponseSource.Close() if created { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: utils.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: rm, - }) - }() + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: rm, + }) if req.Egress != nil && req.Egress.Room != nil { + // ensure room name matches + req.Egress.Room.RoomName = req.Name _, err = s.egressLauncher.StartEgress(ctx, &rpc.StartEgressRequest{ Request: &rpc.StartEgressRequest_RoomComposite{ RoomComposite: req.Egress.Room, @@ -301,13 +300,10 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat } if created { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: utils.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: room, - }) - }() + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: room, + }) } return room, nil diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index 0237117f4..aa10a6399 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -38,7 +38,7 @@ func TestDeleteRoom(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.DeleteRoom(ctx, &livekit.DeleteRoomRequest{ Room: "testroom", }) @@ -52,7 +52,7 @@ func TestMetaDataLimits(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ Room: "testroom", Identity: "123", @@ -82,7 +82,7 @@ func TestMetaDataLimits(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ Room: "testroom", Identity: "123", diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index dba8a1896..efe314c5a 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -32,6 +32,7 @@ import ( "go.uber.org/atomic" "golang.org/x/exp/maps" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/routing/selector" @@ -40,7 +41,6 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" - putil "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" ) @@ -54,7 +54,7 @@ type RTCService struct { isDev bool limits config.LimitConfig parser *uaparser.Parser - agentClient rtc.AgentClient + agentClient agent.Client telemetry telemetry.TelemetryService mu sync.Mutex @@ -67,7 +67,7 @@ func NewRTCService( store ServiceStore, router routing.MessageRouter, currentNode routing.LocalNode, - agentClient rtc.AgentClient, + agentClient agent.Client, telemetry telemetry.TelemetryService, ) *RTCService { s := &RTCService{ @@ -219,14 +219,10 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { var cr connectionResult var initialResponse *livekit.SignalResponse for i := 0; i < 3; i++ { - if err = r.Context().Err(); err != nil { - break - } - connectionTimeout := 3 * time.Second * time.Duration(i+1) ctx := utils.ContextWithAttempt(r.Context(), i) cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) - if err == nil { + if err == nil || errors.Is(err, context.Canceled) { break } if i < 2 { @@ -528,6 +524,13 @@ func (s *RTCService) startConnection( return cr, nil, err } + if created && s.agentClient != nil { + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: cr.Room, + }) + } + // this needs to be started first *before* using router functions on this node cr.StartParticipantSignalResults, err = s.router.StartParticipantSignal(ctx, roomName, pi) if err != nil { @@ -545,16 +548,6 @@ func (s *RTCService) startConnection( return cr, nil, err } - if created && s.agentClient != nil { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: putil.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: cr.Room, - }) - }() - } - return cr, initialResponse, nil } diff --git a/pkg/service/server.go b/pkg/service/server.go index fcb09a648..5dc123fc0 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -125,6 +125,7 @@ func NewLivekitServer(conf *config.Config, mux.HandleFunc("/debug/goroutine", s.debugGoroutines) mux.HandleFunc("/debug/rooms", s.debugInfo) } + mux.Handle(roomServer.PathPrefix(), roomServer) mux.Handle(egressServer.PathPrefix(), egressServer) mux.Handle(ingressServer.PathPrefix(), ingressServer) diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 951837176..9da50679f 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -35,7 +35,7 @@ import ( ) func init() { - prometheus.Init("node", livekit.NodeType_CONTROLLER, "test") + prometheus.Init("node", livekit.NodeType_CONTROLLER) } func TestSignal(t *testing.T) { diff --git a/pkg/service/sip.go b/pkg/service/sip.go index c3989e854..1bcf8f603 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -197,6 +197,7 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, Dtmf: req.Dtmf, + PlayRingtone: req.PlayRingtone, WsUrl: wsUrl, Token: token, } diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 654f26345..3eb4a64c6 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -30,7 +30,7 @@ import ( "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -77,7 +77,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live NewRoomService, NewRTCService, NewAgentService, - rtc.NewAgentClient, + agent.NewAgentClient, getSignalRelayConfig, NewDefaultSignalServer, routing.NewSignalClient, diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 82d232e55..08d493b13 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -8,10 +8,10 @@ package service import ( "fmt" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -60,7 +60,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - agentClient, err := rtc.NewAgentClient(messageBus) + client, err := agent.NewAgentClient(messageBus) if err != nil { return nil, err } @@ -95,7 +95,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - roomService, err := NewRoomService(roomConfig, apiConfig, psrpcConfig, router, roomAllocator, objectStore, agentClient, rtcEgressLauncher, topicFormatter, roomClient, participantClient) + roomService, err := NewRoomService(roomConfig, apiConfig, psrpcConfig, router, roomAllocator, objectStore, client, rtcEgressLauncher, topicFormatter, roomClient, participantClient) if err != nil { return nil, err } @@ -112,15 +112,15 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live return nil, err } sipService := NewSIPService(sipConfig, nodeID, messageBus, sipClient, sipStore, roomService, telemetryService) - rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, agentClient, telemetryService) - agentService, err := NewAgentService(messageBus) + rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, client, telemetryService) + agentService, err := NewAgentService(conf, currentNode, messageBus, keyProvider) if err != nil { return nil, err } clientConfigurationManager := createClientConfiguration() timedVersionGenerator := utils.NewDefaultTimedVersionGenerator() turnAuthHandler := NewTURNAuthHandler(keyProvider) - roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, agentClient, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus) + roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, client, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus) if err != nil { return nil, err } diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index fbf06757c..92f234c5f 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -31,7 +31,8 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/sfu/audio" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/utils" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil" @@ -64,37 +65,40 @@ type ExtPacket struct { KeyFrame bool RawPacket []byte DependencyDescriptor *ExtDependencyDescriptor + AbsCaptureTimeExt *act.AbsCaptureTime } // Buffer contains all packets type Buffer struct { sync.RWMutex - bucket *bucket.Bucket - nacker *nack.NackQueue - maxVideoPkts int - maxAudioPkts int - codecType webrtc.RTPCodecType - payloadType uint8 - extPackets deque.Deque[*ExtPacket] - pPackets []pendingPacket - closeOnce sync.Once - mediaSSRC uint32 - clockRate uint32 - lastReport time.Time - twccExt uint8 - audioLevelExt uint8 - bound bool - closed atomic.Bool - mime string + readCond *sync.Cond + bucket *bucket.Bucket + nacker *nack.NackQueue + maxVideoPkts int + maxAudioPkts int + codecType webrtc.RTPCodecType + payloadType uint8 + extPackets deque.Deque[*ExtPacket] + pPackets []pendingPacket + closeOnce sync.Once + mediaSSRC uint32 + clockRate uint32 + lastReport time.Time + twccExtID uint8 + audioLevelExtID uint8 + bound bool + closed atomic.Bool + mime string snRangeMap *utils.RangeMap[uint64, uint64] latestTSForAudioLevelInitialized bool latestTSForAudioLevel uint32 - twcc *twcc.Responder - audioLevelParams audio.AudioLevelParams - audioLevel *audio.AudioLevel + twcc *twcc.Responder + audioLevelParams audio.AudioLevelParams + audioLevel *audio.AudioLevel + enableAudioLossProxying bool lastPacketRead int @@ -118,7 +122,7 @@ type Buffer struct { logger logger.Logger // dependency descriptor - ddExt uint8 + ddExtID uint8 ddParser *DependencyDescriptorParser paused bool @@ -131,6 +135,8 @@ type Buffer struct { primaryBufferForRTX *Buffer rtxPktBuf []byte + + absCaptureTimeExtID uint8 } // NewBuffer constructs a new Buffer @@ -144,6 +150,7 @@ func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer { pliThrottle: int64(500 * time.Millisecond), logger: l.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU), } + b.readCond = sync.NewCond(&b.RWMutex) b.extPackets.SetMinCapacity(7) return b } @@ -170,7 +177,7 @@ func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) { defer b.Unlock() b.twcc = twcc - b.twccExt = extID + b.twccExtID = extID } func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) { @@ -180,6 +187,13 @@ func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) { b.audioLevelParams = audioLevelParams } +func (b *Buffer) SetAudioLossProxying(enable bool) { + b.Lock() + defer b.Unlock() + + b.enableAudioLossProxying = enable +} + func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability) { b.Lock() defer b.Unlock() @@ -213,18 +227,21 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili for _, ext := range params.HeaderExtensions { switch ext.URI { case dd.ExtensionURI: - b.ddExt = uint8(ext.ID) + b.ddExtID = uint8(ext.ID) frc := NewFrameRateCalculatorDD(b.clockRate, b.logger) for i := range b.frameRateCalculator { b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) } - b.ddParser = NewDependencyDescriptorParser(b.ddExt, b.logger, func(spatial, temporal int32) { + b.ddParser = NewDependencyDescriptorParser(b.ddExtID, b.logger, func(spatial, temporal int32) { frc.SetMaxLayer(spatial, temporal) }) case sdp.AudioLevelURI: - b.audioLevelExt = uint8(ext.ID) + b.audioLevelExtID = uint8(ext.ID) b.audioLevel = audio.NewAudioLevel(b.audioLevelParams) + + case act.AbsCaptureTimeURI: + b.absCaptureTimeExtID = uint8(ext.ID) } } @@ -291,9 +308,10 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } - if b.twcc != nil && b.twccExt != 0 && !b.closed.Load() { - if ext := rtpPacket.GetExtension(b.twccExt); ext != nil { - b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), time.Now().UnixNano(), rtpPacket.Marker) + now := time.Now() + if b.twcc != nil && b.twccExtID != 0 && !b.closed.Load() { + if ext := rtpPacket.GetExtension(b.twccExtID); ext != nil { + b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now.UnixNano(), rtpPacket.Marker) } } @@ -306,7 +324,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } - pb.writeRTX(&rtpPacket) + pb.writeRTX(&rtpPacket, now) return } @@ -315,7 +333,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { copy(packet, pkt) b.pPackets = append(b.pPackets, pendingPacket{ packet: packet, - arrivalTime: time.Now(), + arrivalTime: now, }) b.Unlock() return @@ -324,6 +342,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { b.payloadType = rtpPacket.PayloadType b.calc(pkt, &rtpPacket, time.Now(), false) b.Unlock() + b.readCond.Signal() return } @@ -342,11 +361,11 @@ func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) { if rtpPacket.Padding && len(rtpPacket.Payload) == 0 { continue } - primaryBuffer.writeRTX(&rtpPacket) + primaryBuffer.writeRTX(&rtpPacket, pp.arrivalTime) } } -func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) { +func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime time.Time) (n int, err error) { b.Lock() defer b.Unlock() if !b.bound { @@ -357,19 +376,18 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) { b.rtxPktBuf = make([]byte, bucket.MaxPktSize) } - videoPkt := *rtxPkt - videoPkt.PayloadType = b.payloadType - videoPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) - videoPkt.SSRC = b.mediaSSRC - videoPkt.Payload = rtxPkt.Payload[2:] - n, err = videoPkt.MarshalTo(b.rtxPktBuf) - + repairedPkt := *rtxPkt + repairedPkt.PayloadType = b.payloadType + repairedPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) + repairedPkt.SSRC = b.mediaSSRC + repairedPkt.Payload = rtxPkt.Payload[2:] + n, err = repairedPkt.MarshalTo(b.rtxPktBuf) if err != nil { - b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", videoPkt.SequenceNumber) + b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", repairedPkt.SequenceNumber) return } - b.calc(b.rtxPktBuf[:n], &videoPkt, time.Now(), true) + b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, true) return } @@ -398,24 +416,23 @@ func (b *Buffer) Read(buff []byte) (n int, err error) { } func (b *Buffer) ReadExtended(buf []byte) (*ExtPacket, error) { + b.Lock() for { if b.closed.Load() { + b.Unlock() return nil, io.EOF } - b.Lock() if b.extPackets.Len() > 0 { ep := b.extPackets.PopFront() ep = b.patchExtPacket(ep, buf) if ep == nil { - b.Unlock() continue } b.Unlock() return ep, nil } - b.Unlock() - time.Sleep(10 * time.Millisecond) + b.readCond.Wait() } } @@ -437,6 +454,7 @@ func (b *Buffer) Close() error { } } + b.readCond.Broadcast() if b.onClose != nil { b.onClose() } @@ -662,13 +680,12 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) RTPFlow } func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time, isRTX bool) { - - if b.audioLevelExt != 0 && !isRTX { + if b.audioLevelExtID != 0 && !isRTX { if !b.latestTSForAudioLevelInitialized { b.latestTSForAudioLevelInitialized = true b.latestTSForAudioLevel = p.Timestamp } - if e := p.GetExtension(b.audioLevelExt); e != nil { + if e := p.GetExtension(b.audioLevelExtID); e != nil { ext := rtp.AudioLevelExtension{} if err := ext.Unmarshal(e); err == nil { if (p.Timestamp - b.latestTSForAudioLevel) < (1 << 31) { @@ -728,6 +745,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid } ep.Payload = vp8Packet + case "video/vp9": if ep.DependencyDescriptor == nil { var vp9Packet codecs.VP9Packet @@ -743,8 +761,10 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow ep.Payload = vp9Packet } ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload) + case "video/h264": ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload) + case "video/av1": ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload) } @@ -755,6 +775,15 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow } } + if b.absCaptureTimeExtID != 0 { + extData := rtpPacket.GetExtension(b.absCaptureTimeExtID) + + var actExt act.AbsCaptureTime + if err := actExt.Unmarshal(extData); err == nil { + ep.AbsCaptureTimeExt = &actExt + } + } + return ep } @@ -799,14 +828,16 @@ func (b *Buffer) mayGrowBucket() { return } oldCap := cap - deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId) - if deltaInfo != nil && deltaInfo.Duration > 500*time.Millisecond { - pps := int(time.Duration(deltaInfo.Packets) * time.Second / deltaInfo.Duration) - for pps > cap && cap < maxPkts { - cap = b.bucket.Grow() - } - if cap > oldCap { - b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) + if deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId); deltaInfo != nil { + duration := deltaInfo.EndTime.Sub(deltaInfo.StartTime) + if duration > 500*time.Millisecond { + pps := int(time.Duration(deltaInfo.Packets) * time.Second / duration) + for pps > cap && cap < maxPkts { + cap = b.bucket.Grow() + } + if cap > oldCap { + b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) + } } } } @@ -828,7 +859,12 @@ func (b *Buffer) buildReceptionReport() *rtcp.ReceptionReport { return nil } - return b.rtpStats.GetRtcpReceptionReport(b.mediaSSRC, b.lastFractionLostToReport, b.rrSnapshotId) + proxyLoss := b.lastFractionLostToReport + if b.codecType == webrtc.RTPCodecTypeAudio && !b.enableAudioLossProxying { + proxyLoss = 0 + } + + return b.rtpStats.GetRtcpReceptionReport(b.mediaSSRC, proxyLoss, b.rrSnapshotId) } func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { @@ -949,6 +985,17 @@ func (b *Buffer) GetDeltaStats() *StreamStatsWithLayers { } } +func (b *Buffer) GetLastSenderReportTime() time.Time { + b.RLock() + defer b.RUnlock() + + if b.rtpStats == nil { + return time.Time{} + } + + return b.rtpStats.LastSenderReportTime() +} + func (b *Buffer) GetAudioLevel() (float64, bool) { b.RLock() defer b.RUnlock() diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 2125ca907..d27ef3fe2 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -208,9 +208,12 @@ func TestNewBuffer(t *testing.T) { func TestFractionLostReport(t *testing.T) { buff := NewBuffer(123, 1, 1) require.NotNil(t, buff) - buff.codecType = webrtc.RTPCodecTypeVideo + var wg sync.WaitGroup + + // with loss proxying wg.Add(1) + buff.SetAudioLossProxying(true) buff.SetLastFractionLostReport(55) buff.OnRtcpFeedback(func(fb []rtcp.Packet) { for _, pkt := range fb { @@ -241,6 +244,38 @@ func TestFractionLostReport(t *testing.T) { require.NoError(t, err) } wg.Wait() + + wg.Add(1) + buff.SetAudioLossProxying(false) + buff.OnRtcpFeedback(func(fb []rtcp.Packet) { + for _, pkt := range fb { + switch p := pkt.(type) { + case *rtcp.ReceiverReport: + for _, v := range p.Reports { + require.EqualValues(t, 0, v.FractionLost) + } + wg.Done() + } + } + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{opusCodec}, + }, opusCodec.RTPCodecCapability) + for i := 0; i < 15; i++ { + pkt := rtp.Packet{ + Header: rtp.Header{SequenceNumber: uint16(i), Timestamp: uint32(i)}, + Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}, + } + b, err := pkt.Marshal() + require.NoError(t, err) + if i == 1 { + time.Sleep(1 * time.Second) + } + _, err = buff.Write(b) + require.NoError(t, err) + } + wg.Wait() } func BenchmarkMemcpu(b *testing.B) { diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index da79c301b..1effe38b0 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -21,7 +21,7 @@ import ( "github.com/pion/rtp" "go.uber.org/atomic" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" @@ -90,7 +90,7 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr } _, err := ext.Unmarshal(ddBuf) if err != nil { - if err != dd.ErrDDReaderNoStructure { + if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex { r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf)) } return nil, videoLayer, err diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 4a85d2808..39dd93df3 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -20,7 +20,7 @@ import ( "github.com/pion/rtp" "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) @@ -46,9 +46,9 @@ func (f *testFrameInfo) toDD() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, DependencyDescriptor: &ExtDependencyDescriptor{ - Descriptor: &dependencydescriptor.DependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ FrameNumber: f.framenumber, - FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ + FrameDependencies: &dd.FrameDependencyTemplate{ FrameDiffs: f.frameDiff, }, }, diff --git a/pkg/sfu/buffer/frameintegrity.go b/pkg/sfu/buffer/frameintegrity.go index 536f793cf..935263b7d 100644 --- a/pkg/sfu/buffer/frameintegrity.go +++ b/pkg/sfu/buffer/frameintegrity.go @@ -15,7 +15,7 @@ package buffer import ( - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type FrameEntity struct { diff --git a/pkg/sfu/buffer/frameintegrity_test.go b/pkg/sfu/buffer/frameintegrity_test.go index 144a9f0f2..2815cf5e9 100644 --- a/pkg/sfu/buffer/frameintegrity_test.go +++ b/pkg/sfu/buffer/frameintegrity_test.go @@ -20,47 +20,47 @@ import ( "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) func TestFrameIntegrityChecker(t *testing.T) { fc := NewFrameIntegrityChecker(100, 1000) // first frame out of order - fc.AddPacket(10, 10, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(10, 10, &dd.DependencyDescriptor{}) require.False(t, fc.FrameIntegrity(10)) - fc.AddPacket(9, 10, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true}) require.False(t, fc.FrameIntegrity(10)) - fc.AddPacket(11, 10, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(10)) // single packet frame - fc.AddPacket(100, 100, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(100)) require.False(t, fc.FrameIntegrity(101)) require.False(t, fc.FrameIntegrity(99)) // frame too old than first frame - fc.AddPacket(99, 99, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) // multiple packet frame, out of order - fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{}) require.False(t, fc.FrameIntegrity(2001)) require.False(t, fc.FrameIntegrity(1999)) // out of frame count(100) require.False(t, fc.FrameIntegrity(100)) require.False(t, fc.FrameIntegrity(1900)) - fc.AddPacket(2000, 2001, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true}) require.False(t, fc.FrameIntegrity(2001)) - fc.AddPacket(2002, 2001, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(2001)) // duplicate packet - fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{}) require.True(t, fc.FrameIntegrity(2001)) // frame too old - fc.AddPacket(900, 1900, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) require.False(t, fc.FrameIntegrity(1900)) for frame := uint64(2002); frame < 2102; frame++ { @@ -75,7 +75,7 @@ func TestFrameIntegrityChecker(t *testing.T) { rand.Seed(int64(frame)) rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] }) for i, f := range frames { - fc.AddPacket(f, frame, &dependencydescriptor.DependencyDescriptor{ + fc.AddPacket(f, frame, &dd.DependencyDescriptor{ FirstPacketInFrame: f == firstFrame, LastPacketInFrame: f == lastFrame, }) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 11c9517c6..04028465d 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -55,7 +55,7 @@ func RTPDriftToString(r *livekit.RTPDrift) string { type RTPDeltaInfo struct { StartTime time.Time - Duration time.Duration + EndTime time.Time Packets uint32 Bytes uint64 HeaderBytes uint64 @@ -114,6 +114,11 @@ type RTCPSenderReportData struct { RTPTimestampExt uint64 NTPTimestamp mediatransportutil.NtpTime At time.Time + AtAdjusted time.Time +} + +func (r *RTCPSenderReportData) PropagationDelay() time.Duration { + return r.AtAdjusted.Sub(r.NTPTimestamp.Time()) } func (r *RTCPSenderReportData) ToString() string { @@ -121,7 +126,13 @@ func (r *RTCPSenderReportData) ToString() string { return "" } - return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s", r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, r.At.String()) + return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s, atAdj: %s", + r.NTPTimestamp.Time().String(), + r.RTPTimestamp, + r.RTPTimestampExt, + r.At.String(), + r.AtAdjusted.String(), + ) } func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { @@ -133,6 +144,7 @@ func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddUint32("RTPTimestamp", r.RTPTimestamp) e.AddUint64("RTPTimestampExt", r.RTPTimestampExt) e.AddTime("At", r.At) + e.AddTime("AtAdjusted", r.AtAdjusted) return nil } @@ -495,7 +507,7 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *RTCPSenderReportData, // abnormal delay (maybe due to pacing or maybe due to queuing // in some network element along the way), push back first time // to an earlier instance. - timeSinceReceive := time.Since(srData.At) + timeSinceReceive := time.Since(srData.AtAdjusted) extNowTS := srData.RTPTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9) samplesDiff := int64(extNowTS - extStartTS) if samplesDiff < 0 { @@ -570,7 +582,7 @@ func (r *rtpStatsBase) deltaInfo(snapshotID uint32, extStartSN uint64, extHighes if packetsExpected == 0 { return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, } } @@ -590,7 +602,7 @@ func (r *rtpStatsBase) deltaInfo(snapshotID uint32, extStartSN uint64, extHighes return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: uint32(packetsExpected), Bytes: now.bytes - then.bytes, HeaderBytes: now.headerBytes - then.headerBytes, @@ -868,8 +880,8 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l rtpClockTicks := r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt elapsed := r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) - driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) if elapsed.Seconds() > 0.0 { + driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) ntpReportDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(r.srFirst.NTPTimestamp.Time()), EndTime: timestamppb.New(r.srNewest.NTPTimestamp.Time()), @@ -883,12 +895,12 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l } } - elapsed = r.srNewest.At.Sub(r.srFirst.At) - driftSamples = int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + elapsed = r.srNewest.AtAdjusted.Sub(r.srFirst.AtAdjusted) if elapsed.Seconds() > 0.0 { + driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) rebasedReportDrift = &livekit.RTPDrift{ - StartTime: timestamppb.New(r.srFirst.At), - EndTime: timestamppb.New(r.srNewest.At), + StartTime: timestamppb.New(r.srFirst.AtAdjusted), + EndTime: timestamppb.New(r.srNewest.AtAdjusted), Duration: elapsed.Seconds(), StartTimestamp: r.srFirst.RTPTimestampExt, EndTimestamp: r.srNewest.RTPTimestampExt, @@ -995,9 +1007,8 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { startTime = deltaInfo.StartTime } - endedAt := deltaInfo.StartTime.Add(deltaInfo.Duration) - if endTime.IsZero() || endTime.Before(endedAt) { - endTime = endedAt + if endTime.IsZero() || endTime.Before(deltaInfo.EndTime) { + endTime = deltaInfo.EndTime } packets += deltaInfo.Packets @@ -1036,7 +1047,7 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: packets, Bytes: bytes, HeaderBytes: headerBytes, diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 0a1b61a28..bd1b16341 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -38,22 +38,27 @@ const ( // lower value as that could be the real propagation delay. If it rises, adapt slowly // as it might be a temporary change or slow drift. See below for handling of high deltas // which could be a result of a path change. - cPropagationDelayFallFactor = float64(0.95) - cPropagationDelayRiseFactor = float64(0.05) + cPropagationDelayFallFactor = float64(0.9) + cPropagationDelayRiseFactor = float64(0.1) - // do not adapt to small OR large (outlier) changes - cPropagationDelayDeltaThresholdMin = 5 * time.Millisecond - cPropagationDelayDeltaThresholdMaxFactor = 2 + cPropagationDelaySpikeAdaptationFactor = float64(0.5) // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. // Reset at whichever of the below happens later. + // 1. 10 seconds of persistent high delta. + // 2. at least 2 consecutive reports with high delta. // - // A long term version of delta of propagation delay is maintained and delta propagation delay exceeding - // a factor of the long term version is considered a sharp increase. That will trigger the start of the + // A long term estimate of delta of propagation delay is maintained and delta propagation delay exceeding + // a factor of the long term estimate is considered a sharp increase. That will trigger the start of the // path change condition and if it persists, propagation delay will be reset. - cPropagationDelayDeltaAdaptationFactor = float64(0.05) - cPropagationDelayDeltaHighResetNumReports = 3 - cPropagationDelayDeltaHighResetWait = 10 * time.Second + cPropagationDelayDeltaThresholdMin = 10 * time.Millisecond + cPropagationDelayDeltaThresholdMaxFactor = 2 + cPropagationDelayDeltaHighResetNumReports = 2 + cPropagationDelayDeltaHighResetWait = 10 * time.Second + cPropagationDelayDeltaLongTermAdaptationThreshold = 50 * time.Millisecond + + // number of seconds the current report RTP timestamp can be off from expected RTP timestamp + cReportSlack = float64(60.0) ) type RTPFlowState struct { @@ -83,6 +88,7 @@ type RTPStatsReceiver struct { longTermDeltaPropagationDelay time.Duration propagationDelayDeltaHighCount int propagationDelayDeltaHighStartTime time.Time + propagationDelaySpike time.Duration clockSkewCount int outOfOrderSsenderReportCount int @@ -292,14 +298,38 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) tsCycles := uint64(0) if r.srNewest != nil { - tsCycles = r.srNewest.RTPTimestampExt & 0xFFFF_FFFF_0000_0000 - if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { - tsCycles += (1 << 32) - } + // use time since last sender report to ensure long gaps where the time stamp might + // jump more than half the range + timeSinceLastReport := srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + expectedRTPTimestampExt := r.srNewest.RTPTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9) + lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate)) + ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate)) + isInRange := (srData.RTPTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RTPTimestamp < (1 << 31)) + if isInRange { + lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000 + ubTSCycles := ubound & 0xFFFF_FFFF_0000_0000 + if lbTSCycles == ubTSCycles { + tsCycles = lbTSCycles + } else { + if srData.RTPTimestamp < (1 << 31) { + // rolled over + tsCycles = ubTSCycles + } else { + tsCycles = lbTSCycles + } + } + } else { + // ideally this method should not be required, but there are clients + // negotiating one clock rate, but actually send media at a different rate. + tsCycles = r.srNewest.RTPTimestampExt & 0xFFFF_FFFF_0000_0000 + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { + tsCycles += (1 << 32) + } - if tsCycles >= (1 << 32) { - if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) >= (1<<31) && srData.RTPTimestamp > r.srNewest.RTPTimestamp { - tsCycles -= (1 << 32) + if tsCycles >= (1 << 32) { + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) >= (1<<31) && srData.RTPTimestamp > r.srNewest.RTPTimestamp { + tsCycles -= (1 << 32) + } } } } @@ -365,16 +395,23 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) "receivedDeltaPropagationDelay", deltaPropagationDelay.String(), "deltaHighCount", r.propagationDelayDeltaHighCount, "sinceDeltaHighStart", time.Since(r.propagationDelayDeltaHighStartTime).String(), + "propagationDelaySpike", r.propagationDelaySpike.String(), "first", r.srFirst, "last", r.srNewest, "current", &srDataCopy, } } - initPropagationDelay := func(pd time.Duration) { - r.propagationDelay = pd - r.longTermDeltaPropagationDelay = 0 + resetDelta := func() { r.propagationDelayDeltaHighCount = 0 r.propagationDelayDeltaHighStartTime = time.Time{} + r.propagationDelaySpike = 0 + } + initPropagationDelay := func(pd time.Duration) { + r.propagationDelay = pd + + r.longTermDeltaPropagationDelay = 0 + + resetDelta() } ntpTime := srDataCopy.NTPTimestamp.Time() @@ -385,47 +422,52 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("initializing propagation delay", getPropagationFields()...) } else { deltaPropagationDelay = propagationDelay - r.propagationDelay - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { // ignore small changes - if r.longTermDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { - r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE + if deltaPropagationDelay > cPropagationDelayDeltaThresholdMin { // ignore small changes for path change consideration + if r.longTermDeltaPropagationDelay != 0 && + deltaPropagationDelay > 0 && + deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { + r.logger.Debugw("sharp increase in propagation delay", getPropagationFields()...) r.propagationDelayDeltaHighCount++ if r.propagationDelayDeltaHighStartTime.IsZero() { r.propagationDelayDeltaHighStartTime = time.Now() } + if r.propagationDelaySpike == 0 { + r.propagationDelaySpike = propagationDelay + } else { + r.propagationDelaySpike += time.Duration(cPropagationDelaySpikeAdaptationFactor * float64(propagationDelay-r.propagationDelaySpike)) + } if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) - initPropagationDelay(propagationDelay) + initPropagationDelay(r.propagationDelaySpike) } } else { - r.propagationDelayDeltaHighCount = 0 - r.propagationDelayDeltaHighStartTime = time.Time{} - - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { - factor := cPropagationDelayFallFactor - if propagationDelay > r.propagationDelay { - factor = cPropagationDelayRiseFactor - } - fields := append( - getPropagationFields(), - "adjustedPropagationDelay", r.propagationDelay+time.Duration(factor*float64(propagationDelay-r.propagationDelay)), - ) // TODO-REMOVE - r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE - r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) - } + resetDelta() } } else { - r.propagationDelayDeltaHighCount = 0 - r.propagationDelayDeltaHighStartTime = time.Time{} + resetDelta() + + factor := cPropagationDelayFallFactor + if propagationDelay > r.propagationDelay { + factor = cPropagationDelayRiseFactor + } + r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } + if r.longTermDeltaPropagationDelay == 0 { r.longTermDeltaPropagationDelay = deltaPropagationDelay } else { - r.longTermDeltaPropagationDelay += time.Duration(cPropagationDelayDeltaAdaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + if deltaPropagationDelay < cPropagationDelayDeltaLongTermAdaptationThreshold { + // do not adapt to large +ve spikes, can happen when channel is congested and reports are delivered very late + // if the spike is in fact a path change, it will persist and handled by path change detection above + sinceLastReport := srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + adaptationFactor := min(1.0, float64(sinceLastReport)/float64(cPropagationDelayDeltaHighResetWait)) + r.longTermDeltaPropagationDelay += time.Duration(adaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + } } } // adjust receive time to estimated propagation delay - srDataCopy.At = ntpTime.Add(r.propagationDelay) + srDataCopy.AtAdjusted = ntpTime.Add(r.propagationDelay) r.srNewest = &srDataCopy r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()) @@ -443,6 +485,17 @@ func (r *RTPStatsReceiver) GetRtcpSenderReportData() *RTCPSenderReportData { return &srNewestCopy } +func (r *RTPStatsReceiver) LastSenderReportTime() time.Time { + r.lock.RLock() + defer r.lock.RUnlock() + + if r.srNewest != nil { + return r.srNewest.At + } + + return time.Time{} +} + func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotID uint32) *rtcp.ReceptionReport { r.lock.Lock() defer r.lock.Unlock() diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 4fe39c0c5..be3f12990 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -306,6 +306,8 @@ func (r *RTPStatsSender) Update( "hdrSize", hdrSize, "payloadSize", payloadSize, "paddingSize", paddingSize, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } @@ -372,6 +374,8 @@ func (r *RTPStatsSender) Update( "hdrSize", hdrSize, "payloadSize", payloadSize, "paddingSize", paddingSize, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } @@ -466,6 +470,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "lastRR", r.lastRR, "sinceLastRR", time.Since(r.lastRRTime).String(), "receivedRR", rr, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) return } @@ -545,6 +551,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "packetsInInterval", extReceivedRRSN-s.extLastRRSN, "extHighestSNFromRR", r.extHighestSNFromRR, "packetsLostFromRR", r.packetsLostFromRR, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) continue } @@ -580,6 +588,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "extHighestSNFromRR", r.extHighestSNFromRR, "packetsLostFromRR", r.packetsLostFromRR, "count", r.metadataCacheOverflowCount, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } r.metadataCacheOverflowCount++ @@ -633,8 +643,8 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS return nil } - timeSincePublisherSR := time.Since(publisherSRData.At) - now := publisherSRData.At.Add(timeSincePublisherSR) + timeSincePublisherSR := time.Since(publisherSRData.AtAdjusted) + now := publisherSRData.AtAdjusted.Add(timeSincePublisherSR) nowNTP := mediatransportutil.ToNtpTime(now) nowRTPExt := publisherSRData.RTPTimestampExt - tsOffset + uint64(timeSincePublisherSR.Nanoseconds()*int64(r.params.ClockRate)/1e9) @@ -643,6 +653,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS RTPTimestamp: uint32(nowRTPExt), RTPTimestampExt: nowRTPExt, At: now, + AtAdjusted: now, } getFields := func() []interface{} { @@ -668,7 +679,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RTPTimestampExt windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds() if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) { - if r.clockSkewCount%10 == 0 { + if r.clockSkewCount%100 == 0 { fields := append( getFields(), "timeSinceLastReport", timeSinceLastReport.String(), @@ -735,6 +746,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo "startTime", startTime.String(), "endTime", endTime.String(), "duration", endTime.Sub(startTime).String(), + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) return nil } @@ -775,7 +788,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: packetsExpected - uint32(now.packetsPadding-then.packetsPadding), Bytes: now.bytes - then.bytes, HeaderBytes: now.headerBytes - then.headerBytes, diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go index 94543ea38..8f2253850 100644 --- a/pkg/sfu/codecmunger/codecmunger.go +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -33,7 +33,7 @@ type CodecMunger interface { SetLast(extPkt *buffer.ExtPacket) UpdateOffsets(extPkt *buffer.ExtPacket) - UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) + UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) - UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) + UpdateAndGetPadding(newPicture bool) ([]byte, error) } diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index c845b4ce1..9d5757011 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -45,10 +45,10 @@ func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) { } -func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) { - return 0, 0, nil +func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) { + return 0, nil, nil } -func (n *Null) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { - return 0, nil +func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) { + return nil, nil } diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index c26ea0e3c..97c37bc4c 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -158,10 +158,10 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } -func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32, outputHeader []byte) (int, int, error) { +func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { - return 0, 0, ErrNotVP8 + return 0, nil, ErrNotVP8 } extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M) @@ -170,7 +170,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap if snOutOfOrder { pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId) if !ok { - return 0, 0, ErrOutOfOrderVP8PictureIdCacheMiss + return 0, nil, ErrOutOfOrderVP8PictureIdCacheMiss } // the out-of-order picture id cannot be deleted from the cache @@ -195,11 +195,11 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - n, err := vp8Packet.MarshalTo(outputHeader) + vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { - return 0, 0, err + return 0, nil, err } - return vp8.HeaderSize, n, nil + return vp8.HeaderSize, vp8HeaderBytes, nil } prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() @@ -240,7 +240,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap // if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined // which layer the missing packets belong to. A layer could have multiple packets. So, keep track - // of pictures that are forwarded even though they will be filterd out based on temporal layer + // of pictures that are forwarded even though they will be filtered out based on temporal layer // requirements. That allows forwarding of the complete picture. if vp8.T && vp8.TID > uint8(maxTemporalLayer) { v.exemptedPictureIds.Set(extPictureId, true) @@ -267,7 +267,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap v.pictureIdOffset += 1 } - return 0, 0, ErrFilteredVP8TemporalLayer + return 0, nil, ErrFilteredVP8TemporalLayer } } } @@ -302,14 +302,14 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - n, err := vp8Packet.MarshalTo(outputHeader) + vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { - return 0, 0, err + return 0, nil, err } - return vp8.HeaderSize, n, nil + return vp8.HeaderSize, vp8HeaderBytes, nil } -func (v *VP8) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { +func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { offset := 0 if newPicture { offset = 1 @@ -367,7 +367,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, er IsKeyFrame: true, HeaderSize: headerSize, } - return vp8Packet.MarshalTo(outputHeader) + return vp8Packet.Marshal() } // for testing only diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go index 155c7249c..31d33dd65 100644 --- a/pkg/sfu/codecmunger/vp8_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -166,7 +166,6 @@ func TestUpdateOffsets(t *testing.T) { func TestOutOfOrderPictureId(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -190,17 +189,17 @@ func TestOutOfOrderPictureId(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - v.UpdateAndGet(extPkt, false, false, 2, buf) + v.UpdateAndGet(extPkt, false, false, 2) // out-of-order sequence number not in the missing picture id cache vp8.PictureID = 13466 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err := v.UpdateAndGet(extPkt, true, false, 2, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, true, false, 2) require.Error(t, err) require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) // create a hole in picture id vp8.PictureID = 13469 @@ -223,10 +222,10 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) // all three, the last, the current and the in-between should have been added to missing picture id cache value, ok := v.PictureIdOffset(13467) @@ -262,15 +261,14 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, true, false, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, true, false, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) } func TestTemporalLayerFiltering(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -296,11 +294,11 @@ func TestTemporalLayerFiltering(t *testing.T) { v.SetLast(extPkt) // translate - nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ := v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -310,11 +308,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23334 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -324,11 +322,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23337 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -336,7 +334,6 @@ func TestTemporalLayerFiltering(t *testing.T) { func TestGapInSequenceNumberSamePicture(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 65533, @@ -379,10 +376,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 2, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) // telling there is a gap in sequence number will add pictures to missing picture cache expectedVP8 = &buffer.VP8{ @@ -402,10 +399,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) value, ok := v.PictureIdOffset(13467) require.True(t, ok) @@ -414,7 +411,6 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { func TestUpdateAndGetPadding(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -442,7 +438,7 @@ func TestUpdateAndGetPadding(t *testing.T) { v.SetLast(extPkt) // getting padding with repeat of last picture - n, err := v.UpdateAndGetPadding(false, buf) + buf, err := v.UpdateAndGetPadding(false) require.NoError(t, err) expectedVP8 := buffer.VP8{ FirstByte: 16, @@ -461,10 +457,10 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) // getting padding with new picture - n, err = v.UpdateAndGetPadding(true, buf) + buf, err = v.UpdateAndGetPadding(true) require.NoError(t, err) expectedVP8 = buffer.VP8{ FirstByte: 16, @@ -483,7 +479,7 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) } func TestVP8PictureIdWrapHandler(t *testing.T) { diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 16ac76d1d..f124edb5c 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -22,6 +22,7 @@ import ( "github.com/frostbyte73/core" "github.com/pion/webrtc/v3" "go.uber.org/atomic" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -36,6 +37,7 @@ const ( type ConnectionStatsReceiverProvider interface { GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers + GetLastSenderReportTime() time.Time } type ConnectionStatsSenderProvider interface { @@ -45,14 +47,15 @@ type ConnectionStatsSenderProvider interface { } type ConnectionStatsParams struct { - UpdateInterval time.Duration - MimeType string - IsFECEnabled bool - IncludeRTT bool - IncludeJitter bool - ReceiverProvider ConnectionStatsReceiverProvider - SenderProvider ConnectionStatsSenderProvider - Logger logger.Logger + UpdateInterval time.Duration + MimeType string + IsFECEnabled bool + IncludeRTT bool + IncludeJitter bool + EnableBitrateScore bool + ReceiverProvider ConnectionStatsReceiverProvider + SenderProvider ConnectionStatsSenderProvider + Logger logger.Logger } type ConnectionStats struct { @@ -76,10 +79,11 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { return &ConnectionStats{ params: params, scorer: newQualityScorer(qualityScorerParams{ - PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? - IncludeRTT: params.IncludeRTT, - IncludeJitter: params.IncludeJitter, - Logger: params.Logger, + PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? + IncludeRTT: params.IncludeRTT, + IncludeJitter: params.IncludeJitter, + EnableBitrateScore: params.EnableBitrateScore, + Logger: params.Logger, }), } } @@ -199,11 +203,11 @@ func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQual return cs.scorer.GetMOSAndQuality() } -func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at time.Time) float32 { +func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, lastRTCPAt time.Time, at time.Time) float32 { var stat windowStat if agg != nil { stat.startedAt = agg.StartTime - stat.duration = agg.Duration + stat.duration = agg.EndTime.Sub(agg.StartTime) stat.packetsExpected = agg.Packets + agg.PacketsPadding stat.packetsLost = agg.PacketsLost stat.packetsMissing = agg.PacketsMissing @@ -211,6 +215,8 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size stat.rttMax = agg.RttMax stat.jitterMax = agg.JitterMax + + stat.lastRTCPAt = lastRTCPAt } if at.IsZero() { cs.scorer.Update(&stat) @@ -243,7 +249,7 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, } if time.Since(marker) > noReceiverReportTooLongThreshold { // have not received receiver report for a long time when streaming, run with nil stat - return cs.updateScoreWithAggregate(nil, at), nil + return cs.updateScoreWithAggregate(nil, time.Time{}, at), nil } // wait for receiver report, return current score @@ -261,10 +267,9 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, } if streamingStartedAt.After(agg.StartTime) { - agg.Duration = agg.StartTime.Add(agg.Duration).Sub(streamingStartedAt) agg.StartTime = streamingStartedAt } - return cs.updateScoreWithAggregate(agg, at), streams + return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams } func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { @@ -288,7 +293,7 @@ func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buf deltaInfoList = append(deltaInfoList, s.RTPStats) } agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) - return cs.updateScoreWithAggregate(agg, at), streams + return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams } func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time { @@ -377,7 +382,7 @@ func (cs *ConnectionStats) updateStatsWorker() { // For audio: // // o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight -// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight +// o RED with two packet redundancy can absorb one out of every two packets lost, so packet loss is not as detrimental and therefore lower weight // // For video: // @@ -394,10 +399,10 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { } case strings.EqualFold(mimeType, "audio/red"): - // 10%: fall to GOOD, 30.0%: fall to POOR - plw = 2.0 + // 5%: fall to GOOD, 15.0%: fall to POOR + plw = 4.0 if isFecEnabled { - // 15%: fall to GOOD, 45.0%: fall to POOR + // 7.5%: fall to GOOD, 22.5%: fall to POOR plw /= 1.5 } @@ -426,6 +431,8 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An packetsLost -= deltaStats.PacketsMissing } return &livekit.AnalyticsStream{ + StartTime: timestamppb.New(deltaStats.StartTime), + EndTime: timestamppb.New(deltaStats.EndTime), Ssrc: ssrc, PrimaryPackets: deltaStats.Packets, PrimaryBytes: deltaStats.Bytes, diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 7b07ba8bc..78e217130 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -26,27 +26,11 @@ import ( "github.com/livekit/protocol/logger" ) -func newConnectionStats( - mimeType string, - isFECEnabled bool, - includeRTT bool, - includeJitter bool, - receiverProvider ConnectionStatsReceiverProvider, -) *ConnectionStats { - return NewConnectionStats(ConnectionStatsParams{ - MimeType: mimeType, - IsFECEnabled: isFECEnabled, - IncludeRTT: includeRTT, - IncludeJitter: includeJitter, - ReceiverProvider: receiverProvider, - Logger: logger.GetLogger(), - }) -} - // ----------------------------------------------- type testReceiverProvider struct { - streams map[uint32]*buffer.StreamStatsWithLayers + streams map[uint32]*buffer.StreamStatsWithLayers + lastSenderReportTime time.Time } func newTestReceiverProvider() *testReceiverProvider { @@ -61,12 +45,28 @@ func (trp *testReceiverProvider) GetDeltaStats() map[uint32]*buffer.StreamStatsW return trp.streams } +func (trp *testReceiverProvider) setLastSenderReportTime(at time.Time) { + trp.lastSenderReportTime = at +} + +func (trp *testReceiverProvider) GetLastSenderReportTime() time.Time { + return trp.lastSenderReportTime +} + // ----------------------------------------------- func TestConnectionQuality(t *testing.T) { trp := newTestReceiverProvider() t.Run("quality scorer operation", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + EnableBitrateScore: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -84,7 +84,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -100,7 +100,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 120, PacketsLost: 30, }, @@ -108,7 +108,7 @@ func TestConnectionQuality(t *testing.T) { 2: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 130, PacketsLost: 0, }, @@ -127,7 +127,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -143,7 +143,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -159,7 +159,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -175,7 +175,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 13, }, @@ -192,7 +192,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -208,7 +208,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -224,7 +224,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 30, }, @@ -241,7 +241,7 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // unmute at time so that next window does not satisfy the unmute time threshold. + // unmute at specific time to ensure next window does not satisfy the unmute time threshold. // that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality cs.UpdateMuteAt(false, now.Add(3*time.Second)) @@ -249,7 +249,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 0, }, }, @@ -259,13 +259,14 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // next update with no packets should knock quality down to LOST + // next update with no packets, + // but last RTCP is not set, should knock quality down to POOR now = now.Add(duration) trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 0, }, }, @@ -273,13 +274,46 @@ func TestConnectionQuality(t *testing.T) { cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // another dry spell, but last RTCP is not stale, should keep quality at POOR + now = now.Add(duration) + trp.setLastSenderReportTime(now.Add(time.Second)) + trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + EndTime: now.Add(duration), + Packets: 0, + }, + }, + }) + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // yet another dry spell, but last RTCP is stale, should knock down quality at LOST + now = now.Add(duration) + trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + EndTime: now.Add(duration), + Packets: 0, + }, + }, + }) + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(1.3), mos) require.Equal(t, livekit.ConnectionQuality_LOST, quality) // mute when LOST should not bump up score/quality now = now.Add(duration) cs.UpdateMuteAt(true, now.Add(1*time.Second)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(2.1), mos) + require.Greater(t, float32(1.3), mos) require.Equal(t, livekit.ConnectionQuality_LOST, quality) // unmute and send packets to bring quality back up @@ -290,7 +324,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 0, }, @@ -310,7 +344,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 50, PacketsLost: 5, }, @@ -332,7 +366,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, RttMax: 400, @@ -358,7 +392,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -377,7 +411,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -401,7 +435,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -428,7 +462,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -441,7 +475,14 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent rtt", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, false, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: false, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -455,7 +496,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, RttMax: 700, @@ -469,7 +510,14 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent jitter", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, false, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: false, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -483,7 +531,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, JitterMax: 200, @@ -558,7 +606,7 @@ func TestConnectionQuality(t *testing.T) { }, }, }, - // "audio/red" - no fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR + // "audio/red" - no fec - 0 <= loss < 5%: EXCELLENT, 5% <= loss < 15%: GOOD, >= 15%: POOR { name: "audio/red - no fec", mimeType: "audio/red", @@ -566,23 +614,23 @@ func TestConnectionQuality(t *testing.T) { packetsExpected: 200, expectedQualities: []expectedQuality{ { - packetLossPercentage: 8.0, + packetLossPercentage: 4.0, expectedMOS: 4.6, expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 12.0, + packetLossPercentage: 6.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 39.0, + packetLossPercentage: 19.5, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - fec - 0 <= loss < 15%: EXCELLENT, 15% <= loss < 45%: GOOD, >= 45%: POOR + // "audio/red" - fec - 0 <= loss < 7.5%: EXCELLENT, 7.5% <= loss < 22.5%: GOOD, >= 22.5%: POOR { name: "audio/red - fec", mimeType: "audio/red", @@ -590,17 +638,17 @@ func TestConnectionQuality(t *testing.T) { packetsExpected: 200, expectedQualities: []expectedQuality{ { - packetLossPercentage: 12.0, + packetLossPercentage: 6.0, expectedMOS: 4.6, expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 20.0, + packetLossPercentage: 10.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 60.0, + packetLossPercentage: 30.0, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, @@ -634,7 +682,14 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: tc.mimeType, + IsFECEnabled: tc.isFECEnabled, + IncludeRTT: true, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -645,7 +700,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: tc.packetsExpected, PacketsLost: uint32(math.Ceil(eq.packetLossPercentage * float64(tc.packetsExpected) / 100.0)), }, @@ -727,7 +782,15 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "video/vp8", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + EnableBitrateScore: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -741,7 +804,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 100, Bytes: tc.bytes, }, @@ -814,7 +877,14 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "video/vp8", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -828,7 +898,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 200, }, }, diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 33421fcb9..facef770a 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -61,6 +61,7 @@ type windowStat struct { bytes uint64 rttMax uint32 jitterMax float64 + lastRTCPAt time.Time } func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJitter bool) float64 { @@ -123,8 +124,8 @@ func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJ return score } -func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { - if expectedBitrate == 0 { +func (w *windowStat) calculateBitrateScore(expectedBitrate int64, isEnabled bool) float64 { + if expectedBitrate == 0 || !isEnabled { // unsupported mode OR all layers stopped return cMaxScore } @@ -147,7 +148,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { } func (w *windowStat) String() string { - return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f", + return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f, lastRTCP: %+v", w.startedAt, w.duration, w.packetsExpected, @@ -157,6 +158,7 @@ func (w *windowStat) String() string { w.bytes, w.rttMax, w.jitterMax, + w.lastRTCPAt, ) } @@ -174,16 +176,18 @@ func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddUint64("bytes", w.bytes) e.AddUint32("rttMax", w.rttMax) e.AddFloat64("jitterMax", w.jitterMax) + e.AddTime("lastRTCPAt", w.lastRTCPAt) return nil } // ------------------------------------------ type qualityScorerParams struct { - PacketLossWeight float64 - IncludeRTT bool - IncludeJitter bool - Logger logger.Logger + PacketLossWeight float64 + IncludeRTT bool + IncludeJitter bool + EnableBitrateScore bool + Logger logger.Logger } type qualityScorer struct { @@ -381,7 +385,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // considered (as long as enough time has passed since unmute). // // Similarly, when paused (possibly due to congestion), score is immediately - // set to cMinScore for responsiveness. The layer transision is reest. + // set to cMinScore for responsiveness. The layer transition is reset. // On a resume, quality climbs back up using normal operation. if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() { q.lastUpdateAt = at @@ -392,11 +396,16 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { reason := "none" var score float64 if stat.packetsExpected == 0 { - reason = "dry" - score = qualityTransitionScore[livekit.ConnectionQuality_LOST] + if !stat.lastRTCPAt.IsZero() && at.Sub(stat.lastRTCPAt) > stat.duration { + reason = "dry" + score = qualityTransitionScore[livekit.ConnectionQuality_LOST] + } else { + reason = "rtcp" + score = qualityTransitionScore[livekit.ConnectionQuality_POOR] + } } else { packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) - bitrateScore := stat.calculateBitrateScore(expectedBitrate) + bitrateScore := stat.calculateBitrateScore(expectedBitrate, q.params.EnableBitrateScore) layerScore := math.Max(math.Min(cMaxScore, cMaxScore-(expectedDistance*distanceWeight)), 0.0) minScore := math.Min(packetScore, bitrateScore) @@ -504,7 +513,7 @@ func (q *qualityScorer) isPaused() bool { } func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { - if stat == nil || stat.duration == 0 { + if stat == nil || stat.duration <= 0 { return q.params.PacketLossWeight } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 869f33cbc..6bb41ffc6 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "math" "strings" "sync" "time" @@ -35,9 +36,10 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/utils" ) @@ -145,11 +147,13 @@ func (d DownTrackState) String() string { // ------------------------------------------------------------------- +/* STREAM-ALLOCATOR-DATA type NackInfo struct { Timestamp uint32 SequenceNumber uint16 Attempts uint8 } +*/ type DownTrackStreamAllocatorListener interface { // RTCP received @@ -180,11 +184,13 @@ type DownTrackStreamAllocatorListener interface { // packet(s) sent OnPacketsSent(dt *DownTrack, size int) + /* STREAM-ALLOCATOR-DATA // NACKs received OnNACK(dt *DownTrack, nackInfos []NackInfo) // RTCP Receiver Report received OnRTCPReceiverReport(dt *DownTrack, rr rtcp.ReceptionReport) + */ // check if track should participate in BWE IsBWEEnabled(dt *DownTrack) bool @@ -197,6 +203,7 @@ type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) type DowntrackParams struct { Codecs []webrtc.RTPCodecParameters + Source livekit.TrackSource Receiver TrackReceiver BufferFactory *buffer.Factory SubID livekit.ParticipantID @@ -234,6 +241,7 @@ type DownTrack struct { transportWideExtID int dependencyDescriptorExtID int playoutDelayExtID int + absCaptureTimeExtID int transceiver atomic.Pointer[webrtc.RTPTransceiver] writeStream webrtc.TrackLocalWriter rtcpReader *buffer.RTCPReader @@ -267,8 +275,10 @@ type DownTrack struct { streamAllocatorListener DownTrackStreamAllocatorListener streamAllocatorReportGeneration int streamAllocatorBytesCounter atomic.Uint32 + /* STREAM-ALLOCATOR-DATA bytesSent atomic.Uint32 bytesRetransmitted atomic.Uint32 + */ playoutDelay *PlayoutDelayController @@ -318,7 +328,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { d.forwarder = NewForwarder( d.kind, params.Logger, - d.params.Receiver.GetReferenceLayerRTPTimestamp, + false, d.getExpectedRTPTimestamp, ) @@ -534,13 +544,9 @@ func (d *DownTrack) SubscriberID() livekit.ParticipantID { // Sets RTP header extensions for this track func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { - d.streamAllocatorLock.RLock() - listener := d.streamAllocatorListener - d.streamAllocatorLock.RUnlock() - isBWEEnabled := true - if listener != nil { - isBWEEnabled = listener.IsBWEEnabled(d) + if sal := d.getStreamAllocatorListener(); sal != nil { + isBWEEnabled = sal.IsBWEEnabled(d) } for _, ext := range rtpHeaderExtensions { switch ext.URI { @@ -552,7 +558,7 @@ func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeade } case dd.ExtensionURI: d.dependencyDescriptorExtID = ext.ID - case rtpextension.PlayoutDelayURI: + case pd.PlayoutDelayURI: d.playoutDelayExtID = ext.ID case sdp.TransportCCURI: if isBWEEnabled { @@ -560,6 +566,8 @@ func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeade } else { d.transportWideExtID = 0 } + case act.AbsCaptureTimeURI: + d.absCaptureTimeExtID = ext.ID } } } @@ -620,21 +628,23 @@ func (d *DownTrack) keyFrameRequester() { return time.Duration(interval) * time.Millisecond } - interval := getInterval() - ticker := time.NewTicker(interval) - defer ticker.Stop() + timer := time.NewTimer(math.MaxInt64) + timer.Stop() - for { - if d.IsClosed() { - return - } + defer timer.Stop() + + for !d.IsClosed() { + timer.Reset(getInterval()) select { case _, more := <-d.keyFrameRequesterCh: if !more { return } - case <-ticker.C: + if !timer.Stop() { + <-timer.C + } + case <-timer.C: } locked, layer := d.forwarder.CheckSync() @@ -643,8 +653,6 @@ func (d *DownTrack) keyFrameRequester() { d.params.Receiver.SendPLI(layer, false) d.rtpStats.UpdateLayerLockPliAndTime(1) } - - ticker.Reset(getInterval()) } } @@ -670,13 +678,23 @@ func (d *DownTrack) maxLayerNotifierWorker() { d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw("notifying max subscribed layer", "layer", maxLayerSpatial, "event", event) + d.params.Logger.Debugw( + "notifying max subscribed layer", + "layer", maxLayerSpatial, + "event", event, + "subscriberID", d.SubscriberID(), + ) onMaxSubscribedLayerChanged(d, maxLayerSpatial) } } if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw("notifying max subscribed layer", "layer", buffer.InvalidLayerSpatial, "event", "close") + d.params.Logger.Debugw( + "notifying max subscribed layer", + "layer", buffer.InvalidLayerSpatial, + "event", "close", + "subscriberID", d.SubscriberID(), + ) onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) } } @@ -697,18 +715,14 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { poolEntity := PacketFactory.Get().(*[]byte) payload := *poolEntity - shouldForward, incomingHeaderSize, outgoingHeaderSize, err := d.forwarder.TranslateCodecHeader(extPkt, &tp.rtp, payload) - if !shouldForward { - PacketFactory.Put(poolEntity) - return err - } - n := copy(payload[outgoingHeaderSize:], extPkt.Packet.Payload[incomingHeaderSize:]) - if n != len(extPkt.Packet.Payload[incomingHeaderSize:]) { - d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[incomingHeaderSize:]), "have", n) + copy(payload, tp.codecBytes) + n := copy(payload[len(tp.codecBytes):], extPkt.Packet.Payload[tp.incomingHeaderSize:]) + if n != len(extPkt.Packet.Payload[tp.incomingHeaderSize:]) { + d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[tp.incomingHeaderSize:]), "have", n) PacketFactory.Put(poolEntity) return ErrPayloadOverflow } - payload = payload[:outgoingHeaderSize+n] + payload = payload[:len(tp.codecBytes)+n] hdr, err := d.getTranslatedRTPHeader(extPkt, &tp) if err != nil { @@ -719,13 +733,59 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { var extensions []pacer.ExtensionData if tp.ddBytes != nil { - extensions = []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}} + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.dependencyDescriptorExtID), + Payload: tp.ddBytes, + }, + ) } if d.playoutDelayExtID != 0 && d.playoutDelay != nil { if val := d.playoutDelay.GetDelayExtension(hdr.SequenceNumber); val != nil { - extensions = append(extensions, pacer.ExtensionData{ID: uint8(d.playoutDelayExtID), Payload: val}) + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.playoutDelayExtID), + Payload: val, + }, + ) + + // NOTE: play out delay extension is not cached in sequencer, + // i. e. they will not be added to retransmitted packet. + // But, it is okay as the extension is added till a RTCP Receiver Report for + // the corresponding sequence number is received. + // The extreme case is all packets containing the play out delay are lost and + // all of them retransmitted and an RTCP Receiver Report received for those + // retransmited sequence numbers. But, that is highly improbable, if not impossible. } } + var actBytes []byte + if extPkt.AbsCaptureTimeExt != nil && d.absCaptureTimeExtID != 0 { + // normalize capture time to SFU clock. + // NOTE: even if there is estimated offset populated, just re-map the + // absolute capture time stamp as it should be the same RTCP sender report + // clock domain of publisher. SFU is normalising sender reports of publisher + // to SFU clock before sending to subscribers. So, capture time should be + // normalized to the same clock. Clear out any offset. + _, _, refSenderReport := d.forwarder.GetSenderReportParams() + if refSenderReport != nil { + actExtCopy := *extPkt.AbsCaptureTimeExt + if err = actExtCopy.Rewrite(refSenderReport.PropagationDelay()); err == nil { + actBytes, err = actExtCopy.Marshal() + if err == nil { + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.absCaptureTimeExtID), + Payload: actBytes, + }, + ) + } + } + } + } + if d.sequencer != nil { d.sequencer.push( extPkt.Arrival, @@ -734,9 +794,10 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { tp.rtp.extTimestamp, hdr.Marker, int8(layer), - payload[:outgoingHeaderSize], - incomingHeaderSize, + payload[:len(tp.codecBytes)], + tp.incomingHeaderSize, tp.ddBytes, + actBytes, ) } @@ -866,13 +927,9 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa // Mute enables or disables media forwarding - subscriber triggered func (d *DownTrack) Mute(muted bool) { - d.streamAllocatorLock.RLock() - listener := d.streamAllocatorListener - d.streamAllocatorLock.RUnlock() - isSubscribeMutable := true - if listener != nil { - isSubscribeMutable = listener.IsSubscribeMutable(d) + if sal := d.getStreamAllocatorListener(); sal != nil { + isSubscribeMutable = sal.IsSubscribeMutable(d) } changed := d.forwarder.Mute(muted, isSubscribeMutable) d.handleMute(muted, changed) @@ -988,9 +1045,10 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.rtcpReader.Close() d.rtcpReader.OnPacket(nil) } - d.bindLock.Unlock() + d.connectionStats.Close() + d.rtpStats.Stop() d.params.Logger.Debugw("rtp stats", "direction", "downstream", @@ -1088,7 +1146,7 @@ func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32 } } -func (d *DownTrack) maybeAddTransition(_ int64, distance float64, pauseReason VideoPauseReason) { +func (d *DownTrack) maybeAddTransition(bitrate int64, distance float64, pauseReason VideoPauseReason) { if d.kind == webrtc.RTPCodecTypeAudio { return } @@ -1098,6 +1156,7 @@ func (d *DownTrack) maybeAddTransition(_ int64, distance float64, pauseReason Vi } else { d.connectionStats.UpdatePause(false) d.connectionStats.AddLayerTransition(distance) + d.connectionStats.AddBitrateTransition(bitrate) } } @@ -1303,8 +1362,8 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - layer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.params.Receiver.GetRTCPSenderReportData(layer), tsOffset) + _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + return d.rtpStats.GetRtcpSenderReport(d.ssrc, refSenderReport, tsOffset) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { @@ -1444,15 +1503,16 @@ func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) - payload := make([]byte, 1000) - n, err := d.forwarder.GetPadding(frameEndNeeded, payload) + header, err := d.forwarder.GetPadding(frameEndNeeded) if err != nil { return nil, err } - copy(payload[n:], VP8KeyFrame8x8) - trailerLen := d.maybeAddTrailer(payload[n+len(VP8KeyFrame8x8):]) - return payload[:n+len(VP8KeyFrame8x8)+trailerLen], nil + payload := make([]byte, 1000) + copy(payload, header) + copy(payload[len(header):], VP8KeyFrame8x8) + trailerLen := d.maybeAddTrailer(payload[len(header)+len(VP8KeyFrame8x8):]) + return payload[:len(header)+len(VP8KeyFrame8x8)+trailerLen], nil } func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { @@ -1535,14 +1595,19 @@ func (d *DownTrack) handleRTCP(bytes []byte) { rttToReport = rtt } + /* STREAM-ALLOCATOR-DATA if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnRTCPReceiverReport(d, r) } + */ if d.playoutDelay != nil { - jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) d.playoutDelay.OnSeqAcked(uint16(r.LastSequenceNumber)) - d.playoutDelay.SetJitter(uint32(jitterMs)) + // screen share track has inaccuracy jitter due to its low frame rate and bursty traffic + if d.params.Source != livekit.TrackSource_SCREEN_SHARE { + jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) + d.playoutDelay.SetJitter(uint32(jitterMs)) + } } } if len(rr.Reports) > 0 { @@ -1652,18 +1717,20 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) - nackInfos := make([]NackInfo, 0, len(filtered)) + // STREAM-ALLOCATOR-DATA nackInfos := make([]NackInfo, 0, len(filtered)) for _, epm := range d.sequencer.getExtPacketMetas(filtered) { if disallowedLayers[epm.layer] { continue } nackAcks++ + /* STREAM-ALLOCATOR-DATA nackInfos = append(nackInfos, NackInfo{ SequenceNumber: epm.targetSeqNo, Timestamp: epm.timestamp, Attempts: epm.nacked, }) + */ pktBuff := *src n, err := d.params.Receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) @@ -1702,11 +1769,30 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { payload = payload[:int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] } - var ddBytes []byte - if len(epm.ddBytesSlice) != 0 { - ddBytes = epm.ddBytesSlice - } else { - ddBytes = epm.ddBytes[:epm.ddBytesSize] + var extensions []pacer.ExtensionData + if d.dependencyDescriptorExtID != 0 { + var ddBytes []byte + if len(epm.ddBytesSlice) != 0 { + ddBytes = epm.ddBytesSlice + } else { + ddBytes = epm.ddBytes[:epm.ddBytesSize] + } + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.dependencyDescriptorExtID), + Payload: ddBytes, + }, + ) + } + if d.absCaptureTimeExtID != 0 && len(epm.actBytes) != 0 { + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.absCaptureTimeExtID), + Payload: epm.actBytes, + }, + ) } d.sendingPacket( @@ -1722,7 +1808,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { ) d.pacer.Enqueue(pacer.Packet{ Header: &pkt.Header, - Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: ddBytes}}, + Extensions: extensions, Payload: payload, AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), @@ -1735,6 +1821,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { d.totalRepeatedNACKs.Add(numRepeatedNACKs) d.rtpStats.UpdateNackProcessed(nackAcks, nackMisses, numRepeatedNACKs) + /* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-START // Need to check on the following // - get all NACKs from sequencer even if SFU is not acknowledging, @@ -1750,6 +1837,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { if sal := d.getStreamAllocatorListener(); sal != nil && len(nackInfos) != 0 { sal.OnNACK(d, nackInfos) } + */ } func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (*rtp.Header, error) { @@ -1838,9 +1926,11 @@ func (d *DownTrack) GetNackStats() (totalPackets uint32, totalRepeatedNACKs uint return } +/* STREAM-ALLOCATOR-DATA func (d *DownTrack) GetAndResetBytesSent() (uint32, uint32) { return d.bytesSent.Swap(0), d.bytesRetransmitted.Swap(0) } +*/ func (d *DownTrack) onBindAndConnectedChange() { d.writable.Store(d.connected.Load() && d.bound.Load()) @@ -1947,9 +2037,11 @@ func (d *DownTrack) HandleRTCPSenderReportData( layer int32, publisherSRData *buffer.RTCPSenderReportData, ) error { - currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() + d.forwarder.SetRefSenderReport(isSVC, layer, publisherSRData) + + currentLayer, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() if layer == currentLayer || (layer == 0 && isSVC) { - d.handleRTCPSenderReportData(publisherSRData, tsOffset) + d.handleRTCPSenderReportData(refSenderReport, tsOffset) } return nil } @@ -1980,11 +2072,13 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter size := uint32(hdrSize + payloadSize) d.streamAllocatorBytesCounter.Add(size) + /* STREAM-ALLOCATOR-DATA if spmd.isRTX { d.bytesRetransmitted.Add(size) } else { d.bytesSent.Add(size) } + */ } // update RTPStats @@ -2011,10 +2105,6 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa } if spmd.tp.isResuming { - // adjust first packet time on a resumption so that subsequent switches get a more accurate expected time stamp - currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() - d.handleRTCPSenderReportData(d.params.Receiver.GetRTCPSenderReportData(currentLayer), tsOffset) - if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnResume(d) } diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 3768592f3..68fc7b411 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -96,9 +96,7 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) { // 100µs is enough to amortize the overhead and provide sufficient load balancing. // WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop. step := uint64(2) - utils.ParallelExec(downTracks, threshold, step, func(dt TrackSender) { - writer(dt) - }) + utils.ParallelExec(downTracks, threshold, step, writer) } func (d *DownTrackSpreader) DownTrackCount() int { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index de50a829b..92c354c3b 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -31,7 +31,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/codecmunger" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" ) @@ -44,7 +44,7 @@ const ( TransitionCostSpatial = 10 ResumeBehindThresholdSeconds = float64(0.2) // 200ms - ResumeBehindHighTresholdSeconds = float64(2.0) // 2 seconds + ResumeBehindHighThresholdSeconds = float64(2.0) // 2 seconds LayerSwitchBehindThresholdSeconds = float64(0.05) // 50ms SwitchAheadThresholdSeconds = float64(0.025) // 25ms ) @@ -174,12 +174,14 @@ func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error { // ------------------------------------------------------------------- type TranslationParams struct { - shouldDrop bool - isResuming bool - isSwitching bool - rtp TranslationParamsRTP - ddBytes []byte - marker bool + shouldDrop bool + isResuming bool + isSwitching bool + rtp TranslationParamsRTP + ddBytes []byte + incomingHeaderSize int + codecBytes []byte + marker bool } // ------------------------------------------------------------------- @@ -214,12 +216,12 @@ func (f ForwarderState) String() string { // ------------------------------------------------------------------- type Forwarder struct { - lock sync.RWMutex - codec webrtc.RTPCodecCapability - kind webrtc.RTPCodecType - logger logger.Logger - getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) - getExpectedRTPTimestamp func(at time.Time) (uint64, error) + lock sync.RWMutex + codec webrtc.RTPCodecCapability + kind webrtc.RTPCodecType + logger logger.Logger + skipReferenceTS bool + getExpectedRTPTimestamp func(at time.Time) (uint64, error) muted bool pubMuted bool @@ -231,6 +233,8 @@ type Forwarder struct { lastSSRC uint32 referenceLayerSpatial int32 refTSOffset uint64 + refSenderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData + refIsSVC bool provisional *VideoAllocationProvisional @@ -246,19 +250,19 @@ type Forwarder struct { func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, - getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), + skipReferenceTS bool, getExpectedRTPTimestamp func(at time.Time) (uint64, error), ) *Forwarder { f := &Forwarder{ - kind: kind, - logger: logger, - getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, - getExpectedRTPTimestamp: getExpectedRTPTimestamp, - referenceLayerSpatial: buffer.InvalidLayerSpatial, - lastAllocation: VideoAllocationDefault, - rtpMunger: NewRTPMunger(logger), - vls: videolayerselector.NewNull(logger), - codecMunger: codecmunger.NewNull(logger), + kind: kind, + logger: logger, + skipReferenceTS: skipReferenceTS, + getExpectedRTPTimestamp: getExpectedRTPTimestamp, + referenceLayerSpatial: buffer.InvalidLayerSpatial, + lastAllocation: VideoAllocationDefault, + rtpMunger: NewRTPMunger(logger), + vls: videolayerselector.NewNull(logger), + codecMunger: codecmunger.NewNull(logger), } if f.kind == webrtc.RTPCodecTypeVideo { @@ -556,15 +560,72 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { return layer } -func (f *Forwarder) GetCurrentSpatialAndTSOffset() (int32, uint64) { +func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *buffer.RTCPSenderReportData) { + f.lock.Lock() + defer f.lock.Unlock() + + f.refIsSVC = isSVC + if isSVC { + layer = 0 + } + if layer >= 0 && int(layer) < len(f.refSenderReports) { + f.refSenderReports[layer] = srData + } +} + +func (f *Forwarder) clearRefSenderReportsLocked() { + // On (re)start of fowarding, clear any old publisher sender reports. + // This is done to prevent use of potentially stale publisher sender reports. + // + // It is possible to implement mute using pause/unpause + // which can implemented using a replaceTrack(null)/replaceTrack(track). + // In those cases, the RTP time stamp may not jump across + // the mute/pause valley (for the time it is replaced with null track). + // So, relying on a report that happened before unmute/unpause + // could result in incorrect RTCP sender report on subscriber side. + // + // It could happen like this + // 1. Normal operation: publisher sending sender reports and + // suscribers use reports from publisher to calculate and send + // RTCP sender report. + // 2. Publisher pauses: there are no more reports. + // 3. When paused, subscriber can still use the publisher side sender + // report to send reports. Although the time since last publisher + // sender report is increasing, the reports are correct though. + // 4. Publisher unpauses after 20 seconds. But, it may not have advanced + // RTP Timestamp by that much. Let us say, it advances only by 5 seconds. + // 5. When subscriber starts forwarding packets, it will calculate + // a new time stamp offset to adjust to the new time stamp of publisher. + // 6. But, when that same offset is used on an old publisher sender report + // (i. e. a report from before the pause), the subscriber side sender + // reports jumps ahead in time by 15 seconds. + // + // By clearing sender report on (re)start of a stream, subscribers will wait for a fresh report + // after unmute to send sender report. + for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { + f.refSenderReports[layer] = nil + } +} + +func (f *Forwarder) GetSenderReportParams() (int32, uint64, *buffer.RTCPSenderReportData) { f.lock.RLock() defer f.lock.RUnlock() if f.kind == webrtc.RTPCodecTypeAudio { - return 0, f.rtpMunger.GetPinnedTSOffset() + return 0, f.rtpMunger.GetPinnedTSOffset(), f.refSenderReports[0] } - return f.vls.GetCurrent().Spatial, f.rtpMunger.GetPinnedTSOffset() + currentLayerSpatial := f.vls.GetCurrent().Spatial + if currentLayerSpatial < 0 || currentLayerSpatial > buffer.DefaultMaxLayerSpatial { + return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), nil + } + + refSenderReport := f.refSenderReports[currentLayerSpatial] + if f.refIsSVC { + refSenderReport = f.refSenderReports[0] + } + + return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), refSenderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -1481,12 +1542,40 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) }, ErrUnknownKind } +func (f *Forwarder) getReferenceLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) { + if refLayer < 0 || int(refLayer) > len(f.refSenderReports) || targetLayer < 0 || int(targetLayer) > len(f.refSenderReports) { + return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) + } + + if refLayer == targetLayer || f.refIsSVC { + return ts, nil + } + + srRef := f.refSenderReports[refLayer] + srTarget := f.refSenderReports[targetLayer] + if srRef == nil || srRef.NTPTimestamp == 0 || srTarget == nil || srTarget.NTPTimestamp == 0 { + return 0, fmt.Errorf("unavailable layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) + } + + ntpDiff := srRef.NTPTimestamp.Time().Sub(srTarget.NTPTimestamp.Time()) + rtpDiff := ntpDiff.Nanoseconds() * int64(f.codec.ClockRate) / 1e9 + + // calculate other layer's time stamp at the same time as ref layer's NTP time + normalizedOtherTS := srTarget.RTPTimestamp + uint32(rtpDiff) + + // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers + offset := srRef.RTPTimestamp - normalizedOtherTS + + return ts + offset, nil +} + func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { if !f.started { f.started = true f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) f.codecMunger.SetLast(extPkt) + f.clearRefSenderReportsLocked() f.logger.Debugw( "starting forwarding", "sequenceNumber", extPkt.Packet.SequenceNumber, @@ -1535,27 +1624,29 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e extLastTS := rtpMungerState.ExtLastTS extExpectedTS := extLastTS extRefTS := extExpectedTS + refTS := uint32(extRefTS) switchingAt := time.Now() - if f.getReferenceLayerRTPTimestamp != nil { - ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + if !f.skipReferenceTS { + var err error + refTS, err = f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, f.referenceLayerSpatial, layer) if err != nil { - // error out if extRefTS is not available. It can happen when there is no sender report + // error out if refTS is not available. It can happen when there is no sender report // for the layer being switched to. Can especially happen at the start of the track when layer switches are // potentially happening very quickly. Erroring out and waiting for a layer for which a sender report has been // received will calculate a better offset, but may result in initial adaptation to take a bit longer depending // on how often publisher/remote side sends RTCP sender report. return err } + } - extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(ts) + extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(refTS) - expectedTS32 := uint32(extExpectedTS) - if (ts-expectedTS32) < 1<<31 && ts < expectedTS32 { - extRefTS += (1 << 32) - } - if (expectedTS32-ts) < 1<<31 && expectedTS32 < ts && extRefTS >= 1<<32 { - extRefTS -= (1 << 32) - } + expectedTS := uint32(extExpectedTS) + if (refTS-expectedTS) < 1<<31 && refTS < expectedTS { + extRefTS += (1 << 32) + } + if (expectedTS-refTS) < 1<<31 && expectedTS < refTS && extRefTS >= 1<<32 { + extRefTS -= (1 << 32) } if f.getExpectedRTPTimestamp != nil { @@ -1611,7 +1702,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold { logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS - } else if diffSeconds > ResumeBehindHighTresholdSeconds { + } else if diffSeconds > ResumeBehindHighThresholdSeconds { // could be due to incorrect reference calculation logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS @@ -1625,6 +1716,13 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e extNextTS = extRefTS } f.resumeBehindThreshold = 0.0 + + // sender reports are cleared after calculating switch time stamp + // as relative differences between layers should remain the same. + // TODO: If the relative difference changes a lot, probably have to + // abandon the checks above and just use the expected timestamp + // as the next time stamp. + f.clearRefSenderReportsLocked() } else { // switching between layers, check if extRefTS is too far behind the last sent diffSeconds := float64(int64(extRefTS-extLastTS)) / float64(f.codec.ClockRate) @@ -1675,11 +1773,11 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } // should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error { +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) { if f.lastSSRC != extPkt.Packet.SSRC { if err := f.processSourceSwitch(extPkt, layer); err != nil { tp.shouldDrop = true - return nil + return false, nil } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC @@ -1689,19 +1787,24 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - return nil + return false, nil } - return err + return false, err } tp.rtp = tpRTP - return nil + + if len(extPkt.Packet.Payload) > 0 { + return f.translateCodecHeader(extPkt, tp) + } + + return false, nil } // should be called with lock held func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { tp := TranslationParams{} - if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { + if _, err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { tp.shouldDrop = true return tp, err } @@ -1765,49 +1868,40 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in return tp, nil } - err := f.getTranslationParamsCommon(extPkt, layer, &tp) + isTemporalSwitching, err := f.getTranslationParamsCommon(extPkt, layer, &tp) if tp.shouldDrop { - maybeRollback(result.IsSwitching) + maybeRollback(result.IsSwitching || isTemporalSwitching) return tp, err } - return tp, nil + return tp, err } -func (f *Forwarder) TranslateCodecHeader(extPkt *buffer.ExtPacket, tpr *TranslationParamsRTP, outputBuffer []byte) (bool, int, int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - maybeRollback := func(isSwitching bool) { - if isSwitching { - f.vls.Rollback() - } - } - +func (f *Forwarder) translateCodecHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (bool, error) { // codec specific forwarding check and any needed packet munging tl, isSwitching := f.vls.SelectTemporal(extPkt) - inputSize, outputSize, err := f.codecMunger.UpdateAndGet( + inputSize, codecBytes, err := f.codecMunger.UpdateAndGet( extPkt, - tpr.snOrdering == SequenceNumberOrderingOutOfOrder, - tpr.snOrdering == SequenceNumberOrderingGap, + tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, + tp.rtp.snOrdering == SequenceNumberOrderingGap, tl, - outputBuffer, ) if err != nil { + tp.shouldDrop = true if err == codecmunger.ErrFilteredVP8TemporalLayer || err == codecmunger.ErrOutOfOrderVP8PictureIdCacheMiss { if err == codecmunger.ErrFilteredVP8TemporalLayer { // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - maybeRollback(isSwitching) - return false, 0, 0, nil + return isSwitching, nil } - maybeRollback(isSwitching) - return false, 0, 0, err + return isSwitching, err } + tp.incomingHeaderSize = inputSize + tp.codecBytes = codecBytes - return true, inputSize, outputSize, nil + return isSwitching, nil } func (f *Forwarder) maybeStart() { @@ -1884,11 +1978,11 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S return snts, frameEndNeeded, err } -func (f *Forwarder) GetPadding(frameEndNeeded bool, outputBuffer []byte) (int, error) { +func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { f.lock.Lock() defer f.lock.Unlock() - return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded, outputBuffer) + return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) } func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index aaecb03b8..608ab9155 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -32,7 +32,7 @@ func disable(f *Forwarder) { } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { - f := NewForwarder(kind, logger.GetLogger(), nil, nil) + f := NewForwarder(kind, logger.GetLogger(), true, nil) f.DetermineCodec(codec, nil) return f } @@ -1368,7 +1368,6 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } func TestForwarderGetTranslationParamsVideo(t *testing.T) { - buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -1432,22 +1431,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - isSwitching: true, - isResuming: true, - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23333, - extTimestamp: 0xabcdef, - }, - marker: true, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - require.True(t, f.started) - require.Equal(t, f.lastSSRC, params.SSRC) - expectedVP8 := &buffer.VP8{ FirstByte: 25, I: true, @@ -1464,13 +1447,23 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } marshalledVP8, err := expectedVP8.Marshal() + expectedTP = TranslationParams{ + isSwitching: true, + isResuming: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23333, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + marker: true, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err := f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) - require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) + require.True(t, f.started) + require.Equal(t, f.lastSSRC, params.SSRC) // send a duplicate, should be dropped expectedTP = TranslationParams{ @@ -1518,17 +1511,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23334, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1546,12 +1528,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23334, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // temporal layer matching target, should be forwarded params = &testutils.TestExtPacketParams{ @@ -1577,17 +1565,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23335, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1605,12 +1582,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23335, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // temporal layer higher than target, should be dropped params = &testutils.TestExtPacketParams{ @@ -1636,6 +1619,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ + shouldDrop: true, rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23336, @@ -1646,10 +1630,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedTP, actualTP) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) - require.NoError(t, err) - require.False(t, shouldForward) - // RTP sequence number and VP8 picture id should be contiguous after dropping higher temporal layer picture params = &testutils.TestExtPacketParams{ SequenceNumber: 23338, @@ -1673,17 +1653,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23336, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1701,12 +1670,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23336, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // padding only packet after a gap should be forwarded params = &testutils.TestExtPacketParams{ @@ -1776,19 +1751,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - isSwitching: true, - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23339, - extTimestamp: 0xabcdf0, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 1) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - require.Equal(t, f.lastSSRC, params.SSRC) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1806,12 +1768,20 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + isSwitching: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23339, + extTimestamp: 0xabcdf0, + }, + incomingHeaderSize: 5, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 1) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 5, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) + require.Equal(t, f.lastSSRC, params.SSRC) } func TestForwarderGetSnTsForPadding(t *testing.T) { @@ -1959,7 +1929,6 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) { } func TestForwarderGetPaddingVP8(t *testing.T) { - buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -2010,11 +1979,11 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - n, err := f.GetPadding(true, buf) + buf, err := f.GetPadding(true) require.NoError(t, err) marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) // getting padding with no frame end needed, should get next picture id expectedVP8 = buffer.VP8{ @@ -2032,9 +2001,9 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - n, err = f.GetPadding(false, buf) + buf, err = f.GetPadding(false) require.NoError(t, err) marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) } diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index 2dc756d39..c0ed985b0 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -17,9 +17,10 @@ package sfu import ( "sync" "sync/atomic" + "time" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/protocol/logger" ) @@ -30,11 +31,11 @@ const ( PlayoutDelaySending PlayoutDelayAcked - jitterLowMultiToDelay = 10 - jitterHighMultiToDelay = 15 - jitterHighThreshold = 15 - + jitterMultiToDelay = 10 targetDelayLogThreshold = 500 + + // limit max delay change to make it smoother for a/v sync + maxDelayChangePerSec = 80 ) func (s PlayoutDelayState) String() string { @@ -56,17 +57,20 @@ type PlayoutDelayController struct { currentDelay uint32 extBytes atomic.Value //[]byte sendingAtSeq uint16 + sendingAtTime time.Time logger logger.Logger rtpStats *buffer.RTPStatsSender snapshotID uint32 + + highDelayCount atomic.Uint32 } func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *buffer.RTPStatsSender) (*PlayoutDelayController, error) { if maxDelay == 0 && minDelay > 0 { - maxDelay = rtpextension.MaxPlayoutDelayDefault + maxDelay = pd.MaxPlayoutDelayDefault } - if maxDelay > rtpextension.PlayoutDelayMaxValue { - maxDelay = rtpextension.PlayoutDelayMaxValue + if maxDelay > pd.PlayoutDelayMaxValue { + maxDelay = pd.PlayoutDelayMaxValue } c := &PlayoutDelayController{ currentDelay: minDelay, @@ -87,20 +91,21 @@ func (c *PlayoutDelayController) SetJitter(jitter uint32) { } c.lock.Lock() - multi := jitterLowMultiToDelay - if jitter >= jitterHighThreshold { - multi = jitterHighMultiToDelay - } - targetDelay := jitter * uint32(multi) + targetDelay := jitter * jitterMultiToDelay if nackPercent > 60 { targetDelay += (nackPercent - 60) * 2 } - // increase delay quickly, decrease slowly to make fps more stable - if targetDelay > c.currentDelay { - targetDelay = (targetDelay-c.currentDelay)*3/4 + c.currentDelay - } else { - targetDelay = c.currentDelay - (c.currentDelay-targetDelay)/5 + elapsed := time.Since(c.sendingAtTime) + delayChangeLimit := uint32(maxDelayChangePerSec * elapsed.Seconds()) + if delayChangeLimit > maxDelayChangePerSec { + delayChangeLimit = maxDelayChangePerSec + } + + if targetDelay > c.currentDelay+delayChangeLimit { + targetDelay = c.currentDelay + delayChangeLimit + } else if c.currentDelay > targetDelay+delayChangeLimit { + targetDelay = c.currentDelay - delayChangeLimit } if targetDelay < c.minDelay { targetDelay = c.minDelay @@ -113,7 +118,9 @@ func (c *PlayoutDelayController) SetJitter(jitter uint32) { return } if targetDelay > targetDelayLogThreshold { - c.logger.Debugw("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay) + if c.highDelayCount.Add(1)%100 == 1 { + c.logger.Infow("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay) + } } c.currentDelay = targetDelay c.lock.Unlock() @@ -134,6 +141,7 @@ func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte { c.lock.Lock() c.state.Store(int32(PlayoutDelaySending)) c.sendingAtSeq = seq + c.sendingAtTime = time.Now() c.lock.Unlock() return c.extBytes.Load().([]byte) case PlayoutDelaySending: @@ -145,7 +153,7 @@ func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte { } func (c *PlayoutDelayController) createExtData() error { - delay := rtpextension.PlayoutDelayFromValue( + delay := pd.PlayoutDelayFromValue( uint16(c.currentDelay), uint16(c.maxDelay), ) diff --git a/pkg/sfu/playoutdelay_test.go b/pkg/sfu/playoutdelay_test.go index 05a88f5d0..338b4d64b 100644 --- a/pkg/sfu/playoutdelay_test.go +++ b/pkg/sfu/playoutdelay_test.go @@ -16,33 +16,34 @@ package sfu import ( "testing" + "time" "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/protocol/logger" ) func TestPlayoutDelay(t *testing.T) { stats := buffer.NewRTPStatsSender(buffer.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}) - c, err := NewPlayoutDelayController(100, 1000, logger.GetLogger(), stats) + c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats) require.NoError(t, err) ext := c.GetDelayExtension(100) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) // seq acked before delay changed c.OnSeqAcked(65534) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) c.OnSeqAcked(90) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) // seq acked, no extension sent for new packet c.OnSeqAcked(103) @@ -55,20 +56,23 @@ func TestPlayoutDelay(t *testing.T) { require.Nil(t, ext) // delay changed, generate new extension to send + time.Sleep(200 * time.Millisecond) c.SetJitter(50) + t.Log(c.currentDelay, c.state.Load()) ext = c.GetDelayExtension(108) - var delay rtpextension.PlayOutDelay + var delay pd.PlayOutDelay require.NoError(t, delay.Unmarshal(ext)) require.Greater(t, delay.Min, uint16(100)) // can't go above max + time.Sleep(200 * time.Millisecond) c.SetJitter(10000) ext = c.GetDelayExtension(109) - playoutDelayEqual(t, ext, 1000, 1000) + playoutDelayEqual(t, ext, 120, 120) } func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) { - var delay rtpextension.PlayOutDelay + var delay pd.PlayOutDelay require.NoError(t, delay.Unmarshal(data)) require.Equal(t, min, delay.Min) require.Equal(t, max, delay.Max) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 3df301cda..9bc8849da 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -35,7 +35,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) var ( @@ -84,9 +84,6 @@ type TrackReceiver interface { GetFrameRates() [][]float32 GetTemporalLayerFpsForSpatial(layer int32) (bool, []float32) - GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) - GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData - GetTrackStats() *livekit.RTPStats } @@ -348,11 +345,10 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff ObserveDuration: w.audioConfig.UpdateInterval, SmoothIntervals: w.audioConfig.SmoothIntervals, }) + buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func() { srData := buff.GetSenderReportData() - w.streamTrackerManager.SetRTCPSenderReportData(layer, srData) - w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData) }) @@ -431,8 +427,31 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { return nil } +func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { + ti := w.TrackInfo() + if ti == nil { + return + } + + if w.Kind() == webrtc.RTPCodecTypeAudio || ti.Source == livekit.TrackSource_SCREEN_SHARE { + // screen share tracks have highly variable bitrate, do not use bit rate based quality for those + return + } + + expectedBitrate := int64(0) + for _, vl := range ti.Layers { + l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti) + if l <= layer { + expectedBitrate += int64(vl.Bitrate) + } + } + + w.connectionStats.AddBitrateTransition(expectedBitrate) +} + func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) + w.notifyMaxExpectedLayer(layer) if layer == buffer.InvalidLayerSpatial { w.connectionStats.UpdateLayerMute(true) @@ -464,6 +483,7 @@ func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) }) + w.notifyMaxExpectedLayer(maxPublishedLayer) w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } @@ -628,6 +648,25 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer return deltaStats } +func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time { + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + + latestSRTime := time.Time{} + for _, buff := range w.buffers { + if buff == nil { + continue + } + + srAt := buff.GetLastSenderReportTime() + if srAt.After(latestSRTime) { + latestSRTime = srAt + } + } + + return latestSRTime +} + func (w *WebRTCReceiver) forwardRTP(layer int32) { pktBuf := make([]byte, bucket.MaxPktSize) tracker := w.streamTrackerManager.GetTracker(layer) @@ -829,14 +868,6 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) (bool, []flo return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) -} - -func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - return w.streamTrackerManager.GetRTCPSenderReportData(layer) -} - // closes all track senders in parallel, returns when all are closed func closeTrackSenders(senders []TrackSender) { wg := sync.WaitGroup{} diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 72261f7db..d19181446 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -204,7 +204,7 @@ func TestRedReceiver(t *testing.T) { verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt) } - // and then a few packets with a large timestmap jump, should contain only primary + // and then a few packets with a large timestamp jump, should contain only primary for _, pkt := range generatePkts(header, 4, 40*tsStep) { red.ForwardRTP(&buffer.ExtPacket{ Packet: pkt, diff --git a/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go b/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go new file mode 100644 index 000000000..1eb3f8d2a --- /dev/null +++ b/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go @@ -0,0 +1,114 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abscapturetime + +import ( + "encoding/binary" + "errors" + "time" + + "github.com/livekit/mediatransportutil" +) + +const ( + AbsCaptureTimeURI = "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" +) + +var ( + errInvalidData = errors.New("invalid data") + errTooSmall = errors.New("buffer too small") +) + +// Reference: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/abs-capture-time/ +// +// Data layout of the shortened version of abs-capture-time with a 1-byte header + 8 bytes of data: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=7 | absolute capture timestamp (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | absolute capture timestamp (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | +// +-+-+-+-+-+-+-+-+ +// +//Data layout of the extended version of abs-capture-time with a 1-byte header + 16 bytes of data: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=15| absolute capture timestamp (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | absolute capture timestamp (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | estimated capture clock offset (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | estimated capture clock offset (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | +// +-+-+-+-+-+-+-+-+ + +type AbsCaptureTime struct { + absoluteCaptureTimestamp mediatransportutil.NtpTime + estimatedCaptureClockOffset int64 +} + +func AbsCaptureTimeFromValue(absoluteCaptureTimestamp uint64, estimatedCaptureClockOffset int64) *AbsCaptureTime { + return &AbsCaptureTime{ + absoluteCaptureTimestamp: mediatransportutil.NtpTime(absoluteCaptureTimestamp), + estimatedCaptureClockOffset: estimatedCaptureClockOffset, + } +} + +func (a *AbsCaptureTime) Rewrite(offset time.Duration) error { + if a.absoluteCaptureTimestamp == 0 { + return errInvalidData + } + + capturedAt := a.absoluteCaptureTimestamp.Time().Add(offset) + a.absoluteCaptureTimestamp = mediatransportutil.ToNtpTime(capturedAt) + a.estimatedCaptureClockOffset = 0 + return nil +} + +func (a *AbsCaptureTime) Marshal() ([]byte, error) { + if a.absoluteCaptureTimestamp == 0 { + return nil, errInvalidData + } + + size := 8 + if a.estimatedCaptureClockOffset != 0 { + size += 8 + } + marshalled := make([]byte, size) + binary.BigEndian.PutUint64(marshalled, uint64(a.absoluteCaptureTimestamp)) + if a.estimatedCaptureClockOffset != 0 { + binary.BigEndian.PutUint64(marshalled[8:], uint64(a.estimatedCaptureClockOffset)) + } + return marshalled, nil +} + +func (a *AbsCaptureTime) Unmarshal(marshalled []byte) error { + if len(marshalled) < 8 { + return errTooSmall + } + + a.absoluteCaptureTimestamp = mediatransportutil.NtpTime(binary.BigEndian.Uint64(marshalled)) + if len(marshalled) >= 16 { + a.estimatedCaptureClockOffset = int64(binary.BigEndian.Uint64(marshalled[8:])) + } + return nil +} diff --git a/pkg/sfu/dependencydescriptor/bitstreamreader.go b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go similarity index 100% rename from pkg/sfu/dependencydescriptor/bitstreamreader.go rename to pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go diff --git a/pkg/sfu/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go similarity index 100% rename from pkg/sfu/dependencydescriptor/bitstreamwriter.go rename to pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorextension.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorreader.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go diff --git a/pkg/sfu/rtpextension/playoutdelay.go b/pkg/sfu/rtpextension/playoutdelay/playoutdelay.go similarity index 99% rename from pkg/sfu/rtpextension/playoutdelay.go rename to pkg/sfu/rtpextension/playoutdelay/playoutdelay.go index 2e311a621..d1017846e 100644 --- a/pkg/sfu/rtpextension/playoutdelay.go +++ b/pkg/sfu/rtpextension/playoutdelay/playoutdelay.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rtpextension +package playoutdelay import ( "encoding/binary" diff --git a/pkg/sfu/rtpextension/playoutdelay_test.go b/pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go similarity index 98% rename from pkg/sfu/rtpextension/playoutdelay_test.go rename to pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go index 7c07ed26c..5a2dc078b 100644 --- a/pkg/sfu/rtpextension/playoutdelay_test.go +++ b/pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rtpextension +package playoutdelay import ( "testing" diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 0fafcfc81..a78b23eb6 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -75,6 +75,8 @@ type packetMeta struct { ddBytes [8]byte ddBytesSize uint8 ddBytesSlice []byte + // abs-capture-time of packet + actBytes []byte } type extPacketMeta struct { @@ -134,6 +136,7 @@ func (s *sequencer) push( codecBytes []byte, numCodecBytesIn int, ddBytes []byte, + actBytes []byte, ) { s.Lock() defer s.Unlock() @@ -220,6 +223,8 @@ func (s *sequencer) push( copy(pm.ddBytes[:pm.ddBytesSize], ddBytes) } + pm.actBytes = append([]byte{}, actBytes...) + if extModifiedSN > s.extHighestSN { s.extHighestSN = extModifiedSN } @@ -344,6 +349,7 @@ func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta { } epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...) epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...) + epm.actBytes = append([]byte{}, meta.actBytes...) extPacketMetas = append(extPacketMetas, epm) } } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 39dcec906..f8eabe045 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -29,11 +29,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint64(1); i < 518; i++ { - seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, 0, nil) + seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil) } // send the last two out-of-order - seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil) - seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil) + seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil) + seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} res := seq.getExtPacketMetas(req) @@ -63,14 +63,14 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.extTimestamp, uint64(123)) } - seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil) + seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil) m := seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) m = seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil) + seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil) m = seq.getExtPacketMetas([]uint16{505 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) @@ -99,6 +99,8 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { ddBytesOdd []byte ddBytesEven []byte ddBytesOversized []byte + actBytesOdd []byte + actBytesEven []byte } tests := []struct { @@ -132,6 +134,8 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { ddBytesOdd: []byte{8, 9, 10}, ddBytesEven: []byte{11, 12}, ddBytesOversized: []byte{11, 12, 13, 14, 15, 16, 17, 18, 19}, + actBytesOdd: []byte{0, 1, 2, 3, 4, 5, 6, 7}, + actBytesEven: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, }, args: args{ seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */}, @@ -162,6 +166,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesOversized, len(tt.fields.codecBytesOversized), tt.fields.ddBytesOversized, + tt.fields.actBytesOdd, ) } else { if i.seqNo%2 == 0 { @@ -175,6 +180,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, + tt.fields.actBytesEven, ) } else { n.push( @@ -187,6 +193,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, + tt.fields.actBytesOdd, ) } } @@ -204,6 +211,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOversized, sn.ddBytesSlice) require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } else { if sn.sourceSeqNo%2 == 0 { require.Equal(t, tt.fields.markerEven, sn.marker) @@ -211,12 +219,14 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesEven, sn.actBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } } } @@ -246,6 +256,8 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { numCodecBytesInEven int ddBytesOdd []byte ddBytesEven []byte + actBytesOdd []byte + actBytesEven []byte } tests := []struct { @@ -278,6 +290,8 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { numCodecBytesInEven: 4, ddBytesOdd: []byte{8, 9, 10}, ddBytesEven: []byte{11, 12}, + actBytesOdd: []byte{8, 9, 10}, + actBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5, 12 + 5}, @@ -306,6 +320,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, + tt.fields.actBytesEven, ) } else { n.push( @@ -318,6 +333,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, + tt.fields.actBytesOdd, ) } } @@ -334,12 +350,14 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesEven, sn.actBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } } if !reflect.DeepEqual(got, tt.want) { diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 9afab9c49..585484408 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -134,9 +134,11 @@ func (c *ChannelObserver) GetNackRatio() float64 { return c.nackTracker.GetRatio() } +/* STREAM-ALLOCATOR-DATA func (c *ChannelObserver) GetNackHistory() []string { return c.nackTracker.GetHistory() } +*/ func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { estimateDirection := c.estimateTrend.GetDirection() diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go index b353781e5..c7131a01b 100644 --- a/pkg/sfu/streamallocator/nacktracker.go +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -38,20 +38,22 @@ type NackTracker struct { packets uint32 repeatedNacks uint32 + /* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove when cleaning up experimental stuff history []string + */ } func NewNackTracker(params NackTrackerParams) *NackTracker { return &NackTracker{ - params: params, - history: make([]string, 0, 10), + params: params, + // STREAM-ALLOCATOR-DATA history: make([]string, 0, 10), } } func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) { if n.params.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMaxDuration { - n.updateHistory() + // STREAM-ALLOCATOR-DATA n.updateHistory() n.windowStartTime = time.Time{} n.packets = 0 @@ -104,6 +106,7 @@ func (n *NackTracker) ToString() string { return fmt.Sprintf("n: %s, %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) } +/* STREAM-ALLOCATOR-DATA func (n *NackTracker) GetHistory() []string { return n.history } @@ -115,5 +118,6 @@ func (n *NackTracker) updateHistory() { n.history = append(n.history, n.ToString()) } +*/ // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/ratemonitor.go b/pkg/sfu/streamallocator/ratemonitor.go index 9c2ba243e..f2275822f 100644 --- a/pkg/sfu/streamallocator/ratemonitor.go +++ b/pkg/sfu/streamallocator/ratemonitor.go @@ -16,6 +16,7 @@ package streamallocator import ( "fmt" + "sync" "time" "github.com/livekit/protocol/utils/timeseries" @@ -31,6 +32,7 @@ const ( // ------------------------------------------------ type RateMonitor struct { + mu sync.Mutex bitrateEstimate *timeseries.TimeSeries[int64] managedBytesSent *timeseries.TimeSeries[uint32] managedBytesRetransmitted *timeseries.TimeSeries[uint32] @@ -67,6 +69,9 @@ func NewRateMonitor() *RateMonitor { } func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedBytesRetransmitted uint32, unmanagedBytesSent uint32, unmanagedBytesRetransmitted uint32) { + r.mu.Lock() + defer r.mu.Unlock() + now := time.Now() r.bitrateEstimate.AddSampleAt(estimate, now) r.managedBytesSent.AddSampleAt(managedBytesSent, now) @@ -82,36 +87,36 @@ func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedByt // Reason is that the estimate could be higher than the actual rate by a significant amount. // So, updating periodically to flush out samples that will not contribute to queueing would be good. func (r *RateMonitor) GetQueuingGuess() float64 { - _, _, _, _, _, qd := r.getRates(queueMonitorWindow) - return qd + _, _, _, _, _, queuingDelay := r.getRates(queueMonitorWindow) + return queuingDelay } -func (r *RateMonitor) getRates(monitorDuration time.Duration) (float64, float64, float64, float64, float64, float64) { - threshold := time.Now().Add(-monitorDuration) - bitrateEstimateSamples := r.bitrateEstimate.GetSamplesAfter(threshold) - managedBytesSentSamples := r.managedBytesSent.GetSamplesAfter(threshold) - managedBytesRetransmittedSamples := r.managedBytesRetransmitted.GetSamplesAfter(threshold) - unmanagedBytesSentSamples := r.unmanagedBytesSent.GetSamplesAfter(threshold) - unmanagedBytesRetransmittedSamples := r.unmanagedBytesRetransmitted.GetSamplesAfter(threshold) +func (r *RateMonitor) getRates(monitorDuration time.Duration) (totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay float64) { + r.mu.Lock() + defer r.mu.Unlock() - if len(bitrateEstimateSamples) == 0 || (len(managedBytesSentSamples)+len(managedBytesRetransmittedSamples)+len(unmanagedBytesSentSamples)+len(unmanagedBytesRetransmittedSamples)) == 0 { - return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + threshold := time.Now().Add(-monitorDuration) + if !r.bitrateEstimate.HasSamplesAfter(threshold) || + !(r.managedBytesSent.HasSamplesAfter(threshold) || + r.managedBytesRetransmitted.HasSamplesAfter(threshold) || + r.unmanagedBytesSent.HasSamplesAfter(threshold) || + r.unmanagedBytesRetransmitted.HasSamplesAfter(threshold)) { + return } - totalBitrateEstimate := getTimeWeightedSum(bitrateEstimateSamples) - totalManagedSent := getRate(managedBytesSentSamples) * 8 - totalManagedRetransmitted := getRate(managedBytesRetransmittedSamples) * 8 - totalUnmanagedSent := getRate(unmanagedBytesSentSamples) * 8 - totalUnmanagedRetransmitted := getRate(unmanagedBytesRetransmittedSamples) * 8 + totalBitrateEstimate = getTimeWeightedSum(r.bitrateEstimate.ReverseIterateSamplesAfter(threshold)) + totalManagedSent = getRate(r.managedBytesSent.ReverseIterateSamplesAfter(threshold)) * 8 + totalManagedRetransmitted = getRate(r.managedBytesRetransmitted.ReverseIterateSamplesAfter(threshold)) * 8 + totalUnmanagedSent = getRate(r.unmanagedBytesSent.ReverseIterateSamplesAfter(threshold)) * 8 + totalUnmanagedRetransmitted = getRate(r.unmanagedBytesRetransmitted.ReverseIterateSamplesAfter(threshold)) * 8 totalBits := totalManagedSent + totalManagedRetransmitted + totalUnmanagedSent + totalUnmanagedRetransmitted - queuingDelay := float64(0.0) if totalBits > totalBitrateEstimate { - latestBitrateEstimate := bitrateEstimateSamples[len(bitrateEstimateSamples)-1].Value + latestBitrateEstimate := r.bitrateEstimate.Back().Value excessBits := totalBits - totalBitrateEstimate queuingDelay = excessBits / float64(latestBitrateEstimate) } - return totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay + return } func (r *RateMonitor) updateHistory() { @@ -124,10 +129,12 @@ func (r *RateMonitor) updateHistory() { return } + r.mu.Lock() r.history = append( r.history, fmt.Sprintf("t: %+v, e: %.2f, m: %.2f/%.2f, um: %.2f/%.2f, qd: %.2f", time.Now().UnixMilli(), e, m, mr, um, umr, qd), ) + r.mu.Unlock() } func (r *RateMonitor) GetHistory() []string { @@ -136,37 +143,30 @@ func (r *RateMonitor) GetHistory() []string { // ------------------------------------------------ -func getTimeWeightedSum[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { - if len(samples) < 2 { - return 0.0 - } - +func getTimeWeightedSum[T int64 | uint32](it timeseries.ReverseIterator[T]) float64 { sum := 0.0 - for i := 1; i < len(samples); i++ { - diff := samples[i].At.Sub(samples[i-1].At).Seconds() - sum += diff * float64(samples[i-1].Value) + next := time.Now() + for it.Next() { + diff := next.Sub(it.Value().At).Seconds() + sum += diff * float64(it.Value().Value) + next = it.Value().At } - - diff := time.Now().Sub(samples[len(samples)-1].At).Seconds() - sum += diff * float64(samples[len(samples)-1].Value) return sum } -func getRate[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { - if len(samples) < 2 { - return 0.0 +func getRate[T int64 | uint32](it timeseries.ReverseIterator[T]) float64 { + var sum float64 + var first, last time.Time + for it.Next() { + if last.IsZero() { + last = it.Value().At + } + first = it.Value().At + sum += float64(it.Value().Value) } - sum := 0.0 - // start at 1 as the first sample duration is not available - for i := 1; i < len(samples); i++ { - sum += float64(samples[i].Value) + if duration := last.Sub(first); duration > 0 { + return sum / duration.Seconds() } - - duration := samples[len(samples)-1].At.Sub(samples[0].At) - if duration == 0 { - return 0.0 - } - - return sum / duration.Seconds() + return 0 } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 352b50c2e..e68d51717 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -85,8 +85,8 @@ const ( streamAllocatorSignalResume streamAllocatorSignalSetAllowPause streamAllocatorSignalSetChannelCapacity - streamAllocatorSignalNACK - streamAllocatorSignalRTCPReceiverReport + // STREAM-ALLOCATOR-DATA streamAllocatorSignalNACK + // STREAM-ALLOCATOR-DATA streamAllocatorSignalRTCPReceiverReport ) func (s streamAllocatorSignal) String() string { @@ -111,10 +111,12 @@ func (s streamAllocatorSignal) String() string { return "SET_ALLOW_PAUSE" case streamAllocatorSignalSetChannelCapacity: return "SET_CHANNEL_CAPACITY" - case streamAllocatorSignalNACK: - return "NACK" - case streamAllocatorSignalRTCPReceiverReport: - return "RTCP_RECEIVER_REPORT" + /* STREAM-ALLOCATOR-DATA + case streamAllocatorSignalNACK: + return "NACK" + case streamAllocatorSignalRTCPReceiverReport: + return "RTCP_RECEIVER_REPORT" + */ default: return fmt.Sprintf("%d", int(s)) } @@ -123,6 +125,7 @@ func (s streamAllocatorSignal) String() string { // --------------------------------------------------------------------------- type Event struct { + *StreamAllocator Signal streamAllocatorSignal TrackID livekit.TrackID Data interface{} @@ -157,7 +160,7 @@ type StreamAllocator struct { prober *Prober channelObserver *ChannelObserver - rateMonitor *RateMonitor + // STREAM-ALLOCATOR-DATA rateMonitor *RateMonitor videoTracksMu sync.RWMutex videoTracks map[livekit.TrackID]*Track @@ -166,7 +169,7 @@ type StreamAllocator struct { state streamAllocatorState - eventsQueue *utils.OpsQueue + eventsQueue *utils.TypedOpsQueue[Event] isStopped atomic.Bool } @@ -178,9 +181,9 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { prober: NewProber(ProberParams{ Logger: params.Logger, }), - rateMonitor: NewRateMonitor(), + // STREAM-ALLOCATOR-DATA rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + eventsQueue: utils.NewTypedOpsQueue[Event](utils.OpsQueueParams{ Name: "stream-allocator", MinSize: 64, Logger: params.Logger, @@ -241,10 +244,16 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) track.SetPriority(params.Priority) + trackID := livekit.TrackID(downTrack.ID()) s.videoTracksMu.Lock() - s.videoTracks[livekit.TrackID(downTrack.ID())] = track + oldTrack := s.videoTracks[trackID] + s.videoTracks[trackID] = track s.videoTracksMu.Unlock() + if oldTrack != nil { + oldTrack.DownTrack().SetStreamAllocatorListener(nil) + } + downTrack.SetStreamAllocatorListener(s) if s.prober.IsRunning() { // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate @@ -455,6 +464,7 @@ func (s *StreamAllocator) OnPacketsSent(downTrack *sfu.DownTrack, size int) { s.prober.PacketsSent(size) } +/* STREAM-ALLOCATOR-DATA // called by a video DownTrack when it processes NACKs func (s *StreamAllocator) OnNACK(downTrack *sfu.DownTrack, nackInfos []sfu.NackInfo) { s.postEvent(Event{ @@ -473,6 +483,7 @@ func (s *StreamAllocator) OnRTCPReceiverReport(downTrack *sfu.DownTrack, rr rtcp Data: rr, }) } +*/ // called when prober wants to send packet(s) func (s *StreamAllocator) OnSendProbe(bytesToSend int) { @@ -548,12 +559,6 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) } } -func (s *StreamAllocator) postEvent(event Event) { - s.eventsQueue.Enqueue(func() { - s.handleEvent(&event) - }) -} - func (s *StreamAllocator) ping() { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -570,36 +575,41 @@ func (s *StreamAllocator) ping() { } } -func (s *StreamAllocator) handleEvent(event *Event) { - switch event.Signal { - case streamAllocatorSignalAllocateTrack: - s.handleSignalAllocateTrack(event) - case streamAllocatorSignalAllocateAllTracks: - s.handleSignalAllocateAllTracks(event) - case streamAllocatorSignalAdjustState: - s.handleSignalAdjustState(event) - case streamAllocatorSignalEstimate: - s.handleSignalEstimate(event) - case streamAllocatorSignalPeriodicPing: - s.handleSignalPeriodicPing(event) - case streamAllocatorSignalSendProbe: - s.handleSignalSendProbe(event) - case streamAllocatorSignalProbeClusterDone: - s.handleSignalProbeClusterDone(event) - case streamAllocatorSignalResume: - s.handleSignalResume(event) - case streamAllocatorSignalSetAllowPause: - s.handleSignalSetAllowPause(event) - case streamAllocatorSignalSetChannelCapacity: - s.handleSignalSetChannelCapacity(event) - case streamAllocatorSignalNACK: - s.handleSignalNACK(event) - case streamAllocatorSignalRTCPReceiverReport: - s.handleSignalRTCPReceiverReport(event) - } +func (s *StreamAllocator) postEvent(event Event) { + event.StreamAllocator = s + s.eventsQueue.Enqueue(func(event Event) { + switch event.Signal { + case streamAllocatorSignalAllocateTrack: + event.handleSignalAllocateTrack(event) + case streamAllocatorSignalAllocateAllTracks: + event.handleSignalAllocateAllTracks(event) + case streamAllocatorSignalAdjustState: + event.handleSignalAdjustState(event) + case streamAllocatorSignalEstimate: + event.handleSignalEstimate(event) + case streamAllocatorSignalPeriodicPing: + event.handleSignalPeriodicPing(event) + case streamAllocatorSignalSendProbe: + event.handleSignalSendProbe(event) + case streamAllocatorSignalProbeClusterDone: + event.handleSignalProbeClusterDone(event) + case streamAllocatorSignalResume: + event.handleSignalResume(event) + case streamAllocatorSignalSetAllowPause: + event.handleSignalSetAllowPause(event) + case streamAllocatorSignalSetChannelCapacity: + event.handleSignalSetChannelCapacity(event) + /* STREAM-ALLOCATOR-DATA + case streamAllocatorSignalNACK: + event.s.handleSignalNACK(event) + case streamAllocatorSignalRTCPReceiverReport: + event.s.handleSignalRTCPReceiverReport(event) + */ + } + }, event) } -func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { +func (s *StreamAllocator) handleSignalAllocateTrack(event Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] if track != nil { @@ -612,7 +622,7 @@ func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { } } -func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { +func (s *StreamAllocator) handleSignalAllocateAllTracks(Event) { s.videoTracksMu.Lock() s.isAllocateAllPending = false s.videoTracksMu.Unlock() @@ -622,14 +632,14 @@ func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { } } -func (s *StreamAllocator) handleSignalAdjustState(event *Event) { +func (s *StreamAllocator) handleSignalAdjustState(Event) { s.adjustState() } -func (s *StreamAllocator) handleSignalEstimate(event *Event) { +func (s *StreamAllocator) handleSignalEstimate(event Event) { receivedEstimate, _ := event.Data.(int64) s.lastReceivedEstimate = receivedEstimate - s.monitorRate(receivedEstimate) + // s.monitorRate(receivedEstimate) // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails if s.probeController.IsInProbe() { @@ -639,7 +649,7 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { } } -func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { +func (s *StreamAllocator) handleSignalPeriodicPing(Event) { // finalize probe if necessary trend, _ := s.channelObserver.GetTrend() isHandled, isNotFailing, isGoalReached := s.probeController.MaybeFinalizeProbe( @@ -656,10 +666,10 @@ func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { s.maybeProbe() } - s.updateTracksHistory() + // s.updateTracksHistory() } -func (s *StreamAllocator) handleSignalSendProbe(event *Event) { +func (s *StreamAllocator) handleSignalSendProbe(event Event) { bytesToSend := event.Data.(int) if bytesToSend <= 0 { return @@ -680,12 +690,12 @@ func (s *StreamAllocator) handleSignalSendProbe(event *Event) { } } -func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { +func (s *StreamAllocator) handleSignalProbeClusterDone(event Event) { info, _ := event.Data.(ProbeClusterInfo) s.probeController.ProbeClusterDone(info) } -func (s *StreamAllocator) handleSignalResume(event *Event) { +func (s *StreamAllocator) handleSignalResume(event Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] s.videoTracksMu.Unlock() @@ -699,11 +709,11 @@ func (s *StreamAllocator) handleSignalResume(event *Event) { } } -func (s *StreamAllocator) handleSignalSetAllowPause(event *Event) { +func (s *StreamAllocator) handleSignalSetAllowPause(event Event) { s.allowPause = event.Data.(bool) } -func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { +func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) { s.overriddenChannelCapacity = event.Data.(int64) if s.overriddenChannelCapacity > 0 { s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) @@ -713,7 +723,8 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { } } -func (s *StreamAllocator) handleSignalNACK(event *Event) { +/* STREAM-ALLOCATOR-DATA +func (s *StreamAllocator) handleSignalNACK(event Event) { nackInfos := event.Data.([]sfu.NackInfo) s.videoTracksMu.Lock() @@ -725,7 +736,7 @@ func (s *StreamAllocator) handleSignalNACK(event *Event) { } } -func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { +func (s *StreamAllocator) handleSignalRTCPReceiverReport(event Event) { rr := event.Data.(rtcp.ReceptionReport) s.videoTracksMu.Lock() @@ -736,6 +747,7 @@ func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { track.ProcessRTCPReceiverReport(rr) } } +*/ func (s *StreamAllocator) setState(state streamAllocatorState) { if s.state == state { @@ -817,6 +829,7 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "commitThreshold(bps)", commitThreshold, "channel", s.channelObserver.ToString(), ) + /* STREAM-ALLOCATOR-DATA s.params.Logger.Debugw( fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity: experimental", action), "rateHistory", s.rateMonitor.GetHistory(), @@ -824,6 +837,7 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "nackHistory", s.channelObserver.GetNackHistory(), "trackHistory", s.getTracksHistory(), ) + */ if estimateToCommit > commitThreshold { // estimate to commit is either higher or within tolerance of expected uage, skip committing and re-allocating return @@ -1401,6 +1415,7 @@ func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { return maxDistanceSorter } +/* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO // Monitor sent rate vs estimate to figure out queuing on congestion. // Idea here is to pause all managed tracks on congestion detection immediately till queue drains. @@ -1443,6 +1458,7 @@ func (s *StreamAllocator) getTracksHistory() map[livekit.TrackID]string { return history } +*/ // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index be93b9e4b..1a0669585 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -15,14 +15,8 @@ package streamallocator import ( - "fmt" - "sort" - "time" - - "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/pion/rtcp" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -41,6 +35,7 @@ type Track struct { totalPackets uint32 totalRepeatedNacks uint32 + /* STREAM-ALLOCATOR-DATA nackInfos map[uint16]sfu.NackInfo // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental nackHistory []string @@ -53,6 +48,7 @@ type Track struct { maxRTT uint32 // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental receiverReportHistory []string + */ isDirty bool @@ -67,15 +63,17 @@ func NewTrack( logger logger.Logger, ) *Track { t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, + downTrack: downTrack, + source: source, + isSimulcast: isSimulcast, + publisherID: publisherID, + logger: logger, + /* STREAM-ALLOCATOR-DATA nackInfos: make(map[uint16]sfu.NackInfo), nackHistory: make([]string, 0, 10), receiverReportHistory: make([]string, 0, 10), - streamState: StreamStateInactive, + */ + streamState: StreamStateInactive, } t.SetPriority(0) t.SetMaxLayer(downTrack.MaxLayer()) @@ -220,6 +218,7 @@ func (t *Track) GetNackDelta() (uint32, uint32) { return packetDelta, nackDelta } +/* STREAM-ALLOCATOR-DATA func (t *Track) UpdateNack(nackInfos []sfu.NackInfo) { for _, ni := range nackInfos { t.nackInfos[ni.SequenceNumber] = ni @@ -363,6 +362,7 @@ func (t *Track) updateReceiverReportHistory() { fmt.Sprintf("t: %+v, l: %d, p: %d, rtt: %d", time.Now().Format(time.UnixDate), dl, dp, maxRTT), ) } +*/ // ------------------------------------------------ diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go index be8009eb1..29b876c70 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd.go +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -21,7 +21,7 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type StreamTrackerDependencyDescriptor struct { diff --git a/pkg/sfu/streamtracker/streamtracker_dd_test.go b/pkg/sfu/streamtracker/streamtracker_dd_test.go index f638e4e2d..8e3f4001c 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd_test.go +++ b/pkg/sfu/streamtracker/streamtracker_dd_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index b5f36dc97..0158c55c7 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -15,8 +15,6 @@ package sfu import ( - "fmt" - "math" "sort" "sync" "time" @@ -33,10 +31,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) -const ( - senderReportThresholdSeconds = float64(60.0) -) - // --------------------------------------------------- type StreamTrackerManagerListener interface { @@ -69,10 +63,6 @@ type StreamTrackerManager struct { maxExpectedLayer int32 paused bool - senderReportMu sync.RWMutex - senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData - layerOffsets [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerSpatial + 1]uint32 - closed core.Fuse listener StreamTrackerManagerListener @@ -550,110 +540,6 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } -func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { - srRef := s.senderReports[ref] - srOther := s.senderReports[other] - if srRef == nil || srRef.NTPTimestamp == 0 || srOther == nil || srOther.NTPTimestamp == 0 { - return - } - - ntpDiff := srRef.NTPTimestamp.Time().Sub(srOther.NTPTimestamp.Time()) - if math.Abs(ntpDiff.Seconds()) > senderReportThresholdSeconds { - // offset is updated only if the layers' sender reports are close enough. - // - // Rationale: higher layers could be paused for extended periods of time - // due to adaptive stream/dynacast or publisher constraints like CPU/bandwidth. - // The check is to avoid using very old reports. - return - } - rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 - - // calculate other layer's time stamp at the same time as ref layer's NTP time - normalizedOtherTS := srOther.RTPTimestamp + uint32(rtpDiff) - - // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers - offset := srRef.RTPTimestamp - normalizedOtherTS - - // use minimal offset to indicate value availability in the extremely unlikely case of - // both layers using the same timestamp - if offset == 0 { - s.logger.Debugw( - "using default offset", - "ref", ref, - "refNTP", srRef.NTPTimestamp.Time().String(), - "refRTP", srRef.RTPTimestamp, - "other", other, - "otherNTP", srOther.NTPTimestamp.Time().String(), - "otherRTP", srOther.RTPTimestamp, - ) - offset = 1 - } - - s.layerOffsets[ref][other] = offset -} - -func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srData *buffer.RTCPSenderReportData) { - s.senderReportMu.Lock() - defer s.senderReportMu.Unlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - return - } - - s.senderReports[layer] = srData - - // (re)fill offsets as necessary for received layer. - for i := int32(0); i < buffer.DefaultMaxLayerSpatial+1; i++ { - if i == layer { - continue - } - - // treating layer for which report was received as reference layer - s.updateLayerOffsetLocked(layer, i) - - // and the other way - s.updateLayerOffsetLocked(i, layer) - } -} - -func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - s.senderReportMu.Lock() - defer s.senderReportMu.Unlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - return nil - } - - // SVC-TODO: better SVC detection - if s.isSVC { - // there is only one stream in SVC - layer = 0 - } - - return s.senderReports[layer] -} - -func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - s.senderReportMu.RLock() - defer s.senderReportMu.RUnlock() - - if layer < 0 || int(layer) >= len(s.layerOffsets[0]) || referenceLayer < 0 || int(referenceLayer) >= len(s.layerOffsets) { - return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) - } - - // SVC-TODO: better SVC detection - if s.isSVC { - // there is only one stream in SVC - return ts, nil - } - - if layer != referenceLayer && s.layerOffsets[referenceLayer][layer] == 0 { - return 0, fmt.Errorf("offset unavailable, target: %d, reference: %d", layer, referenceLayer) - } - - return ts + s.layerOffsets[referenceLayer][layer], nil -} - func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { s.lock.RLock() defer s.lock.RUnlock() diff --git a/pkg/sfu/videolayerselector/decodetarget.go b/pkg/sfu/videolayerselector/decodetarget.go index 7204b329f..70114f21f 100644 --- a/pkg/sfu/videolayerselector/decodetarget.go +++ b/pkg/sfu/videolayerselector/decodetarget.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type DecodeTarget struct { diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 15d9df1eb..acfe5b85c 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -20,7 +20,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dede "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dede "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index 2b346dedb..a562988f9 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 7b226ed0c..836783864 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -15,7 +15,7 @@ package videolayerselector import ( - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1d0a9c546..1a5d22853 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -161,18 +161,14 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context, ) { t.enqueue(func() { isConnected := false - hasWorker := false if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok { - hasWorker = true isConnected = worker.IsConnected() + if worker.ClosedAt().IsZero() { + prometheus.SubParticipant() + } worker.Close() } - if hasWorker { - // signifies we had incremented participant count - prometheus.SubParticipant() - } - if isConnected && shouldSendEvent { t.NotifyEvent(ctx, &livekit.WebhookEvent{ Event: webhook.EventParticipantLeft, diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index ba6dd4507..703615693 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils/hwstats" ) const ( @@ -41,11 +42,13 @@ var ( sysDroppedPacketsStart uint32 promSysPacketGauge *prometheus.GaugeVec promSysDroppedPacketPctGauge prometheus.Gauge + + cpuStats *hwstats.CPUStats ) -func Init(nodeID string, nodeType livekit.NodeType, env string) { +func Init(nodeID string, nodeType livekit.NodeType) error { if initialized.Swap(true) { - return + return nil } MessageCounter = prometheus.NewCounterVec( @@ -53,7 +56,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "messages", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status"}, ) @@ -63,7 +66,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "service_operation", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status", "error_type"}, ) @@ -73,7 +76,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "twirp_request_status", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"service", "method", "status", "code"}, ) @@ -83,7 +86,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "packet_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level packet count. Count starts at 0 when service is first started.", }, []string{"type"}, @@ -94,7 +97,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level dropped outgoing packet percentage.", }, ) @@ -107,10 +110,18 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() - initPacketStats(nodeID, nodeType, env) - initRoomStats(nodeID, nodeType, env) - rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}) - initQualityStats(nodeID, nodeType, env) + initPacketStats(nodeID, nodeType) + initRoomStats(nodeID, nodeType) + rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}) + initQualityStats(nodeID, nodeType) + + var err error + cpuStats, err = hwstats.NewCPUStats(nil) + if err != nil { + return err + } + + return nil } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { @@ -119,9 +130,10 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats return nil, false, err } - cpuLoad, numCPUs, err := getCPUStats() - if err != nil { - return nil, false, err + var cpuLoad float64 + cpuIdle := cpuStats.GetCPUIdle() + if cpuIdle > 0 { + cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU()) } // On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin @@ -198,8 +210,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec, ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec, ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec, - NumCpus: numCPUs, - CpuLoad: cpuLoad, + NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer + CpuLoad: float32(cpuLoad), MemoryTotal: memTotal, MemoryUsed: memUsed, LoadAvgLast1Min: float32(loadAvg.Loadavg1), diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 6b643e903..3efd1eb28 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -67,55 +67,55 @@ var ( promPacketBytesOutgoingRetransmit prometheus.Counter ) -func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { +func initPacketStats(nodeID string, nodeType livekit.NodeType) { promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "nack", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "pli", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "fir", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promStreamLabels) promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "percent", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100}, }, promStreamLabels) promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "jitter", Name: "us", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, // 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000}, @@ -124,20 +124,20 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "rtt", Name: "ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000}, }, promStreamLabels) promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "participant_join", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state"}) promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "connection", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) prometheus.MustRegister(promPacketTotal) diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go index 481b7558f..c9eaf6b65 100644 --- a/pkg/telemetry/prometheus/quality.go +++ b/pkg/telemetry/prometheus/quality.go @@ -26,26 +26,26 @@ var ( qualityDrop *prometheus.CounterVec ) -func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) { +func initQualityStats(nodeID string, nodeType livekit.NodeType) { qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "rating", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0, 1, 2}, }) qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "score", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5}, }) qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "drop", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"direction"}) prometheus.MustRegister(qualityRating) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index 91c7cc2ba..6c7540d78 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -47,18 +47,18 @@ var ( promSessionStartTime *prometheus.HistogramVec ) -func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { +func initRoomStats(nodeID string, nodeType livekit.NodeType) { promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "duration_seconds", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{ 5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60, }, @@ -67,37 +67,37 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "participant", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "published_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribed_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "publish_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind", "state"}) promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribe_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state", "error"}) promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "session", Name: "start_time_ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15), }, []string{"protocol_version"}) diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index 086192532..1ec931563 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -162,7 +162,10 @@ func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { s.mu.Lock() defer s.mu.Unlock() if s.ri == nil && ri.GetSid() != "" { - s.ri = ri + s.ri = &livekit.Room{ + Sid: ri.Sid, + Name: ri.Name, + } s.maybeStart() } } @@ -170,8 +173,11 @@ func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) { s.mu.Lock() defer s.mu.Unlock() - if s.pi == nil { - s.pi = pi + if s.pi == nil && pi != nil { + s.pi = &livekit.ParticipantInfo{ + Sid: pi.Sid, + Identity: pi.Identity, + } s.maybeStart() } } diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 1758c8b45..89051ee92 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -29,7 +29,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } type telemetryServiceFixture struct { diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index d48a6d6d5..18bde9e55 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -158,6 +158,8 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } // find aggregates across streams + startTime := time.Time{} + endTime := time.Time{} scoreSum := float32(0.0) // used for average minScore := float32(0.0) // min score in batched stats var scores []float32 // used for median @@ -183,6 +185,16 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } for _, analyticsStream := range stat.Streams { + start := analyticsStream.StartTime.AsTime() + if startTime.IsZero() || startTime.After(start) { + startTime = start + } + + end := analyticsStream.EndTime.AsTime() + if endTime.IsZero() || endTime.Before(end) { + endTime = end + } + if analyticsStream.Rtt > maxRtt { maxRtt = analyticsStream.Rtt } @@ -216,6 +228,8 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } } } + coalescedStream.StartTime = timestamppb.New(startTime) + coalescedStream.EndTime = timestamppb.New(endTime) coalescedStream.Rtt = maxRtt coalescedStream.Jitter = maxJitter diff --git a/pkg/testutils/timeout.go b/pkg/testutils/timeout.go index 11debef92..7745b2edc 100644 --- a/pkg/testutils/timeout.go +++ b/pkg/testutils/timeout.go @@ -24,15 +24,19 @@ var ( ConnectTimeout = 30 * time.Second ) -func WithTimeout(t *testing.T, f func() string) { - ctx, cancel := context.WithTimeout(context.Background(), ConnectTimeout) +func WithTimeout(t *testing.T, f func() string, timeouts ...time.Duration) { + timeout := ConnectTimeout + if len(timeouts) > 0 { + timeout = timeouts[0] + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() lastErr := "" for { select { case <-ctx.Done(): if lastErr != "" { - t.Fatalf("did not reach expected state after %v: %s", ConnectTimeout, lastErr) + t.Fatalf("did not reach expected state after %v: %s", timeout, lastErr) } case <-time.After(10 * time.Millisecond): lastErr = f() diff --git a/pkg/utils/iceconfigcache.go b/pkg/utils/iceconfigcache.go new file mode 100644 index 000000000..e42ed2bb6 --- /dev/null +++ b/pkg/utils/iceconfigcache.go @@ -0,0 +1,42 @@ +package utils + +import ( + "time" + + "github.com/jellydator/ttlcache/v3" + + "github.com/livekit/protocol/livekit" +) + +const ( + iceConfigTTLMin = 5 * time.Minute +) + +type IceConfigCache[T comparable] struct { + c *ttlcache.Cache[T, *livekit.ICEConfig] +} + +func NewIceConfigCache[T comparable](ttl time.Duration) *IceConfigCache[T] { + cache := ttlcache.New( + ttlcache.WithTTL[T, *livekit.ICEConfig](max(ttl, iceConfigTTLMin)), + ttlcache.WithDisableTouchOnHit[T, *livekit.ICEConfig](), + ) + go cache.Start() + + return &IceConfigCache[T]{cache} +} + +func (icc *IceConfigCache[T]) Stop() { + icc.c.Stop() +} + +func (icc *IceConfigCache[T]) Put(key T, iceConfig *livekit.ICEConfig) { + icc.c.Set(key, iceConfig, ttlcache.DefaultTTL) +} + +func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { + if it := icc.c.Get(key); it != nil { + return it.Value() + } + return &livekit.ICEConfig{} +} diff --git a/pkg/utils/iceconfigcache_test.go b/pkg/utils/iceconfigcache_test.go new file mode 100644 index 000000000..78feebb0e --- /dev/null +++ b/pkg/utils/iceconfigcache_test.go @@ -0,0 +1,18 @@ +package utils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" +) + +func TestIceConfigCache(t *testing.T) { + cache := NewIceConfigCache[string](10 * time.Second) + t.Cleanup(cache.Stop) + + cache.Put("test", &livekit.ICEConfig{}) + require.NotNil(t, cache) +} diff --git a/pkg/utils/incrementaldispatcher.go b/pkg/utils/incrementaldispatcher.go new file mode 100644 index 000000000..2aaaf9ffd --- /dev/null +++ b/pkg/utils/incrementaldispatcher.go @@ -0,0 +1,86 @@ +/* + * Copyright 2024 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "sync" + + "github.com/frostbyte73/core" +) + +// IncrementalDispatcher is a dispatcher that allows multiple consumers to consume items as they become +// available, while producers can add items at anytime. +type IncrementalDispatcher[T any] struct { + done core.Fuse + lock sync.RWMutex + cond *sync.Cond + items []T +} + +func NewIncrementalDispatcher[T any]() *IncrementalDispatcher[T] { + p := &IncrementalDispatcher[T]{} + p.cond = sync.NewCond(&p.lock) + return p +} + +func (d *IncrementalDispatcher[T]) Add(item T) { + if d.done.IsBroken() { + return + } + d.lock.Lock() + d.items = append(d.items, item) + d.lock.Unlock() + d.cond.Broadcast() +} + +func (d *IncrementalDispatcher[T]) Done() { + d.lock.Lock() + d.done.Break() + d.cond.Broadcast() + d.lock.Unlock() +} + +func (d *IncrementalDispatcher[T]) ForEach(fn func(T)) { + idx := 0 + dispatchFromIdx := func() { + var itemsToDispatch []T + d.lock.RLock() + for idx < len(d.items) { + itemsToDispatch = append(itemsToDispatch, d.items[idx]) + idx++ + } + d.lock.RUnlock() + for _, item := range itemsToDispatch { + fn(item) + } + } + for !d.done.IsBroken() { + dispatchFromIdx() + d.lock.Lock() + // need to check again because Done may have been called while dispatching + if d.done.IsBroken() { + d.lock.Unlock() + break + } + if idx == len(d.items) { + d.cond.Wait() + } + d.lock.Unlock() + } + + dispatchFromIdx() +} diff --git a/pkg/utils/incrementaldispatcher_test.go b/pkg/utils/incrementaldispatcher_test.go new file mode 100644 index 000000000..828019559 --- /dev/null +++ b/pkg/utils/incrementaldispatcher_test.go @@ -0,0 +1,96 @@ +/* + * Copyright 2024 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils_test + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/livekit-server/pkg/utils" +) + +func TestForEach(t *testing.T) { + producer := utils.NewIncrementalDispatcher[int]() + go func() { + defer producer.Done() + producer.Add(1) + producer.Add(2) + producer.Add(3) + }() + + sum := 0 + producer.ForEach(func(item int) { + sum += item + }) + + require.Equal(t, 6, sum) +} + +func TestConcurrentConsumption(t *testing.T) { + producer := utils.NewIncrementalDispatcher[int]() + numConsumers := 100 + sums := make([]atomic.Int32, numConsumers) + var wg sync.WaitGroup + + for i := 0; i < numConsumers; i++ { + wg.Add(1) + i := i + go func() { + defer wg.Done() + producer.ForEach(func(item int) { + sums[i].Add(int32(item)) + }) + }() + } + + // Add items + expectedSum := 0 + for i := 0; i < 20; i++ { + expectedSum += i + producer.Add(i) + } + + for i := 0; i < numConsumers; i++ { + testutils.WithTimeout(t, func() string { + if sums[i].Load() != int32(expectedSum) { + return fmt.Sprintf("consumer %d did not consume all the items. expected %d, actual: %d", + i, expectedSum, sums[i].Load()) + } + return "" + }, time.Second) + } + + // keep adding and ensure it's consumed + for i := 20; i < 30; i++ { + expectedSum += i + producer.Add(i) + } + + // wait for all consumers to finish + producer.Done() + wg.Wait() + + for i := 0; i < numConsumers; i++ { + require.Equal(t, int32(expectedSum), sums[i].Load(), "consumer %d did not match", i) + } +} diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index bda4ceea1..c4e00e4ec 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -19,8 +19,8 @@ import ( "sync" "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" ) type OpsQueueParams struct { @@ -30,28 +30,66 @@ type OpsQueueParams struct { Logger logger.Logger } +type UntypedQueueOp func() + +func (op UntypedQueueOp) run() { + op() +} + type OpsQueue struct { + opsQueueBase[UntypedQueueOp] +} + +func NewOpsQueue(params OpsQueueParams) *OpsQueue { + return &OpsQueue{*newOpsQueueBase[UntypedQueueOp](params)} +} + +type typedQueueOp[T any] struct { + fn func(T) + arg T +} + +func (op typedQueueOp[T]) run() { + op.fn(op.arg) +} + +type TypedOpsQueue[T any] struct { + opsQueueBase[typedQueueOp[T]] +} + +func NewTypedOpsQueue[T any](params OpsQueueParams) *TypedOpsQueue[T] { + return &TypedOpsQueue[T]{*newOpsQueueBase[typedQueueOp[T]](params)} +} + +func (oq *TypedOpsQueue[T]) Enqueue(fn func(T), arg T) { + oq.opsQueueBase.Enqueue(typedQueueOp[T]{fn, arg}) +} + +type opsQueueItem interface { + run() +} + +type opsQueueBase[T opsQueueItem] struct { params OpsQueueParams lock sync.Mutex - ops deque.Deque[func()] + ops deque.Deque[T] wake chan struct{} isStarted bool doneChan chan struct{} isStopped bool } -func NewOpsQueue(params OpsQueueParams) *OpsQueue { - oq := &OpsQueue{ +func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *opsQueueBase[T] { + return &opsQueueBase[T]{ params: params, + ops: *deque.New[T](min(bits.Len64(uint64(params.MinSize-1)), 7)), wake: make(chan struct{}, 1), doneChan: make(chan struct{}), } - oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(oq.params.MinSize-1)), 7))) - return oq } -func (oq *OpsQueue) Start() { +func (oq *opsQueueBase[T]) Start() { oq.lock.Lock() if oq.isStarted { oq.lock.Unlock() @@ -64,7 +102,7 @@ func (oq *OpsQueue) Start() { go oq.process() } -func (oq *OpsQueue) Stop() <-chan struct{} { +func (oq *opsQueueBase[T]) Stop() <-chan struct{} { oq.lock.Lock() if oq.isStopped { oq.lock.Unlock() @@ -77,7 +115,7 @@ func (oq *OpsQueue) Stop() <-chan struct{} { return oq.doneChan } -func (oq *OpsQueue) Enqueue(op func()) { +func (oq *opsQueueBase[T]) Enqueue(op T) { oq.lock.Lock() defer oq.lock.Unlock() @@ -94,7 +132,7 @@ func (oq *OpsQueue) Enqueue(op func()) { } } -func (oq *OpsQueue) process() { +func (oq *opsQueueBase[T]) process() { defer close(oq.doneChan) for { @@ -113,7 +151,7 @@ func (oq *OpsQueue) process() { op := oq.ops.PopFront() oq.lock.Unlock() - op() + op.run() } } } diff --git a/test/agent.go b/test/agent.go index cc04cad84..244084ffb 100644 --- a/test/agent.go +++ b/test/agent.go @@ -25,7 +25,6 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/utils" ) type agentClient struct { @@ -38,11 +37,13 @@ type agentClient struct { participantAvailability atomic.Int32 participantJobs atomic.Int32 + requestedJobs chan *livekit.Job + done chan struct{} } -func newAgentClient(token string) (*agentClient, error) { - host := fmt.Sprintf("ws://localhost:%d", defaultServerPort) +func newAgentClient(token string, port uint32) (*agentClient, error) { + host := fmt.Sprintf("ws://localhost:%d", port) u, err := url.Parse(host + "/agent") if err != nil { return nil, err @@ -57,25 +58,24 @@ func newAgentClient(token string) (*agentClient, error) { } return &agentClient{ - conn: conn, - done: make(chan struct{}), + conn: conn, + requestedJobs: make(chan *livekit.Job, 100), + done: make(chan struct{}), }, nil } -func (c *agentClient) Run(jobType livekit.JobType) (err error) { +func (c *agentClient) Run(jobType livekit.JobType, namespace string) (err error) { go c.read() - workerID := utils.NewGuid("W_") - switch jobType { case livekit.JobType_JT_ROOM: err = c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Register{ Register: &livekit.RegisterWorkerRequest{ - Type: livekit.JobType_JT_ROOM, - WorkerId: workerID, - Version: "version", - Name: "name", + Type: livekit.JobType_JT_ROOM, + Version: "version", + Namespace: &namespace, + Name: "name", }, }, }) @@ -84,10 +84,10 @@ func (c *agentClient) Run(jobType livekit.JobType) (err error) { err = c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Register{ Register: &livekit.RegisterWorkerRequest{ - Type: livekit.JobType_JT_PUBLISHER, - WorkerId: workerID, - Version: "version", - Name: "name", + Type: livekit.JobType_JT_PUBLISHER, + Version: "version", + Namespace: &namespace, + Name: "name", }, }, }) @@ -139,6 +139,8 @@ func (c *agentClient) handleAvailability(req *livekit.AvailabilityRequest) { c.participantAvailability.Inc() } + c.requestedJobs <- req.Job + c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Availability{ Availability: &livekit.AvailabilityResponse{ @@ -157,16 +159,23 @@ func (c *agentClient) write(msg *livekit.WorkerMessage) error { c.mu.Lock() defer c.mu.Unlock() - b, err := proto.Marshal(msg) - if err != nil { - return err - } + select { + case <-c.done: + return nil + default: + b, err := proto.Marshal(msg) + if err != nil { + return err + } - return c.conn.WriteMessage(websocket.BinaryMessage, b) + return c.conn.WriteMessage(websocket.BinaryMessage, b) + } } func (c *agentClient) close() { + c.mu.Lock() close(c.done) _ = c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) _ = c.conn.Close() + c.mu.Unlock() } diff --git a/test/agent_test.go b/test/agent_test.go index a8825577b..88e44b19c 100644 --- a/test/agent_test.go +++ b/test/agent_test.go @@ -15,72 +15,184 @@ package test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/testutils" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" ) +var ( + RegisterTimeout = 2 * time.Second + AssignJobTimeout = 3 * time.Second +) + func TestAgents(t *testing.T) { _, finish := setupSingleNodeTest("TestAgents") defer finish() - ac1, err := newAgentClient(agentToken()) + + ac1, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac2, err := newAgentClient(agentToken()) + ac2, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac3, err := newAgentClient(agentToken()) + ac3, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac4, err := newAgentClient(agentToken()) + ac4, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) defer ac1.close() defer ac2.close() defer ac3.close() defer ac4.close() - ac1.Run(livekit.JobType_JT_ROOM) - ac2.Run(livekit.JobType_JT_ROOM) - ac3.Run(livekit.JobType_JT_PUBLISHER) - ac4.Run(livekit.JobType_JT_PUBLISHER) + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_ROOM, "namespace") + ac3.Run(livekit.JobType_JT_PUBLISHER, "namespace") + ac4.Run(livekit.JobType_JT_PUBLISHER, "namespace") - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 { + return "worker not registered" + } - require.Equal(t, int32(1), ac1.registered.Load()) - require.Equal(t, int32(1), ac2.registered.Load()) - require.Equal(t, int32(1), ac3.registered.Load()) - require.Equal(t, int32(1), ac4.registered.Load()) + return "" + }, RegisterTimeout) c1 := createRTCClient("c1", defaultServerPort, nil) c2 := createRTCClient("c2", defaultServerPort, nil) waitUntilConnected(t, c1, c2) // publish 2 tracks - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") require.NoError(t, err) defer t1.Stop() t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") require.NoError(t, err) defer t2.Stop() - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } - require.Equal(t, int32(1), ac1.roomJobs.Load()+ac2.roomJobs.Load()) - require.Equal(t, int32(1), ac3.participantJobs.Load()+ac4.participantJobs.Load()) + if ac3.participantJobs.Load()+ac4.participantJobs.Load() != 1 { + return fmt.Sprintf("participant jobs not assigned, ac3: %d, ac4: %d", ac3.participantJobs.Load(), ac4.participantJobs.Load()) + } + + return "" + }, 6 * time.Second) // publish 2 tracks - t3, err := c2.AddStaticTrack("audio/opus", "audio", "webcam") + t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro") require.NoError(t, err) defer t3.Stop() t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam") require.NoError(t, err) defer t4.Stop() - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job must be assigned 1 time" + } - require.Equal(t, int32(1), ac1.roomJobs.Load()+ac2.roomJobs.Load()) - require.Equal(t, int32(2), ac3.participantJobs.Load()+ac4.participantJobs.Load()) + if ac3.participantJobs.Load()+ac4.participantJobs.Load() != 2 { + return "2 publisher jobs must assigned" + } + + return "" + }, AssignJobTimeout) +} + +func TestAgentNamespaces(t *testing.T) { + _, finish := setupSingleNodeTest("TestAgentNamespaces") + defer finish() + + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_ROOM, "namespace2") + + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) + + c1 := createRTCClient("c1", defaultServerPort, nil) + waitUntilConnected(t, c1) + + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } + + job1 := <-ac1.requestedJobs + job2 := <-ac2.requestedJobs + + if job1.Namespace != "namespace" { + return "namespace is not 'namespace'" + } + + if job2.Namespace != "namespace2" { + return "namespace is not 'namespace2'" + } + + if job1.Id == job2.Id { + return "job ids are the same" + } + + return "" + }, AssignJobTimeout) + +} + +func TestAgentMultiNode(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestAgentMultiNode") + defer finish() + + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_PUBLISHER, "namespace") + + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) + + c1 := createRTCClient("c1", secondServerPort, nil) // Create a room on the second node + waitUntilConnected(t, c1) + + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") + require.NoError(t, err) + defer t1.Stop() + + time.Sleep(time.Second * 10) + + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 { + return "room job not assigned" + } + + if ac2.participantJobs.Load() != 1 { + return "participant job not assigned" + } + + return "" + }, AssignJobTimeout) } func agentToken() string { diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 0ce079c23..81db35ed4 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -58,7 +58,7 @@ var roomClient livekit.RoomService func init() { config.InitLoggerFromConfig(&config.DefaultConfig.Logging) - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } func setupSingleNodeTest(name string) (*service.LivekitServer, func()) { diff --git a/version/version.go b/version/version.go index 17d23954c..96df3705f 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.5.3" +const Version = "1.6.0"