diff --git a/.github/banner_dark.png b/.github/banner_dark.png new file mode 100644 index 000000000..d19c687ce Binary files /dev/null and b/.github/banner_dark.png differ diff --git a/.github/banner_light.png b/.github/banner_light.png new file mode 100644 index 000000000..788b1f94b Binary files /dev/null and b/.github/banner_light.png differ diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 4025f7fbb..13719e47c 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Test on: @@ -19,7 +33,7 @@ jobs: - run: redis-cli ping - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '1.18' diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 0fa3a1749..e08c7f1ad 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Release to Docker # Controls when the action will run. @@ -25,7 +39,7 @@ jobs: type=semver,pattern=v{{major}}.{{minor}} - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '>=1.18' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d5ed958e6..7e0110d0a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Release on: @@ -19,7 +33,7 @@ jobs: run: git fetch --force --tags - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '>=1.18' diff --git a/.gitignore b/.gitignore index b44d8bdac..dec7e8aad 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,6 @@ proto/ .DS_Store # IDE -.idea +.idea/ dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4cd59d959..f8e54c4e1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + before: hooks: - go mod tidy diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e095..000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 239034913..000000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/livekit-server.iml b/.idea/livekit-server.iml deleted file mode 100644 index adec66b98..000000000 --- a/.idea/livekit-server.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 9c32c0523..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/protoeditor.xml b/.idea/protoeditor.xml deleted file mode 100644 index 6f5d7aadf..000000000 --- a/.idea/protoeditor.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 28b5fb8be..f4893fd8c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,148 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.4] - 2023-07-08 + +### Added +- Add dependency descriptor stream tracker for svc codecs (#1788) +- Full reconnect on publication mismatch on resume. (#1823) +- Pacer interface in down stream path. (#1835) +- retry egress on timeout/resource exhausted (#1852) + +### Fixed +- Send Room metadata updates immediately after update (#1787) +- Do not send ParticipantJoined webhook if connection was resumed (#1795) +- Reduce memory leaks by avoiding references in closure. (#1809) +- Honor bind address passed as `--bind` also for RTC ports (#1815) +- Avoid dangling downtracks by always deleting them in receiver close. (#1842) +- Better cleanup of subscriptions with needsCleanup. (#1845) +- Fix nack issue for svc codecs (#1856) +- Fixed hidden participant update were still sent when track is published (#1857) +- Fixed Redis lockup when unlocking room with canceled request context (#1859) + +### Changed +- Improvements to A/V sync (#1773 #1781 #1784 ) +- Improved probing to be less disruptive in low bandwidth scenarios (#1782 #1834 #1839) +- Do not mute forwarder when paused due to bandwidth congestion. (#1796) +- Improvements to congestion controller (#1800 #1802 ) +- Close participant on full reconnect. (#1818) +- Do not process events after participant close. (#1824) +- Improvements to dependency descriptor based selection forwarder (#1808) +- Discount out-of-order packets in downstream score. (#1831) +- Adaptive stream to select highest layer of equal dimensions (#1841) +- Return 404 with DeleteRoom/RemoveParticipant when deleting non-existent resources (#1860) + +## [1.4.3] - 2023-06-03 + +### Added +- Send quality stats to prometheus. (#1708) +- Support for disabling publishing codec on specific devices (#1728) +- Add support for bypass_transcoding field in ingress (#1741) +- Include await_start_signal for Web Egress (#1759) + +### Fixed +- Handle time stamp increment across mute for A/V sync (#1705) +- Additional A/V sync improvements (#1712 #1724 #1737 #1738 #1764) +- Check egress status on UpdateStream failure (#1716) +- Start signal relay sessions with the correct node (#1721) +- Fix unwrap for out-of-order packet (#1729) +- Fix dynacast for svc codec (#1742 #1743) +- Ignore receiver reports that have a sequence number before first packet (#1745) +- Fix node stats updates on Windows (#1748) +- Avoid reconnect loop for unsupported downtrack (#1754) +- Perform unsubscribe in parallel to avoid blocking (#1760) + +### Changed +- Make signal close async. (#1711 #1722) +- Don't add nack if it is already present in track codec (#1714) +- Tweaked connection quality algorithm to be less sensitive to jitter (#1719) +- Adjust sender report time stamp for slow publishers (#1740) +- Split probe controller from StreamAllocator (#1751) + +## [1.4.2] - 2023-04-27 +### Added +- VP9 codec with SVC support (#1586) +- Support for source-specific permissions and client-initiated metadata updates (#1590) +- Batch support for signal relay (#1593 #1596) +- Support for simulating subscriber bandwidth (#1609) +- Support for subscription limits (#1629) +- Send Room updates when participant counts change (#1647) + +### Fixed +- Fixed process return code to 0 (#1589) +- Fixed VP9 stutter when not using dependency descriptors (#1595) +- Fixed stutter when using dependency descriptors (#1600) +- Fixed Redis cluster support when using Egress or Ingress (#1606) +- Fixed simulcast parsing error for slower clients (camera and screenshare) (#1621) +- Don't close RTCP reader if Downtrack will be resumed (#1632) +- Restore VP8 munger state properly. (#1634) +- Fixed incorrect node routing when using signal relay (#1645) +- Do not send hidden participants to others after resume (#1689) +- Fix for potential webhook delivery delays (#1690) + +### Changed +- Refactored video layer selector (#1588 #1591 #1592) +- Improved transport fallback when client is resuming (#1597) +- Improved webhook reliability with delivery retries (#1607 #1615) +- Congestion controller improvements (#1614 #1616 #1617 #1623 #1628 #1631 #1652) +- Reduced memory usage by releasing ParticipantInfo after JoinResponse is transmitted (#1619) +- Ensure safe access in sequencer (#1625) +- Run quality scorer when there are no streams. (#1633) +- Participant version is only incremented after updates (#1646) +- Connection quality attribution improvements (#1653 #1664) +- Remove disallowed subscriptions on close. (#1668) +- A/V sync improvements (#1681 #1684 #1687 #1693 #1695 #1696 #1698 #1704) +- RTCP sender reports every three seconds. (#1692) + +### Removed +- Remove deprecated (non-psrpc) egress client (#1701) + +## [1.4.1] - 2023-04-05 +### Added +- Added prometheus metrics for internal signaling API #1571 + +### Fixed +- Fix regressions in RTC when using redis with psrpc signaling #1584 #1582 #1580 #1567 +- Fix required bitrate assessment under channel congestion #1577 + +### Changed +- Improve DTLS reliability in regions with internet filters #1568 +- Reduce memory usage from logging #1576 + +## [1.4.0] - 2023-03-27 +### Added +- Added config to disable active RED encoding. Use NACK instead #1476 #1477 +- Added option to skip TCP fallback if TCP RTT is high #1484 +- psrpc based signaling between signal and RTC #1485 +- Connection quality algorithm revamp #1490 #1491 #1493 #1496 #1497 #1500 #1505 #1507 #1509 #1516 #1520 #1521 #1527 #1528 #1536 +- Support for topics in data channel messages #1489 +- Added active filter to ListEgress #1517 +- Handling for React Native and Rust SDK ClientInfo #1544 + +### Fixed +- Fixed unsubscribed speakers stuck as speaking to clients #1475 +- Do not include packet in RED if timestamp is too far back #1478 +- Prevent PLI layer lock getting stuck #1481 +- Fix a case of changing video quality not succeeding #1483 +- Resync on pub muted for audio to avoid jump in sequence numbers on unmute #1487 +- Fixed a case of data race #1492 +- Inform reconnecting participant about recently disconnected users #1495 +- Send room update that may be missed by reconnected participant #1499 +- Fixed regression for AV1 forwarding #1538 +- Ensure sequence number continuity #1539 +- Give proper grace period when recorder is still in the room #1547 +- Fix sequence number offset on packet drop #1556 +- Fix signal client message buffer size #1561 + +### Changed +- Reduce lock scope getting RTCP sender reports #1473 +- Avoid duplicate queueReconcile in subscription manager #1474 +- Do not log TURN errors with prefix "error when handling datagram" #1494 +- Improvements to TCP fallback mode #1498 +- Unify forwarder between dependency descriptor and no DD case. #1543 +- Increase sequence number cache to handle high rate tracks #1560 + + ## [1.3.5] - 2023-02-25 ### Added - Allow for strict ACKs to be disabled or subscriber peer connections #1410 diff --git a/Dockerfile b/Dockerfile index a02bbb617..f35b65ada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM golang:1.20-alpine as builder ARG TARGETPLATFORM diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..692adc992 --- /dev/null +++ b/NOTICE @@ -0,0 +1,13 @@ +Copyright 2023 LiveKit, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index d3aad5166..3692b94e5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,15 @@ -# LiveKit: High-performance WebRTC + + + + + The LiveKit icon, the name of the repository and some sample code in the background. + + -LiveKit is an open source project that provides scalable, multi-user conferencing based on WebRTC. It's designed to -provide everything you need to build real-time video/audio/data capabilities in your applications. +# LiveKit: Real-time video, audio and data for developers + +[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. +It's designed to provide everything you need to build real-time video audio data capabilities in your applications. LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -9,7 +17,7 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu [![Slack community](https://img.shields.io/endpoint?url=https%3A%2F%2Flivekit.io%2Fbadges%2Fslack)](https://livekit.io/join-slack) [![Twitter Follow](https://img.shields.io/twitter/follow/livekitted)](https://twitter.com/livekitted) [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/livekit/livekit)](https://github.com/livekit/livekit/releases/latest) -[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/livekit/livekit/Test)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/livekit/livekit/buildtest.yaml?branch=master)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) [![License](https://img.shields.io/github/license/livekit/livekit)](https://github.com/livekit/livekit/blob/master/LICENSE) ## Features @@ -32,10 +40,12 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu https://docs.livekit.io -## Try it live +## Live Demos -Head to [our playground](https://livekit.io/playground) and give it a spin. Build a Zoom-like conferencing app in under -100 lines of code! +- [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit-examples/meet)) +- [Spatial Audio](https://spatial-audio-demo.livekit.io/) ([source](https://github.com/livekit-examples/spatial-audio)) +- Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) +- [AI voice assistant using ChatGPT](https://livekit.io/kitt) ([source](https://github.com/livekit-examples/kitt)) ## SDKs & Tools @@ -107,8 +117,9 @@ Client SDKs enable your frontend to include interactive, multi-user experiences. Compose example + - Flutter + Flutter (all platforms) client-sdk-flutter @@ -139,6 +150,15 @@ Client SDKs enable your frontend to include interactive, multi-user experiences. native + + + Rust + + client-sdk-rust + + + + ### Server SDKs @@ -159,8 +179,9 @@ enabling you to build automations that behave like end-users. ### Ecosystem & Tools -- [Egress](https://github.com/livekit/egress) - export and record your rooms - [CLI](https://github.com/livekit/livekit-cli) - command line interface & load tester +- [Egress](https://github.com/livekit/egress) - export and record your rooms +- [Ingress](https://github.com/livekit/ingress) - ingest streams from RTMP / OBS Studio - [Docker image](https://hub.docker.com/r/livekit/livekit-server) - [Helm charts](https://github.com/livekit/livekit-helm) @@ -235,11 +256,14 @@ simulation. ## Deployment ### Use LiveKit Cloud -LiveKit Cloud is the fastest and most reliable way to run LiveKit. Every project gets free monthly bandwidth and transcoding credits. + +LiveKit Cloud is the fastest and most reliable way to run LiveKit. Every project gets free monthly bandwidth and +transcoding credits. Sign up for [LiveKit Cloud](https://cloud.livekit.io/). ### Self-host + Read our [deployment docs](https://docs.livekit.io/deploy/) for more information. ## Building from source @@ -266,3 +290,15 @@ We welcome your contributions toward improving LiveKit! Please join us ## License LiveKit server is licensed under Apache License v2.0. + + +
+ + + + + + + +
LiveKit Ecosystem
Client SDKsComponents · JavaScript · iOS/macOS · Android · Flutter · React Native · Rust · Python · Unity (web) · Unity (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin · PHP (community) · Python (community)
ServicesLivekit server · Egress · Ingress
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI
+ diff --git a/bootstrap.sh b/bootstrap.sh index 4d7085f29..3109ddc38 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + if ! command -v mage &> /dev/null then diff --git a/cmd/server/commands.go b/cmd/server/commands.go index 927668a0a..99499a50f 100644 --- a/cmd/server/commands.go +++ b/cmd/server/commands.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/cmd/server/main.go b/cmd/server/main.go index 994e71b34..58664107c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,8 +1,23 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( "fmt" "math/rand" + "net" "os" "os/signal" "runtime" @@ -12,7 +27,6 @@ import ( "github.com/urfave/cli/v2" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/logger" @@ -103,8 +117,9 @@ func init() { func main() { defer func() { - rtc.Recover(logger.GetLogger()) - os.Exit(1) + if rtc.Recover(logger.GetLogger()) != nil { + os.Exit(1) + } }() generatedFlags, err := config.GenerateCLIFlags(baseFlags, true) @@ -187,7 +202,7 @@ func getConfig(c *cli.Context) (*config.Config, error) { if err != nil { return nil, err } - serverlogger.InitFromConfig(conf.Logging) + config.InitLoggerFromConfig(conf.Logging) if c.String("config") == "" && c.String("config-body") == "" && conf.Development { // use single port UDP when no config is provided @@ -204,11 +219,26 @@ func getConfig(c *cli.Context) (*config.Config, error) { conf.Keys = map[string]string{ "devkey": "secret", } + shouldMatchRTCIP := false // when dev mode and using shared keys, we'll bind to localhost by default if conf.BindAddresses == nil { conf.BindAddresses = []string{ "127.0.0.1", - "[::1]", + "::1", + } + } else { + // if non-loopback addresses are provided, then we'll match RTC IP to bind address + // our IP discovery ignores loopback addresses + for _, addr := range conf.BindAddresses { + ip := net.ParseIP(addr) + if ip != nil && !ip.IsLoopback() && !ip.IsUnspecified() { + shouldMatchRTCIP = true + } + } + } + if shouldMatchRTCIP { + for _, bindAddr := range conf.BindAddresses { + conf.RTC.IPs.Includes = append(conf.RTC.IPs.Includes, bindAddr+"/24") } } } diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go index 931692a64..dc82bf086 100644 --- a/cmd/server/main_test.go +++ b/cmd/server/main_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/config-sample.yaml b/config-sample.yaml index 8cc964656..bdf63e6a0 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -163,6 +163,23 @@ keys: # urls: # - https://your-host.com/handler +# Signal Relay +# since v1.4.0, a more reliable, psrpc based signal relay is available +# this gives us the ability to reliably proxy messages between a signal server and RTC node +# signal_relay: +# # disabled by default. will be enabled by default in future versions +# enabled: true +# # amount of time a message delivery is tried before giving up +# retry_timeout: 30s +# # minimum amount of time to wait for RTC node to ack, +# # retries use exponentially increasing wait on every subsequent try +# # with an upper bound of max_retry_interval +# min_retry_interval: 500ms +# # maximum amount of time to wait for RTC node to ack +# max_retry_interval: 5s +# # number of messages to buffer before dropping +# stream_buffer_size: 1000 + # customize audio level sensitivity # audio: # # minimum level to be considered active, 0-127, where 0 is loudest @@ -195,7 +212,7 @@ keys: # # set UDP port range for TURN relay to connect to LiveKit SFU, by default it uses a any available port # relay_range_start: 1024 # relay_range_end: 30000 -# # set external_tl to true if using a L4 load balancer to terminate TLS. when enabled, +# # set external_tls to true if using a L4 load balancer to terminate TLS. when enabled, # # LiveKit expects unencrypted traffic on tls_port, and still advertise tls_port as a TURN/TLS candidate. # external_tls: true # # needs to match tls cert domain @@ -207,15 +224,9 @@ keys: # ingress server # ingress: # # Prefix used to generate RTMP URLs for RTMP ingress. -# # The stream_key will be appended to this base and returned as part of the -# # ingress info # rtmp_base_url: "rtmp://my.domain.com/live" - -# egress server -# egress: -# # Whether to use the PSRPC enabled RPC implementation. This requires livekit egress version >=1.5.4 -# # The legacy, non PSRPC RPC implementation will be removed eventually -# use_psrpc: false +# # Prefix used to generate WHIP URLs for WHIP ingress. +# whip_base_url: "http://my.domain.com/whip" # Region of the current node. Required if using regionaware node selector # region: us-west-2 @@ -244,3 +255,9 @@ keys: # num_tracks: -1 # # defaults to 1 GB/s, or just under 10 Gbps # bytes_per_sec: 1_000_000_000 +# # how many tracks (audio / video) that a single participant can subscribe at same time. +# # if the limit is exceeded, subscriptions will be pending until any subscribed track has been unsubscribed. +# # value less or equal than 0 means no limit. +# subscription_limit_video: 0 +# subscription_limit_audio: 0 + diff --git a/go.mod b/go.mod index 2f3676438..1d7e21bf7 100644 --- a/go.mod +++ b/go.mod @@ -4,52 +4,50 @@ go 1.18 require ( github.com/bep/debounce v1.2.1 - github.com/d5/tengo/v2 v2.13.0 + github.com/d5/tengo/v2 v2.16.1 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 - github.com/frostbyte73/core v0.0.4 - github.com/gammazero/deque v0.1.0 - github.com/gammazero/workerpool v1.1.2 + github.com/frostbyte73/core v0.0.9 + github.com/gammazero/deque v0.2.1 + github.com/gammazero/workerpool v1.1.3 github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/hashicorp/golang-lru/v2 v2.0.4 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a - github.com/livekit/protocol v1.5.0 - github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d - github.com/mackerelio/go-osstat v0.2.3 - github.com/magefile/mage v1.14.0 - github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 + github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a + github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2 + github.com/livekit/psrpc v0.3.2 + github.com/mackerelio/go-osstat v0.2.4 + github.com/magefile/mage v1.15.0 + github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pion/ice/v2 v2.3.1 - github.com/pion/interceptor v0.1.12 - github.com/pion/logging v0.2.2 + github.com/pion/dtls/v2 v2.2.7 + github.com/pion/ice/v2 v2.3.9 + github.com/pion/interceptor v0.1.17 github.com/pion/rtcp v1.2.10 - github.com/pion/rtp v1.7.13 + github.com/pion/rtp v1.8.0 github.com/pion/sdp/v3 v3.0.6 - github.com/pion/stun v0.4.0 - github.com/pion/transport/v2 v2.0.2 - github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.56 + github.com/pion/transport/v2 v2.2.1 + github.com/pion/turn/v2 v2.1.2 + github.com/pion/webrtc/v3 v3.2.13 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.14.0 - github.com/redis/go-redis/v9 v9.0.2 - github.com/rs/cors v1.8.3 - github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a - github.com/stretchr/testify v1.8.2 + github.com/prometheus/client_golang v1.16.0 + github.com/redis/go-redis/v9 v9.0.5 + github.com/rs/cors v1.9.0 + github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.24.2 + github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 - go.uber.org/atomic v1.10.0 - go.uber.org/zap v1.24.0 - golang.org/x/sync v0.1.0 - google.golang.org/protobuf v1.29.0 + go.uber.org/atomic v1.11.0 + golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 + golang.org/x/sync v0.3.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -61,44 +59,48 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.24.0 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nats.go v1.26.0 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect - github.com/pion/dtls/v2 v2.2.6 // indirect + github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/sctp v1.8.6 // indirect - github.com/pion/srtp/v2 v2.0.12 // indirect - github.com/pion/udp/v2 v2.0.1 // indirect + github.com/pion/sctp v1.8.7 // indirect + github.com/pion/srtp/v2 v2.0.15 // indirect + github.com/pion/stun v0.6.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.7.0 // indirect - golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect - google.golang.org/grpc v1.53.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + golang.org/x/tools v0.9.3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.57.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index dbff2e09b..9a2e52f3c 100644 --- a/go.sum +++ b/go.sum @@ -1,71 +1,21 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= -github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d5/tengo/v2 v2.13.0 h1:4pZ5mR4vjOejpp+PMeIMpjZdObK7iwWoLTpVyhT+0Jk= -github.com/d5/tengo/v2 v2.13.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= +github.com/d5/tengo/v2 v2.16.1 h1:/N6dqiGu9toqANInZEOQMM8I06icdZnmb+81DG/lZdw= +github.com/d5/tengo/v2 v2.16.1/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -79,76 +29,40 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/florianl/go-tc v0.4.2 h1:jan5zcOWCLhA9SRBHZhQ0SSAq7cmDUagiRPngAi5AOQ= github.com/florianl/go-tc v0.4.2/go.mod h1:2W1jSMFryiYlpQigr4ZpSSpE9XNze+bW7cTsCXWbMwo= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frostbyte73/core v0.0.4 h1:CwwoYfKPdNSO/QbOOMWMRSYoNW14ov4XHnt094AuMX8= -github.com/frostbyte73/core v0.0.4/go.mod h1:mqHHSVFS5DE6kSdhU1/s9Mm0YCnLB8Ou2DD/eX1Zbr4= +github.com/frostbyte73/core v0.0.9 h1:AmE9GjgGpPsWk9ZkmY3HsYUs2hf2tZt+/W6r49URBQI= +github.com/frostbyte73/core v0.0.9/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= -github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M= -github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= -github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= +github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= +github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= +github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -156,42 +70,30 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= +github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= @@ -202,51 +104,40 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9 github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo= github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ= github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= -github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a h1:5UkGQpskXp7HcBmyrCwWtO7ygDWbqtjN09Yva4l/nyE= -github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a/go.mod h1:1Dlx20JPoIKGP45eo+yuj0HjeE25zmyeX/EWHiPCjFw= -github.com/livekit/protocol v1.5.0 h1:jFGSkSEv0PTjUlrW/WnmERejwxyHOSE9If4VU33PYgk= -github.com/livekit/protocol v1.5.0/go.mod h1:hkK/G0wwFiLUGp9F5kxeQxq2CQuIzkmfBwKhTsc71us= -github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d h1:3wfbd8zi7zGQCR+xfG3r2k9m2RwXUiIzR0SN4BHewwU= -github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= -github.com/mackerelio/go-osstat v0.2.3 h1:jAMXD5erlDE39kdX2CU7YwCGRcxIO33u/p8+Fhe5dJw= -github.com/mackerelio/go-osstat v0.2.3/go.mod h1:DQbPOnsss9JHIXgBStc/dnhhir3gbd3YH+Dbdi7ptMA= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a h1:JWpPHcMFuw0fP4swE89CfMgeUXiSN5IKvCJL/5HLI3A= +github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14= +github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2 h1:KxQIooCpXmn+qzxQxNbxBtRXstEFd2/7ihH4Pp1dOc4= +github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2/go.mod h1:3Dt53NrYnuA7pAJjAjXLJ2q5rU3JKoebvMttZPZWDH8= +github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= +github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= +github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 h1:CEy7VRV/Vbm7YLuZo3pGKa5GlPX4zzric6dEubIJTx0= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2/go.mod h1:otjOyjeqm3LALYcmX2AQIGH0VlojDoSd8aGOzsHAnBc= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= @@ -269,19 +160,12 @@ github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46N github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o= -github.com/nats-io/nats.go v1.24.0 h1:CRiD8L5GOQu/DcfkmgBcTTIQORMwizF+rPk6T0RaHVQ= -github.com/nats-io/nats.go v1.24.0/go.mod h1:dVQF+BK3SzUZpwyzHedXsvH3EO38aVKuOPkkHlv5hXA= -github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= -github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nats.go v1.26.0 h1:fWJTYPnZ8DzxIaqIHOAMfColuznchnd5Ab5dbJpgPIE= +github.com/nats-io/nats.go v1.26.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= +github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -295,15 +179,15 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= -github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= -github.com/pion/ice/v2 v2.3.1 h1:FQCmUfZe2Jpe7LYStVBOP6z1DiSzbIateih3TztgTjc= -github.com/pion/ice/v2 v2.3.1/go.mod h1:aq2kc6MtYNcn4XmMhobAv6hTNJiHzvD0yXRz80+bnP8= -github.com/pion/interceptor v0.1.12 h1:CslaNriCFUItiXS5o+hh5lpL0t0ytQkFnUcbbCs2Zq8= -github.com/pion/interceptor v0.1.12/go.mod h1:bDtgAD9dRkBZpWHGKaoKb42FhDHTG2rX8Ii9LRALLVA= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/ice/v2 v2.3.9 h1:7yZpHf3PhPxJGT4JkMj1Y8Rl5cQ6fB709iz99aeMd/U= +github.com/pion/ice/v2 v2.3.9/go.mod h1:lT3kv5uUIlHfXHU/ZRD7uKD/ufM202+eTa3C/umgGf4= +github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= @@ -312,198 +196,114 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtp v1.8.0 h1:SYD7040IR+NqrGBOc2GDU5iDjAR+0m5rnX/EWCUMNhw= +github.com/pion/rtp v1.8.0/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.6 h1:CUex11Vkt9YS++VhLf8b55O3VqKrWL6W3SDwX4jAqsI= -github.com/pion/sctp v1.8.6/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= +github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= +github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.12 h1:WrmiVCubGMOAObBU1vwWjG0H3VSyQHawKeer2PVA5rY= -github.com/pion/srtp/v2 v2.0.12/go.mod h1:C3Ep44hlOo2qEYaq4ddsmK5dL63eLehXFbHaZ9F5V9Y= -github.com/pion/stun v0.4.0 h1:vgRrbBE2htWHy7l3Zsxckk7rkjnjOsSM7PHZnBwo8rk= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= +github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= +github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.0.2 h1:St+8o+1PEzPT51O9bv+tH/KYYLMNR5Vwm5Z3Qkjsywg= -github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= -github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= -github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= -github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.56 h1:ScaiqKQN3liQwT+kJwOBaYP6TwSfixzdUnZmzHAo0a0= -github.com/pion/webrtc/v3 v3.1.56/go.mod h1:7VhbA6ihqJlz6R/INHjyh1b8HpiV9Ct4UQvE1OB/xoM= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/turn/v2 v2.1.2 h1:wj0cAoGKltaZ790XEGW9HwoUewqjliwmhtxCuB2ApyM= +github.com/pion/turn/v2 v2.1.2/go.mod h1:1kjnPkBcex3dhCU2Am+AAmxDcGhLX3WnMfmkNpvSTQU= +github.com/pion/webrtc/v3 v3.2.13 h1://ltbnahZewBWHvQYunlyLVWrHrsoyxYDfi3Ux6V4Gk= +github.com/pion/webrtc/v3 v3.2.13/go.mod h1:KS57v8u+fNMYAVM6gNsceIHtciyHlnfPNXU/7klJMFU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= -github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= +github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= -github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= -github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM= -github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk= -github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 h1:/yRP+0AN7mf5DkD3BAI6TOFnd51gEoDEb8o35jIFtgw= +golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -515,73 +315,36 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -595,12 +358,10 @@ golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -608,186 +369,71 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= -google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/install-livekit.sh b/install-livekit.sh index 3dd0f27ee..e0b243a81 100755 --- a/install-livekit.sh +++ b/install-livekit.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # LiveKit install script for Linux set -u diff --git a/magefile.go b/magefile.go index 1742de3df..1b3fdfa81 100644 --- a/magefile.go +++ b/magefile.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage // +build mage diff --git a/magefile_unix.go b/magefile_unix.go index a84892371..186f6f4d4 100644 --- a/magefile_unix.go +++ b/magefile_unix.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage && !windows // +build mage,!windows diff --git a/magefile_windows.go b/magefile_windows.go index 3276726bb..9e25fe722 100644 --- a/magefile_windows.go +++ b/magefile_windows.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage // +build mage diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index e7f84896e..8c37b7ba4 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -1,6 +1,24 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration -// configurations for livekit-client, add more configuration to StaticConfigurations as need +import ( + "github.com/livekit/protocol/livekit" +) + +// StaticConfigurations list specific device-side limitations that should be disabled at a global level var StaticConfigurations = []ConfigurationItem{ // { // Match: &ScriptMatch{Expr: `c.protocol <= 5 || c.browser == "firefox"`}, @@ -14,4 +32,13 @@ var StaticConfigurations = []ConfigurationItem{ // }}}, // Merge: false, // }, + { + Match: &ScriptMatch{Expr: `c.device_model == "Xiaomi 2201117TI" && c.os == "android"`}, + Configuration: &livekit.ClientConfiguration{ + DisabledCodecs: &livekit.DisabledCodecs{ + Publish: []*livekit.Codec{{Mime: "video/h264"}}, + }, + }, + Merge: false, + }, } diff --git a/pkg/clientconfiguration/conf_test.go b/pkg/clientconfiguration/conf_test.go index 46f271735..093a98f19 100644 --- a/pkg/clientconfiguration/conf_test.go +++ b/pkg/clientconfiguration/conf_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/match.go b/pkg/clientconfiguration/match.go index a060d83cc..3c3514220 100644 --- a/pkg/clientconfiguration/match.go +++ b/pkg/clientconfiguration/match.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/staticconfiguration.go b/pkg/clientconfiguration/staticconfiguration.go index 83f9c2dfb..2071d9112 100644 --- a/pkg/clientconfiguration/staticconfiguration.go +++ b/pkg/clientconfiguration/staticconfiguration.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/types.go b/pkg/clientconfiguration/types.go index 5e7a8ca2f..b014518cf 100644 --- a/pkg/clientconfiguration/types.go +++ b/pkg/clientconfiguration/types.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/config/config.go b/pkg/config/config.go index 8aad90afb..6b285da12 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( @@ -13,15 +27,12 @@ import ( "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" redisLiveKit "github.com/livekit/protocol/redis" ) -var DefaultStunServers = []string{ - "stun.l.google.com:19302", - "stun1.l.google.com:19302", -} - type CongestionControlProbeMode string type StreamTrackerType string @@ -39,13 +50,13 @@ const ( ) var ( - ErrKeyFileIncorrectPermission = errors.New("key file must have 0600 permission") + ErrKeyFileIncorrectPermission = errors.New("key file others permissions must be set to 0") ErrKeysNotSet = errors.New("one of key-file or keys must be provided") ) type Config struct { Port uint32 `yaml:"port"` - BindAddresses []string `yaml:"bind_addresses"` + BindAddresses []string `yaml:"bind_addresses,omitempty"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` @@ -54,7 +65,6 @@ type Config struct { Video VideoConfig `yaml:"video,omitempty"` Room RoomConfig `yaml:"room,omitempty"` TURN TURNConfig `yaml:"turn,omitempty"` - Egress EgressConfig `yaml:"egress,omitempty"` Ingress IngressConfig `yaml:"ingress,omitempty"` WebHook WebHookConfig `yaml:"webhook,omitempty"` NodeSelector NodeSelectorConfig `yaml:"node_selector,omitempty"` @@ -71,21 +81,11 @@ type Config struct { } type RTCConfig struct { - UDPPort uint32 `yaml:"udp_port,omitempty"` - TCPPort uint32 `yaml:"tcp_port,omitempty"` - ICEPortRangeStart uint32 `yaml:"port_range_start,omitempty"` - ICEPortRangeEnd uint32 `yaml:"port_range_end,omitempty"` - NodeIP string `yaml:"node_ip,omitempty"` - NodeIPAutoGenerated bool `yaml:"-"` - STUNServers []string `yaml:"stun_servers,omitempty"` - TURNServers []TURNServer `yaml:"turn_servers,omitempty"` - UseExternalIP bool `yaml:"use_external_ip"` - UseICELite bool `yaml:"use_ice_lite,omitempty"` - Interfaces InterfacesConfig `yaml:"interfaces"` - IPs IPsConfig `yaml:"ips"` - EnableLoopbackCandidate bool `yaml:"enable_loopback_candidate"` - UseMDNS bool `yaml:"use_mdns"` - StrictACKs bool `yaml:"strict_acks"` + rtcconfig.RTCConfig `yaml:",inline"` + + TURNServers []TURNServer `yaml:"turn_servers,omitempty"` + + StrictACKs bool `yaml:"strict_acks,omitempty"` // Number of packets to buffer for NACK PacketBufferSize int `yaml:"packet_buffer_size,omitempty"` @@ -98,9 +98,6 @@ type RTCConfig struct { // allow TCP and TURN/TLS fallback AllowTCPFallback *bool `yaml:"allow_tcp_fallback,omitempty"` - // for testing, disable UDP - ForceTCP bool `yaml:"force_tcp,omitempty"` - // force a reconnect on a publication error ReconnectOnPublicationError *bool `yaml:"reconnect_on_publication_error,omitempty"` @@ -122,43 +119,64 @@ type PLIThrottleConfig struct { HighQuality time.Duration `yaml:"high_quality,omitempty"` } +type CongestionControlProbeConfig struct { + BaseInterval time.Duration `yaml:"base_interval,omitempty"` + BackoffFactor float64 `yaml:"backoff_factor,omitempty"` + MaxInterval time.Duration `yaml:"max_interval,omitempty"` + + SettleWait time.Duration `yaml:"settle_wait,omitempty"` + SettleWaitMax time.Duration `yaml:"settle_wait_max,omitempty"` + + TrendWait time.Duration `yaml:"trend_wait,omitempty"` + + OveragePct int64 `yaml:"overage_pct,omitempty"` + MinBps int64 `yaml:"min_bps,omitempty"` + MinDuration time.Duration `yaml:"min_duration,omitempty"` + MaxDuration time.Duration `yaml:"max_duration,omitempty"` + DurationOverflowFactor float64 `yaml:"duration_overflow_factor,omitempty"` + DurationIncreaseFactor float64 `yaml:"duration_increase_factor,omitempty"` +} + +type CongestionControlChannelObserverConfig struct { + EstimateRequiredSamples int `yaml:"estimate_required_samples,omitempty"` + EstimateDownwardTrendThreshold float64 `yaml:"estimate_downward_trend_threshold,omitempty"` + EstimateCollapseThreshold time.Duration `yaml:"estimate_collapse_threshold,omitempty"` + EstimateValidityWindow time.Duration `yaml:"estimate_validity_window,omitempty"` + NackWindowMinDuration time.Duration `yaml:"nack_window_min_duration,omitempty"` + NackWindowMaxDuration time.Duration `yaml:"nack_window_max_duration,omitempty"` + NackRatioThreshold float64 `yaml:"nack_ratio_threshold,omitempty"` +} + type CongestionControlConfig struct { - Enabled bool `yaml:"enabled"` - AllowPause bool `yaml:"allow_pause"` - UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` - ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` - MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` -} - -type InterfacesConfig struct { - Includes []string `yaml:"includes"` - Excludes []string `yaml:"excludes"` -} - -type IPsConfig struct { - Includes []string `yaml:"includes"` - Excludes []string `yaml:"excludes"` + Enabled bool `yaml:"enabled"` + AllowPause bool `yaml:"allow_pause"` + UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` + ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` + MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` + ProbeConfig CongestionControlProbeConfig `yaml:"probe_config,omitempty"` + ChannelObserverProbeConfig CongestionControlChannelObserverConfig `yaml:"channel_observer_probe_config,omitempty"` + ChannelObserverNonProbeConfig CongestionControlChannelObserverConfig `yaml:"channel_observer_non_probe_config,omitempty"` } type AudioConfig struct { // minimum level to be considered active, 0-127, where 0 is loudest - ActiveLevel uint8 `yaml:"active_level"` + ActiveLevel uint8 `yaml:"active_level,omitempty"` // percentile to measure, a participant is considered active if it has exceeded the ActiveLevel more than // MinPercentile% of the time - MinPercentile uint8 `yaml:"min_percentile"` + MinPercentile uint8 `yaml:"min_percentile,omitempty"` // interval to update clients, in ms - UpdateInterval uint32 `yaml:"update_interval"` + UpdateInterval uint32 `yaml:"update_interval,omitempty"` // smoothing for audioLevel values sent to the client. // audioLevel will be an average of `smooth_intervals`, 0 to disable - SmoothIntervals uint32 `yaml:"smooth_intervals"` + SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"` // enable red encoding downtrack for opus only audio up track - ActiveREDEncoding bool `yaml:"active_red_encoding"` + ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` } type StreamTrackerPacketConfig struct { - SamplesRequired uint32 `yaml:"samples_required"` // number of samples needed per cycle - CyclesRequired uint32 `yaml:"cycles_required"` // number of cycles needed to be active - CycleDuration time.Duration `yaml:"cycle_duration"` + SamplesRequired uint32 `yaml:"samples_required,omitempty"` // number of samples needed per cycle + CyclesRequired uint32 `yaml:"cycles_required,omitempty"` // number of cycles needed to be active + CycleDuration time.Duration `yaml:"cycle_duration,omitempty"` } type StreamTrackerFrameConfig struct { @@ -173,8 +191,8 @@ type StreamTrackerConfig struct { } type StreamTrackersConfig struct { - Video StreamTrackerConfig `yaml:"video"` - Screenshare StreamTrackerConfig `yaml:"screenshare"` + Video StreamTrackerConfig `yaml:"video,omitempty"` + Screenshare StreamTrackerConfig `yaml:"screenshare,omitempty"` } type VideoConfig struct { @@ -184,12 +202,12 @@ type VideoConfig struct { type RoomConfig struct { // enable rooms to be automatically created - AutoCreate bool `yaml:"auto_create"` - EnabledCodecs []CodecSpec `yaml:"enabled_codecs"` - MaxParticipants uint32 `yaml:"max_participants"` - EmptyTimeout uint32 `yaml:"empty_timeout"` - EnableRemoteUnmute bool `yaml:"enable_remote_unmute"` - MaxMetadataSize uint32 `yaml:"max_metadata_size"` + AutoCreate bool `yaml:"auto_create,omitempty"` + EnabledCodecs []CodecSpec `yaml:"enabled_codecs,omitempty"` + MaxParticipants uint32 `yaml:"max_participants,omitempty"` + EmptyTimeout uint32 `yaml:"empty_timeout,omitempty"` + EnableRemoteUnmute bool `yaml:"enable_remote_unmute,omitempty"` + MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` } type CodecSpec struct { @@ -204,14 +222,14 @@ type LoggingConfig struct { type TURNConfig struct { Enabled bool `yaml:"enabled"` - Domain string `yaml:"domain"` - CertFile string `yaml:"cert_file"` - KeyFile string `yaml:"key_file"` - TLSPort int `yaml:"tls_port"` - UDPPort int `yaml:"udp_port"` + Domain string `yaml:"domain,omitempty"` + CertFile string `yaml:"cert_file,omitempty"` + KeyFile string `yaml:"key_file,omitempty"` + TLSPort int `yaml:"tls_port,omitempty"` + UDPPort int `yaml:"udp_port,omitempty"` RelayPortRangeStart uint16 `yaml:"relay_range_start,omitempty"` RelayPortRangeEnd uint16 `yaml:"relay_range_end,omitempty"` - ExternalTLS bool `yaml:"external_tls"` + ExternalTLS bool `yaml:"external_tls,omitempty"` } type WebHookConfig struct { @@ -222,18 +240,18 @@ type WebHookConfig struct { type NodeSelectorConfig struct { Kind string `yaml:"kind"` - SortBy string `yaml:"sort_by"` - CPULoadLimit float32 `yaml:"cpu_load_limit"` - SysloadLimit float32 `yaml:"sysload_limit"` - Regions []RegionConfig `yaml:"regions"` + SortBy string `yaml:"sort_by,omitempty"` + CPULoadLimit float32 `yaml:"cpu_load_limit,omitempty"` + SysloadLimit float32 `yaml:"sysload_limit,omitempty"` + Regions []RegionConfig `yaml:"regions,omitempty"` } type SignalRelayConfig struct { Enabled bool `yaml:"enabled"` - MaxAttempts int `yaml:"max_attempts"` - Timeout time.Duration `yaml:"timeout"` - Backoff time.Duration `yaml:"backoff"` - StreamBufferSize int `yaml:"stream_buffer_size"` + RetryTimeout time.Duration `yaml:"retry_timeout,omitempty"` + MinRetryInterval time.Duration `yaml:"min_retry_interval,omitempty"` + MaxRetryInterval time.Duration `yaml:"max_retry_interval,omitempty"` + StreamBufferSize int `yaml:"stream_buffer_size,omitempty"` } // RegionConfig lists available regions and their latitude/longitude, so the selector would prefer @@ -245,16 +263,15 @@ type RegionConfig struct { } type LimitConfig struct { - NumTracks int32 `yaml:"num_tracks"` - BytesPerSec float32 `yaml:"bytes_per_sec"` -} - -type EgressConfig struct { - UsePsRPC bool `yaml:"use_psrpc"` + NumTracks int32 `yaml:"num_tracks,omitempty"` + BytesPerSec float32 `yaml:"bytes_per_sec,omitempty"` + SubscriptionLimitVideo int32 `yaml:"subscription_limit_video,omitempty"` + SubscriptionLimitAudio int32 `yaml:"subscription_limit_audio,omitempty"` } type IngressConfig struct { RTMPBaseURL string `yaml:"rtmp_base_url"` + WHIPBaseURL string `yaml:"whip_base_url"` } // not exposed to YAML @@ -273,156 +290,198 @@ func DefaultAPIConfig() APIConfig { } } -func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []cli.Flag) (*Config, error) { - // start with defaults - conf := &Config{ - Port: 7880, - RTC: RTCConfig{ +var DefaultConfig = Config{ + Port: 7880, + RTC: RTCConfig{ + RTCConfig: rtcconfig.RTCConfig{ UseExternalIP: false, TCPPort: 7881, UDPPort: 0, ICEPortRangeStart: 0, ICEPortRangeEnd: 0, STUNServers: []string{}, - PacketBufferSize: 500, - StrictACKs: true, - PLIThrottle: PLIThrottleConfig{ - LowQuality: 500 * time.Millisecond, - MidQuality: time.Second, - HighQuality: time.Second, - }, - CongestionControl: CongestionControlConfig{ - Enabled: true, - AllowPause: false, - ProbeMode: CongestionControlProbeModePadding, - }, }, - Audio: AudioConfig{ - ActiveLevel: 35, // -35dBov - MinPercentile: 40, - UpdateInterval: 400, - SmoothIntervals: 2, + PacketBufferSize: 500, + StrictACKs: true, + PLIThrottle: PLIThrottleConfig{ + LowQuality: 500 * time.Millisecond, + MidQuality: time.Second, + HighQuality: time.Second, }, - Video: VideoConfig{ - DynacastPauseDelay: 5 * time.Second, - StreamTracker: StreamTrackersConfig{ - Video: StreamTrackerConfig{ - StreamTrackerType: StreamTrackerTypePacket, - BitrateReportInterval: map[int32]time.Duration{ - 0: 1 * time.Second, - 1: 1 * time.Second, - 2: 1 * time.Second, - }, - PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: StreamTrackerPacketConfig{ - SamplesRequired: 1, - CyclesRequired: 4, - CycleDuration: 500 * time.Millisecond, - }, - 1: StreamTrackerPacketConfig{ - SamplesRequired: 5, - CyclesRequired: 20, - CycleDuration: 500 * time.Millisecond, - }, - 2: StreamTrackerPacketConfig{ - SamplesRequired: 5, - CyclesRequired: 20, - CycleDuration: 500 * time.Millisecond, - }, - }, - FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: StreamTrackerFrameConfig{ - MinFPS: 5.0, - }, - 1: StreamTrackerFrameConfig{ - MinFPS: 5.0, - }, - 2: StreamTrackerFrameConfig{ - MinFPS: 5.0, - }, - }, - }, - Screenshare: StreamTrackerConfig{ - StreamTrackerType: StreamTrackerTypePacket, - BitrateReportInterval: map[int32]time.Duration{ - 0: 4 * time.Second, - 1: 4 * time.Second, - 2: 4 * time.Second, - }, - PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: StreamTrackerPacketConfig{ - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - 1: StreamTrackerPacketConfig{ - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - 2: StreamTrackerPacketConfig{ - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - }, - FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: StreamTrackerFrameConfig{ - MinFPS: 0.5, - }, - 1: StreamTrackerFrameConfig{ - MinFPS: 0.5, - }, - 2: StreamTrackerFrameConfig{ - MinFPS: 0.5, - }, - }, - }, - }, - }, - Redis: redisLiveKit.RedisConfig{}, - Room: RoomConfig{ - AutoCreate: true, - EnabledCodecs: []CodecSpec{ - {Mime: webrtc.MimeTypeOpus}, - {Mime: "audio/red"}, - {Mime: webrtc.MimeTypeVP8}, - {Mime: webrtc.MimeTypeH264}, - // {Mime: webrtc.MimeTypeAV1}, - // {Mime: webrtc.MimeTypeVP9}, - }, - EmptyTimeout: 5 * 60, - }, - Logging: LoggingConfig{ - PionLevel: "error", - }, - TURN: TURNConfig{ - Enabled: false, - }, - NodeSelector: NodeSelectorConfig{ - Kind: "any", - SortBy: "random", - SysloadLimit: 0.9, - CPULoadLimit: 0.9, - }, - SignalRelay: SignalRelayConfig{ - Enabled: false, - MaxAttempts: 3, - Timeout: 500 * time.Millisecond, - Backoff: 500 * time.Millisecond, - StreamBufferSize: 1000, - }, - Keys: map[string]string{}, - } + CongestionControl: CongestionControlConfig{ + Enabled: true, + AllowPause: false, + ProbeMode: CongestionControlProbeModePadding, + ProbeConfig: CongestionControlProbeConfig{ + BaseInterval: 3 * time.Second, + BackoffFactor: 1.5, + MaxInterval: 2 * time.Minute, + SettleWait: 250 * time.Millisecond, + SettleWaitMax: 10 * time.Second, + + TrendWait: 2 * time.Second, + + OveragePct: 120, + MinBps: 200_000, + MinDuration: 200 * time.Millisecond, + MaxDuration: 20 * time.Second, + DurationOverflowFactor: 1.25, + DurationIncreaseFactor: 1.5, + }, + ChannelObserverProbeConfig: CongestionControlChannelObserverConfig{ + EstimateRequiredSamples: 3, + EstimateDownwardTrendThreshold: 0.0, + EstimateCollapseThreshold: 0, + EstimateValidityWindow: 10 * time.Second, + NackWindowMinDuration: 500 * time.Millisecond, + NackWindowMaxDuration: 1 * time.Second, + NackRatioThreshold: 0.04, + }, + ChannelObserverNonProbeConfig: CongestionControlChannelObserverConfig{ + EstimateRequiredSamples: 8, + EstimateDownwardTrendThreshold: -0.5, + EstimateCollapseThreshold: 500 * time.Millisecond, + EstimateValidityWindow: 10 * time.Second, + NackWindowMinDuration: 1 * time.Second, + NackWindowMaxDuration: 2 * time.Second, + NackRatioThreshold: 0.08, + }, + }, + }, + Audio: AudioConfig{ + ActiveLevel: 35, // -35dBov + MinPercentile: 40, + UpdateInterval: 400, + SmoothIntervals: 2, + }, + Video: VideoConfig{ + DynacastPauseDelay: 5 * time.Second, + StreamTracker: StreamTrackersConfig{ + Video: StreamTrackerConfig{ + StreamTrackerType: StreamTrackerTypePacket, + BitrateReportInterval: map[int32]time.Duration{ + 0: 1 * time.Second, + 1: 1 * time.Second, + 2: 1 * time.Second, + }, + PacketTracker: map[int32]StreamTrackerPacketConfig{ + 0: { + SamplesRequired: 1, + CyclesRequired: 4, + CycleDuration: 500 * time.Millisecond, + }, + 1: { + SamplesRequired: 5, + CyclesRequired: 20, + CycleDuration: 500 * time.Millisecond, + }, + 2: { + SamplesRequired: 5, + CyclesRequired: 20, + CycleDuration: 500 * time.Millisecond, + }, + }, + FrameTracker: map[int32]StreamTrackerFrameConfig{ + 0: { + MinFPS: 5.0, + }, + 1: { + MinFPS: 5.0, + }, + 2: { + MinFPS: 5.0, + }, + }, + }, + Screenshare: StreamTrackerConfig{ + StreamTrackerType: StreamTrackerTypePacket, + BitrateReportInterval: map[int32]time.Duration{ + 0: 4 * time.Second, + 1: 4 * time.Second, + 2: 4 * time.Second, + }, + PacketTracker: map[int32]StreamTrackerPacketConfig{ + 0: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + 1: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + 2: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + }, + FrameTracker: map[int32]StreamTrackerFrameConfig{ + 0: { + MinFPS: 0.5, + }, + 1: { + MinFPS: 0.5, + }, + 2: { + MinFPS: 0.5, + }, + }, + }, + }, + }, + Redis: redisLiveKit.RedisConfig{}, + Room: RoomConfig{ + AutoCreate: true, + EnabledCodecs: []CodecSpec{ + {Mime: webrtc.MimeTypeOpus}, + {Mime: "audio/red"}, + {Mime: webrtc.MimeTypeVP8}, + {Mime: webrtc.MimeTypeH264}, + // {Mime: webrtc.MimeTypeAV1}, + // {Mime: webrtc.MimeTypeVP9}, + }, + EmptyTimeout: 5 * 60, + }, + Logging: LoggingConfig{ + PionLevel: "error", + }, + TURN: TURNConfig{ + Enabled: false, + }, + NodeSelector: NodeSelectorConfig{ + Kind: "any", + SortBy: "random", + SysloadLimit: 0.9, + CPULoadLimit: 0.9, + }, + SignalRelay: SignalRelayConfig{ + Enabled: false, + RetryTimeout: 7500 * time.Millisecond, + MinRetryInterval: 500 * time.Millisecond, + MaxRetryInterval: 4 * time.Second, + StreamBufferSize: 1000, + }, + Keys: map[string]string{}, +} + +func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []cli.Flag) (*Config, error) { + // start with defaults + conf := DefaultConfig if confString != "" { decoder := yaml.NewDecoder(strings.NewReader(confString)) decoder.KnownFields(strictMode) - if err := decoder.Decode(conf); err != nil { + if err := decoder.Decode(&conf); err != nil { return nil, fmt.Errorf("could not parse config: %v", err) } } + if err := conf.RTC.Validate(conf.Development); err != nil { + return nil, fmt.Errorf("could not validate RTC config: %v", err) + } + if c != nil { if err := conf.updateFromCLI(c, baseFlags); err != nil { return nil, err @@ -436,17 +495,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c } conf.KeyFile = file - // set defaults for ports if none are set - if conf.RTC.UDPPort == 0 && conf.RTC.ICEPortRangeStart == 0 { - // to make it easier to run in dev mode/docker, default to single port - if conf.Development { - conf.RTC.UDPPort = 7882 - } else { - conf.RTC.ICEPortRangeStart = 50000 - conf.RTC.ICEPortRangeEnd = 60000 - } - } - // set defaults for Turn relay if none are set if conf.TURN.RelayPortRangeStart == 0 || conf.TURN.RelayPortRangeEnd == 0 { // to make it easier to run in dev mode/docker, default to two ports @@ -459,14 +507,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c } } - if conf.RTC.NodeIP == "" { - conf.RTC.NodeIP, err = conf.determineIP() - if err != nil { - return nil, err - } - conf.RTC.NodeIPAutoGenerated = true - } - if conf.LogLevel != "" { conf.Logging.Level = conf.LogLevel } @@ -478,7 +518,7 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Environment = "dev" } - return conf, nil + return &conf, nil } func (conf *Config) IsTURNSEnabled() bool { @@ -514,13 +554,22 @@ func (conf *Config) ToCLIFlagNames(existingFlags []cli.Flag) map[string]reflect. for i := 0; i < currNode.TypeNode.NumField(); i++ { // inspect yaml tag from struct field to get path field := currNode.TypeNode.Type().Field(i) - yamlTag := strings.SplitN(field.Tag.Get("yaml"), ",", 2)[0] - if yamlTag == "" || yamlTag == "-" { + yamlTagArray := strings.SplitN(field.Tag.Get("yaml"), ",", 2) + yamlTag := yamlTagArray[0] + isInline := false + if len(yamlTagArray) > 1 && yamlTagArray[1] == "inline" { + isInline = true + } + if (yamlTag == "" && (!isInline || currNode.TagPrefix == "")) || yamlTag == "-" { continue } yamlPath := yamlTag if currNode.TagPrefix != "" { - yamlPath = fmt.Sprintf("%s.%s", currNode.TagPrefix, yamlTag) + if isInline { + yamlPath = currNode.TagPrefix + } else { + yamlPath = fmt.Sprintf("%s.%s", currNode.TagPrefix, yamlTag) + } } if existingFlagNames[yamlPath] { continue @@ -542,9 +591,10 @@ func (conf *Config) ToCLIFlagNames(existingFlags []cli.Flag) map[string]reflect. func (conf *Config) ValidateKeys() error { // prefer keyfile if set if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return err - } else if st.Mode().Perm() != 0600 { + } else if st.Mode().Perm()&otherFilter != 0000 { return ErrKeyFileIncorrectPermission } f, err := os.Open(conf.KeyFile) @@ -576,7 +626,7 @@ func (conf *Config) ValidateKeys() error { func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) { blankConfig := &Config{} - flags := []cli.Flag{} + flags := make([]cli.Flag, 0) for name, value := range blankConfig.ToCLIFlagNames(existingFlags) { kind := value.Kind() if kind == reflect.Ptr { @@ -628,6 +678,13 @@ func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) Usage: generatedCLIFlagUsage, Hidden: hidden, } + case reflect.Float64: + flag = &cli.Float64Flag{ + Name: name, + EnvVars: []string{envVar}, + Usage: generatedCLIFlagUsage, + Hidden: hidden, + } case reflect.Slice: // TODO continue @@ -679,6 +736,8 @@ func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { configValue.SetUint(c.Uint64(flagName)) case reflect.Float32: configValue.SetFloat(c.Float64(flagName)) + case reflect.Float64: + configValue.SetFloat(c.Float64(flagName)) // case reflect.Slice: // // TODO // case reflect.Map: @@ -741,3 +800,13 @@ func (conf *Config) unmarshalKeys(keys string) error { } return nil } + +// Note: only pass in logr.Logger with default depth +func SetLogger(l logger.Logger) { + logger.SetLogger(l, "livekit") +} + +func InitLoggerFromConfig(config LoggingConfig) { + pionlogger.SetLogLevel(config.PionLevel) + logger.InitFromConfig(config.Config, "livekit") +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 912aacc33..0e4719fff 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( diff --git a/pkg/config/ip.go b/pkg/config/ip.go deleted file mode 100644 index e854c8e2d..000000000 --- a/pkg/config/ip.go +++ /dev/null @@ -1,200 +0,0 @@ -package config - -import ( - "context" - "fmt" - "net" - "time" - - "github.com/pion/stun" - "github.com/pkg/errors" - - "github.com/livekit/protocol/logger" -) - -func (conf *Config) determineIP() (string, error) { - if conf.RTC.UseExternalIP { - stunServers := conf.RTC.STUNServers - if len(stunServers) == 0 { - stunServers = DefaultStunServers - } - var err error - for i := 0; i < 3; i++ { - var ip string - ip, err = GetExternalIP(context.Background(), stunServers, nil) - if err == nil { - return ip, nil - } else { - time.Sleep(500 * time.Millisecond) - } - } - return "", errors.Errorf("could not resolve external IP: %v", err) - } - - // use local ip instead - addresses, err := GetLocalIPAddresses(false) - if len(addresses) > 0 { - return addresses[0], err - } - return "", err -} - -func GetLocalIPAddresses(includeLoopback bool) ([]string, error) { - ifaces, err := net.Interfaces() - if err != nil { - return nil, err - } - loopBacks := make([]string, 0) - addresses := make([]string, 0) - for _, iface := range ifaces { - addrs, err := iface.Addrs() - if err != nil { - continue - } - for _, addr := range addrs { - var ip net.IP - switch typedAddr := addr.(type) { - case *net.IPNet: - ip = typedAddr.IP.To4() - case *net.IPAddr: - ip = typedAddr.IP.To4() - default: - continue - } - if ip == nil { - continue - } - if ip.IsLoopback() { - loopBacks = append(loopBacks, ip.String()) - } else { - addresses = append(addresses, ip.String()) - } - } - } - - if includeLoopback { - addresses = append(addresses, loopBacks...) - } - - if len(addresses) > 0 { - return addresses, nil - } - if len(loopBacks) > 0 { - return loopBacks, nil - } - return nil, fmt.Errorf("could not find local IP address") -} - -// GetExternalIP return external IP for localAddr from stun server. If localAddr is nil, a local address is chosen automatically, -// else the address will be used to validate the external IP is accessible from the outside. -func GetExternalIP(ctx context.Context, stunServers []string, localAddr net.Addr) (string, error) { - if len(stunServers) == 0 { - return "", errors.New("STUN servers are required but not defined") - } - dialer := &net.Dialer{ - LocalAddr: localAddr, - } - conn, err := dialer.Dial("udp4", stunServers[0]) - if err != nil { - return "", err - } - c, err := stun.NewClient(conn) - if err != nil { - return "", err - } - defer c.Close() - - message, err := stun.Build(stun.TransactionID, stun.BindingRequest) - if err != nil { - return "", err - } - - var stunErr error - // sufficiently large buffer to not block it - ipChan := make(chan string, 20) - err = c.Start(message, func(res stun.Event) { - if res.Error != nil { - stunErr = res.Error - return - } - - var xorAddr stun.XORMappedAddress - if err := xorAddr.GetFrom(res.Message); err != nil { - stunErr = err - return - } - ip := xorAddr.IP.To4() - if ip != nil { - ipChan <- ip.String() - } - }) - if err != nil { - return "", err - } - - ctx1, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - select { - case nodeIP := <-ipChan: - if localAddr == nil { - return nodeIP, nil - } - _ = c.Close() - return nodeIP, validateExternalIP(ctx1, nodeIP, localAddr.(*net.UDPAddr)) - case <-ctx1.Done(): - msg := "could not determine public IP" - if stunErr != nil { - return "", errors.Wrap(stunErr, msg) - } else { - return "", fmt.Errorf(msg) - } - } -} - -// validateExternalIP validates that the external IP is accessible from the outside by listen the local address, -// it will send a magic string to the external IP and check the string is received by the local address. -func validateExternalIP(ctx context.Context, nodeIP string, addr *net.UDPAddr) error { - srv, err := net.ListenUDP("udp", addr) - if err != nil { - return err - } - defer srv.Close() - - magicString := "9#B8D2Nvg2xg5P$ZRwJ+f)*^Nne6*W3WamGY" - - validCh := make(chan struct{}) - go func() { - buf := make([]byte, 1024) - for { - n, err := srv.Read(buf) - if err != nil { - logger.Debugw("error reading from UDP socket", "err", err) - return - } - if string(buf[:n]) == magicString { - close(validCh) - return - } - } - }() - - cli, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP(nodeIP), Port: srv.LocalAddr().(*net.UDPAddr).Port}) - if err != nil { - return err - } - defer cli.Close() - - if _, err = cli.Write([]byte(magicString)); err != nil { - return err - } - - ctx1, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - select { - case <-validCh: - return nil - case <-ctx1.Done(): - break - } - return fmt.Errorf("could not validate external IP") -} diff --git a/pkg/logger/logadapter.go b/pkg/logger/logadapter.go deleted file mode 100644 index a55a68ed0..000000000 --- a/pkg/logger/logadapter.go +++ /dev/null @@ -1,118 +0,0 @@ -package serverlogger - -import ( - "fmt" - "strings" - - "go.uber.org/zap/zapcore" - - "github.com/livekit/protocol/logger" -) - -// implements webrtc.LeveledLogger -type logAdapter struct { - logger logger.Logger - level zapcore.Level - ignoredPrefixes []string -} - -func (l *logAdapter) Trace(msg string) { - // ignore trace -} - -func (l *logAdapter) Tracef(format string, args ...interface{}) { - // ignore trace -} - -func (l *logAdapter) Debug(msg string) { - if l.level > zapcore.DebugLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Debugw(msg) -} - -func (l *logAdapter) Debugf(format string, args ...interface{}) { - if l.level > zapcore.DebugLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Debugw(msg) -} - -func (l *logAdapter) Info(msg string) { - if l.level > zapcore.InfoLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Infow(msg) -} - -func (l *logAdapter) Infof(format string, args ...interface{}) { - if l.level > zapcore.InfoLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Infow(msg) -} - -func (l *logAdapter) Warn(msg string) { - if l.level > zapcore.WarnLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Warnw(msg, nil) -} - -func (l *logAdapter) Warnf(format string, args ...interface{}) { - if l.level > zapcore.WarnLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Warnw(msg, nil) -} - -func (l *logAdapter) Error(msg string) { - if l.level > zapcore.ErrorLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Errorw(msg, nil) -} - -func (l *logAdapter) Errorf(format string, args ...interface{}) { - if l.level > zapcore.ErrorLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Errorw(msg, nil) -} - -func (l *logAdapter) shouldIgnore(msg string) bool { - for _, prefix := range l.ignoredPrefixes { - if strings.HasPrefix(msg, prefix) { - return true - } - } - return false -} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go deleted file mode 100644 index 7449ef555..000000000 --- a/pkg/logger/logger.go +++ /dev/null @@ -1,66 +0,0 @@ -package serverlogger - -import ( - "github.com/pion/logging" - "go.uber.org/zap/zapcore" - - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/config" -) - -var ( - pionLevel zapcore.Level - pionIgnoredPrefixes = map[string][]string{ - "ice": { - "pingAllCandidates called with no candidate pairs", - "failed to send packet: io: read/write on closed pipe", - "Ignoring remote candidate with tcpType active", - "discard message from", - "Failed to discover mDNS candidate", - "Failed to read from candidate tcp", - "remote mDNS candidate added, but mDNS is disabled", - }, - "pc": { - "Failed to accept RTCP stream is already closed", - "Failed to accept RTP stream is already closed", - "Incoming unhandled RTCP ssrc", - }, - "tcp_mux": { - "Error reading first packet from", - "error closing connection", - }, - "turn": { - "error when handling datagram", - }, - } -) - -// implements webrtc.LoggerFactory -type LoggerFactory struct { - logger logger.Logger -} - -func NewLoggerFactory(logger logger.Logger) *LoggerFactory { - return &LoggerFactory{ - logger: logger, - } -} - -func (f *LoggerFactory) NewLogger(scope string) logging.LeveledLogger { - return &logAdapter{ - logger: f.logger.WithName(scope), - level: pionLevel, - ignoredPrefixes: pionIgnoredPrefixes[scope], - } -} - -// Note: only pass in logr.Logger with default depth -func SetLogger(l logger.Logger) { - logger.SetLogger(l, "livekit") -} - -func InitFromConfig(config config.LoggingConfig) { - pionLevel = logger.ParseZapLevel(config.PionLevel) - logger.InitFromConfig(config.Config, "livekit") -} diff --git a/pkg/routing/errors.go b/pkg/routing/errors.go index 050b6b90f..4b6af0686 100644 --- a/pkg/routing/errors.go +++ b/pkg/routing/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import "errors" diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 20c090809..7a450bd16 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( @@ -23,6 +37,7 @@ type MessageSink interface { WriteMessage(msg proto.Message) error IsClosed() bool Close() + ConnectionID() livekit.ConnectionID } //counterfeiter:generate . MessageSource @@ -31,19 +46,21 @@ type MessageSource interface { ReadChan() <-chan proto.Message IsClosed() bool Close() + ConnectionID() livekit.ConnectionID } type ParticipantInit struct { - Identity livekit.ParticipantIdentity - Name livekit.ParticipantName - Reconnect bool - ReconnectReason livekit.ReconnectReason - AutoSubscribe bool - Client *livekit.ClientInfo - Grants *auth.ClaimGrants - Region string - AdaptiveStream bool - ID livekit.ParticipantID + Identity livekit.ParticipantIdentity + Name livekit.ParticipantName + Reconnect bool + ReconnectReason livekit.ReconnectReason + AutoSubscribe bool + Client *livekit.ClientInfo + Grants *auth.ClaimGrants + Region string + AdaptiveStream bool + ID livekit.ParticipantID + SubscriberAllowPause *bool } type NewParticipantCallback func( @@ -117,7 +134,7 @@ func (pi *ParticipantInit) ToStartSession(roomName livekit.RoomName, connectionI return nil, err } - return &livekit.StartSession{ + ss := &livekit.StartSession{ RoomName: string(roomName), Identity: string(pi.Identity), Name: string(pi.Name), @@ -130,7 +147,13 @@ func (pi *ParticipantInit) ToStartSession(roomName livekit.RoomName, connectionI GrantsJson: string(claims), AdaptiveStream: pi.AdaptiveStream, ParticipantId: string(pi.ID), - }, nil + } + if pi.SubscriberAllowPause != nil { + subscriberAllowPause := *pi.SubscriberAllowPause + ss.SubscriberAllowPause = &subscriberAllowPause + } + + return ss, nil } func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (*ParticipantInit, error) { @@ -139,7 +162,7 @@ func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (* return nil, err } - return &ParticipantInit{ + pi := &ParticipantInit{ Identity: livekit.ParticipantIdentity(ss.Identity), Name: livekit.ParticipantName(ss.Name), Reconnect: ss.Reconnect, @@ -150,5 +173,11 @@ func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (* Region: region, AdaptiveStream: ss.AdaptiveStream, ID: livekit.ParticipantID(ss.ParticipantId), - }, nil + } + if ss.SubscriberAllowPause != nil { + subscriberAllowPause := *ss.SubscriberAllowPause + pi.SubscriberAllowPause = &subscriberAllowPause + } + + return pi, nil } diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index a70cec005..b0fcbbccb 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( @@ -38,7 +52,7 @@ func NewLocalRouter(currentNode LocalNode, signalClient SignalClient) *LocalRout signalClient: signalClient, requestChannels: make(map[string]*MessageChannel), responseChannels: make(map[string]*MessageChannel), - rtcMessageChan: NewMessageChannel(localRTCChannelSize), + rtcMessageChan: NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize), } } @@ -88,12 +102,12 @@ func (r *LocalRouter) StartParticipantSignal(ctx context.Context, roomName livek } func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) { - connectionID, reqSink, resSource, err = r.signalClient.StartParticipantSignal(ctx, roomName, pi, livekit.NodeID(r.currentNode.Id)) + connectionID, reqSink, resSource, err = r.signalClient.StartParticipantSignal(ctx, roomName, pi, nodeID) if err != nil { logger.Errorw("could not handle new participant", err, "room", roomName, "participant", pi.Identity, - "connectionID", connectionID, + "connID", connectionID, ) } return @@ -103,17 +117,17 @@ func (r *LocalRouter) WriteParticipantRTC(_ context.Context, roomName livekit.Ro r.lock.Lock() if r.rtcMessageChan.IsClosed() { // create a new one - r.rtcMessageChan = NewMessageChannel(localRTCChannelSize) + r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) } r.lock.Unlock() - msg.ParticipantKey = string(participantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(participantKey(roomName, identity)) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) return r.writeRTCMessage(r.rtcMessageChan, msg) } func (r *LocalRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomName, msg *livekit.RTCNodeMessage) error { - msg.ParticipantKey = string(participantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(participantKey(roomName, "")) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) return r.WriteNodeRTC(ctx, r.currentNode.Id, msg) } @@ -121,7 +135,7 @@ func (r *LocalRouter) WriteNodeRTC(_ context.Context, _ string, msg *livekit.RTC r.lock.Lock() if r.rtcMessageChan.IsClosed() { // create a new one - r.rtcMessageChan = NewMessageChannel(localRTCChannelSize) + r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) } r.lock.Unlock() return r.writeRTCMessage(r.rtcMessageChan, msg) @@ -254,7 +268,7 @@ func (r *LocalRouter) getOrCreateMessageChannel(target map[string]*MessageChanne return mc } - mc = NewMessageChannel(DefaultMessageChannelSize) + mc = NewMessageChannel(livekit.ConnectionID(key), DefaultMessageChannelSize) mc.OnClose(func() { r.lock.Lock() delete(target, key) diff --git a/pkg/routing/messagechannel.go b/pkg/routing/messagechannel.go index de5bd9075..e761f3add 100644 --- a/pkg/routing/messagechannel.go +++ b/pkg/routing/messagechannel.go @@ -1,26 +1,43 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( "sync" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/proto" ) const DefaultMessageChannelSize = 200 type MessageChannel struct { - msgChan chan proto.Message - onClose func() - isClosed bool - lock sync.RWMutex + connectionID livekit.ConnectionID + msgChan chan proto.Message + onClose func() + isClosed bool + lock sync.RWMutex } -func NewDefaultMessageChannel() *MessageChannel { - return NewMessageChannel(DefaultMessageChannelSize) +func NewDefaultMessageChannel(connectionID livekit.ConnectionID) *MessageChannel { + return NewMessageChannel(connectionID, DefaultMessageChannelSize) } -func NewMessageChannel(size int) *MessageChannel { +func NewMessageChannel(connectionID livekit.ConnectionID, size int) *MessageChannel { return &MessageChannel{ + connectionID: connectionID, // allow some buffer to avoid blocked writes msgChan: make(chan proto.Message, size), } @@ -71,3 +88,7 @@ func (m *MessageChannel) Close() { m.onClose() } } + +func (m *MessageChannel) ConnectionID() livekit.ConnectionID { + return m.connectionID +} diff --git a/pkg/routing/messagechannel_test.go b/pkg/routing/messagechannel_test.go index b79a00795..5d78c2104 100644 --- a/pkg/routing/messagechannel_test.go +++ b/pkg/routing/messagechannel_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing_test import ( @@ -11,7 +25,7 @@ import ( func TestMessageChannel_WriteMessageClosed(t *testing.T) { // ensure it doesn't panic when written to after closing - m := routing.NewMessageChannel(routing.DefaultMessageChannelSize) + m := routing.NewMessageChannel(livekit.ConnectionID("test"), routing.DefaultMessageChannelSize) go func() { for msg := range m.ReadChan() { if msg == nil { diff --git a/pkg/routing/node.go b/pkg/routing/node.go index e39f5ac8c..16dc769a2 100644 --- a/pkg/routing/node.go +++ b/pkg/routing/node.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/redis.go b/pkg/routing/redis.go index db781cbe9..5d81ce088 100644 --- a/pkg/routing/redis.go +++ b/pkg/routing/redis.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( @@ -98,16 +112,24 @@ func publishSignalMessage(rc redis.UniversalClient, nodeID livekit.NodeID, conne type RTCNodeSink struct { rc redis.UniversalClient nodeID livekit.NodeID + connectionID livekit.ConnectionID participantKey livekit.ParticipantKey participantKeyB62 livekit.ParticipantKey isClosed atomic.Bool onClose func() } -func NewRTCNodeSink(rc redis.UniversalClient, nodeID livekit.NodeID, participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey) *RTCNodeSink { +func NewRTCNodeSink( + rc redis.UniversalClient, + nodeID livekit.NodeID, + connectionID livekit.ConnectionID, + participantKey livekit.ParticipantKey, + participantKeyB62 livekit.ParticipantKey, +) *RTCNodeSink { return &RTCNodeSink{ rc: rc, nodeID: nodeID, + connectionID: connectionID, participantKey: participantKey, participantKeyB62: participantKeyB62, } @@ -137,6 +159,12 @@ func (s *RTCNodeSink) OnClose(f func()) { s.onClose = f } +func (s *RTCNodeSink) ConnectionID() livekit.ConnectionID { + return s.connectionID +} + +// ---------------------------------------------------------------------- + type SignalNodeSink struct { rc redis.UniversalClient nodeID livekit.NodeID @@ -177,3 +205,7 @@ func (s *SignalNodeSink) IsClosed() bool { func (s *SignalNodeSink) OnClose(f func()) { s.onClose = f } + +func (s *SignalNodeSink) ConnectionID() livekit.ConnectionID { + return s.connectionID +} diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index be1a8e025..ef679261f 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( @@ -150,13 +164,19 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek } if r.usePSRPCSignal { - return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) + connectionID, reqSink, resSource, err = r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) + if err != nil { + return + } + + // map signal & rtc nodes + err = r.setParticipantSignalNode(connectionID, r.currentNode.Id) + return } - // create a new connection id connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) - pKey := participantKeyLegacy(roomName, pi.Identity) - pKeyB62 := participantKey(roomName, pi.Identity) + pKey := ParticipantKeyLegacy(roomName, pi.Identity) + pKeyB62 := ParticipantKey(roomName, pi.Identity) // map signal & rtc nodes if err = r.setParticipantSignalNode(connectionID, r.currentNode.Id); err != nil { @@ -167,7 +187,7 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek // set up response channel before sending StartSession and be ready to receive responses. resChan := r.getOrCreateMessageChannel(r.responseChannels, string(connectionID)) - sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), pKey, pKeyB62) + sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), connectionID, pKey, pKeyB62) // serialize claims ss, err := pi.ToStartSession(roomName, connectionID) @@ -185,16 +205,16 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek } func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { - pkey := participantKeyLegacy(roomName, identity) - pkeyB62 := participantKey(roomName, identity) + pkey := ParticipantKeyLegacy(roomName, identity) + pkeyB62 := ParticipantKey(roomName, identity) rtcNode, err := r.getParticipantRTCNode(pkey, pkeyB62) if err != nil { return err } - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), pkey, pkeyB62) - msg.ParticipantKey = string(participantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(participantKey(roomName, identity)) + rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), livekit.ConnectionID("ephemeral"), pkey, pkeyB62) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) return r.writeRTCMessage(rtcSink, msg) } @@ -203,13 +223,13 @@ func (r *RedisRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomNam if err != nil { return err } - msg.ParticipantKey = string(participantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(participantKey(roomName, "")) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) return r.WriteNodeRTC(ctx, node.Id, msg) } func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *livekit.RTCNodeMessage) error { - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) + rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), livekit.ConnectionID("ephemeral"), livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) return r.writeRTCMessage(rtcSink, msg) } @@ -229,7 +249,7 @@ func (r *RedisRouter) startParticipantRTC(ss *livekit.StartSession, participantK return err } - if err := r.setParticipantRTCNode(participantKey, participantKeyB62, rtcNode.Id); err != nil { + if err := r.SetParticipantRTCNode(participantKey, participantKeyB62, rtcNode.Id); err != nil { return err } @@ -328,7 +348,7 @@ func (r *RedisRouter) Stop() { r.cancel() } -func (r *RedisRouter) setParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, nodeID string) error { +func (r *RedisRouter) SetParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, nodeID string) error { var err error if participantKey != "" { err1 := r.rc.Set(r.ctx, participantRTCKey(participantKey), nodeID, participantMappingTTL).Err() diff --git a/pkg/routing/routingfakes/fake_message_sink.go b/pkg/routing/routingfakes/fake_message_sink.go index ec53c4309..9359b2ac2 100644 --- a/pkg/routing/routingfakes/fake_message_sink.go +++ b/pkg/routing/routingfakes/fake_message_sink.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -13,6 +14,16 @@ type FakeMessageSink struct { closeMutex sync.RWMutex closeArgsForCall []struct { } + ConnectionIDStub func() livekit.ConnectionID + connectionIDMutex sync.RWMutex + connectionIDArgsForCall []struct { + } + connectionIDReturns struct { + result1 livekit.ConnectionID + } + connectionIDReturnsOnCall map[int]struct { + result1 livekit.ConnectionID + } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -62,6 +73,59 @@ func (fake *FakeMessageSink) CloseCalls(stub func()) { fake.CloseStub = stub } +func (fake *FakeMessageSink) ConnectionID() livekit.ConnectionID { + fake.connectionIDMutex.Lock() + ret, specificReturn := fake.connectionIDReturnsOnCall[len(fake.connectionIDArgsForCall)] + fake.connectionIDArgsForCall = append(fake.connectionIDArgsForCall, struct { + }{}) + stub := fake.ConnectionIDStub + fakeReturns := fake.connectionIDReturns + fake.recordInvocation("ConnectionID", []interface{}{}) + fake.connectionIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessageSink) ConnectionIDCallCount() int { + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() + return len(fake.connectionIDArgsForCall) +} + +func (fake *FakeMessageSink) ConnectionIDCalls(stub func() livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = stub +} + +func (fake *FakeMessageSink) ConnectionIDReturns(result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + fake.connectionIDReturns = struct { + result1 livekit.ConnectionID + }{result1} +} + +func (fake *FakeMessageSink) ConnectionIDReturnsOnCall(i int, result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + if fake.connectionIDReturnsOnCall == nil { + fake.connectionIDReturnsOnCall = make(map[int]struct { + result1 livekit.ConnectionID + }) + } + fake.connectionIDReturnsOnCall[i] = struct { + result1 livekit.ConnectionID + }{result1} +} + func (fake *FakeMessageSink) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -181,6 +245,8 @@ func (fake *FakeMessageSink) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.writeMessageMutex.RLock() diff --git a/pkg/routing/routingfakes/fake_message_source.go b/pkg/routing/routingfakes/fake_message_source.go index acfe7606c..40c48eb56 100644 --- a/pkg/routing/routingfakes/fake_message_source.go +++ b/pkg/routing/routingfakes/fake_message_source.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -13,6 +14,16 @@ type FakeMessageSource struct { closeMutex sync.RWMutex closeArgsForCall []struct { } + ConnectionIDStub func() livekit.ConnectionID + connectionIDMutex sync.RWMutex + connectionIDArgsForCall []struct { + } + connectionIDReturns struct { + result1 livekit.ConnectionID + } + connectionIDReturnsOnCall map[int]struct { + result1 livekit.ConnectionID + } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -61,6 +72,59 @@ func (fake *FakeMessageSource) CloseCalls(stub func()) { fake.CloseStub = stub } +func (fake *FakeMessageSource) ConnectionID() livekit.ConnectionID { + fake.connectionIDMutex.Lock() + ret, specificReturn := fake.connectionIDReturnsOnCall[len(fake.connectionIDArgsForCall)] + fake.connectionIDArgsForCall = append(fake.connectionIDArgsForCall, struct { + }{}) + stub := fake.ConnectionIDStub + fakeReturns := fake.connectionIDReturns + fake.recordInvocation("ConnectionID", []interface{}{}) + fake.connectionIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessageSource) ConnectionIDCallCount() int { + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() + return len(fake.connectionIDArgsForCall) +} + +func (fake *FakeMessageSource) ConnectionIDCalls(stub func() livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = stub +} + +func (fake *FakeMessageSource) ConnectionIDReturns(result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + fake.connectionIDReturns = struct { + result1 livekit.ConnectionID + }{result1} +} + +func (fake *FakeMessageSource) ConnectionIDReturnsOnCall(i int, result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + if fake.connectionIDReturnsOnCall == nil { + fake.connectionIDReturnsOnCall = make(map[int]struct { + result1 livekit.ConnectionID + }) + } + fake.connectionIDReturnsOnCall[i] = struct { + result1 livekit.ConnectionID + }{result1} +} + func (fake *FakeMessageSource) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -172,6 +236,8 @@ func (fake *FakeMessageSource) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.readChanMutex.RLock() diff --git a/pkg/routing/routingfakes/fake_signal_client.go b/pkg/routing/routingfakes/fake_signal_client.go index 884dac4d5..0562b7c44 100644 --- a/pkg/routing/routingfakes/fake_signal_client.go +++ b/pkg/routing/routingfakes/fake_signal_client.go @@ -10,6 +10,16 @@ import ( ) type FakeSignalClient struct { + ActiveCountStub func() int + activeCountMutex sync.RWMutex + activeCountArgsForCall []struct { + } + activeCountReturns struct { + result1 int + } + activeCountReturnsOnCall map[int]struct { + result1 int + } StartParticipantSignalStub func(context.Context, livekit.RoomName, routing.ParticipantInit, livekit.NodeID) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) startParticipantSignalMutex sync.RWMutex startParticipantSignalArgsForCall []struct { @@ -34,6 +44,59 @@ type FakeSignalClient struct { invocationsMutex sync.RWMutex } +func (fake *FakeSignalClient) ActiveCount() int { + fake.activeCountMutex.Lock() + ret, specificReturn := fake.activeCountReturnsOnCall[len(fake.activeCountArgsForCall)] + fake.activeCountArgsForCall = append(fake.activeCountArgsForCall, struct { + }{}) + stub := fake.ActiveCountStub + fakeReturns := fake.activeCountReturns + fake.recordInvocation("ActiveCount", []interface{}{}) + fake.activeCountMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSignalClient) ActiveCountCallCount() int { + fake.activeCountMutex.RLock() + defer fake.activeCountMutex.RUnlock() + return len(fake.activeCountArgsForCall) +} + +func (fake *FakeSignalClient) ActiveCountCalls(stub func() int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = stub +} + +func (fake *FakeSignalClient) ActiveCountReturns(result1 int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = nil + fake.activeCountReturns = struct { + result1 int + }{result1} +} + +func (fake *FakeSignalClient) ActiveCountReturnsOnCall(i int, result1 int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = nil + if fake.activeCountReturnsOnCall == nil { + fake.activeCountReturnsOnCall = make(map[int]struct { + result1 int + }) + } + fake.activeCountReturnsOnCall[i] = struct { + result1 int + }{result1} +} + func (fake *FakeSignalClient) StartParticipantSignal(arg1 context.Context, arg2 livekit.RoomName, arg3 routing.ParticipantInit, arg4 livekit.NodeID) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) { fake.startParticipantSignalMutex.Lock() ret, specificReturn := fake.startParticipantSignalReturnsOnCall[len(fake.startParticipantSignalArgsForCall)] @@ -110,6 +173,8 @@ func (fake *FakeSignalClient) StartParticipantSignalReturnsOnCall(i int, result1 func (fake *FakeSignalClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.activeCountMutex.RLock() + defer fake.activeCountMutex.RUnlock() fake.startParticipantSignalMutex.RLock() defer fake.startParticipantSignalMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/routing/selector/any.go b/pkg/routing/selector/any.go index 399ad4947..71f09ba87 100644 --- a/pkg/routing/selector/any.go +++ b/pkg/routing/selector/any.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/cpuload.go b/pkg/routing/selector/cpuload.go index 61197907b..1cd04c4c0 100644 --- a/pkg/routing/selector/cpuload.go +++ b/pkg/routing/selector/cpuload.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/cpuload_test.go b/pkg/routing/selector/cpuload_test.go index c8afd5bcc..33bca4717 100644 --- a/pkg/routing/selector/cpuload_test.go +++ b/pkg/routing/selector/cpuload_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/errors.go b/pkg/routing/selector/errors.go index 9c05a269c..c011f67af 100644 --- a/pkg/routing/selector/errors.go +++ b/pkg/routing/selector/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import "errors" diff --git a/pkg/routing/selector/interfaces.go b/pkg/routing/selector/interfaces.go index aee027564..60d001e18 100644 --- a/pkg/routing/selector/interfaces.go +++ b/pkg/routing/selector/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/regionaware.go b/pkg/routing/selector/regionaware.go index 257862247..61f494f70 100644 --- a/pkg/routing/selector/regionaware.go +++ b/pkg/routing/selector/regionaware.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/regionaware_test.go b/pkg/routing/selector/regionaware_test.go index 1645c4526..75a5bef31 100644 --- a/pkg/routing/selector/regionaware_test.go +++ b/pkg/routing/selector/regionaware_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/sortby_test.go b/pkg/routing/selector/sortby_test.go index 1e391c650..31de0027b 100644 --- a/pkg/routing/selector/sortby_test.go +++ b/pkg/routing/selector/sortby_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/sysload.go b/pkg/routing/selector/sysload.go index 821f092ab..909311a0e 100644 --- a/pkg/routing/selector/sysload.go +++ b/pkg/routing/selector/sysload.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/sysload_test.go b/pkg/routing/selector/sysload_test.go index 1941d8e7c..ac7d59a25 100644 --- a/pkg/routing/selector/sysload_test.go +++ b/pkg/routing/selector/sysload_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/utils.go b/pkg/routing/selector/utils.go index 7ab021374..2ba0b3876 100644 --- a/pkg/routing/selector/utils.go +++ b/pkg/routing/selector/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/utils_test.go b/pkg/routing/selector/utils_test.go index 46038be7f..4f62f6db4 100644 --- a/pkg/routing/selector/utils_test.go +++ b/pkg/routing/selector/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 7bbb4df93..ecb1c6c7c 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -1,23 +1,46 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( "context" + "errors" + "sync" + "time" + "go.uber.org/atomic" "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" - "github.com/livekit/psrpc/middleware" + "github.com/livekit/psrpc/pkg/middleware" ) +var ErrSignalWriteFailed = errors.New("signal write failed") +var ErrSignalMessageDropped = errors.New("signal message dropped") + //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate //counterfeiter:generate . SignalClient type SignalClient interface { + ActiveCount() int StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) } @@ -25,15 +48,16 @@ type signalClient struct { nodeID livekit.NodeID config config.SignalRelayConfig client rpc.TypedSignalClient + active atomic.Int32 } func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config.SignalRelayConfig) (SignalClient, error) { - ri := middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - }) - c, err := rpc.NewTypedSignalClient(nodeID, bus, psrpc.WithClientStreamInterceptors(ri)) + c, err := rpc.NewTypedSignalClient( + nodeID, + bus, + middleware.WithClientMetrics(prometheus.PSRPCMetricsObserver{}), + psrpc.WithClientChannelSize(config.StreamBufferSize), + ) if err != nil { return nil, err } @@ -45,6 +69,10 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. }, nil } +func (r *signalClient) ActiveCount() int { + return int(r.active.Load()) +} + func (r *signalClient) StartParticipantSignal( ctx context.Context, roomName livekit.RoomName, @@ -62,60 +90,281 @@ func (r *signalClient) StartParticipantSignal( return } - logger.Debugw( - "starting signal connection", + l := logger.GetLogger().WithValues( "room", roomName, "reqNodeID", nodeID, "participant", pi.Identity, - "connectionID", connectionID, + "connID", connectionID, ) + l.Debugw("starting signal connection") + stream, err := r.client.RelaySignal(ctx, nodeID) if err != nil { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } err = stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) if err != nil { stream.Close(err) + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } - resChan := NewDefaultMessageChannel() + sink := NewSignalMessageSink(SignalSinkParams[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]{ + Logger: l, + Stream: stream, + Config: r.config, + Writer: signalRequestMessageWriter{}, + CloseOnFailure: true, + BlockOnClose: true, + ConnectionID: connectionID, + }) + resChan := NewDefaultMessageChannel(connectionID) go func() { - var err error - for msg := range stream.Channel() { - if err = resChan.WriteMessage(msg.Response); err != nil { - break - } - } + r.active.Inc() + defer r.active.Dec() - logger.Debugw("participant signal stream closed", - "error", err, - "room", ss.RoomName, - "participant", ss.Identity, - "connectionID", connectionID, + err := CopySignalStreamToMessageChannel[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]( + stream, + resChan, + signalResponseMessageReader{}, + r.config, ) + l.Infow("signal stream closed", "error", err) resChan.Close() }() - return connectionID, &relaySignalRequestSink{stream}, resChan, nil + return connectionID, sink, resChan, nil } -type relaySignalRequestSink struct { - psrpc.ClientStream[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse] +type signalRequestMessageWriter struct{} + +func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalRequest { + r := &rpc.RelaySignalRequest{ + Seq: seq, + Requests: make([]*livekit.SignalRequest, 0, len(msgs)), + Close: close, + } + for _, m := range msgs { + r.Requests = append(r.Requests, m.(*livekit.SignalRequest)) + } + return r } -func (s *relaySignalRequestSink) Close() { - s.ClientStream.Close(nil) +type signalResponseMessageReader struct{} + +func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto.Message, error) { + msgs := make([]proto.Message, 0, len(rm.Responses)) + for _, m := range rm.Responses { + msgs = append(msgs, m) + } + return msgs, nil } -func (s *relaySignalRequestSink) IsClosed() bool { - return s.Context().Err() != nil +type RelaySignalMessage interface { + proto.Message + GetSeq() uint64 + GetClose() bool } -func (s *relaySignalRequestSink) WriteMessage(msg proto.Message) error { - return s.Send(&rpc.RelaySignalRequest{Request: msg.(*livekit.SignalRequest)}) +type SignalMessageWriter[SendType RelaySignalMessage] interface { + Write(seq uint64, close bool, msgs []proto.Message) SendType +} + +type SignalMessageReader[RecvType RelaySignalMessage] interface { + Read(msg RecvType) ([]proto.Message, error) +} + +func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( + stream psrpc.Stream[SendType, RecvType], + ch *MessageChannel, + reader SignalMessageReader[RecvType], + config config.SignalRelayConfig, +) error { + r := &signalMessageReader[SendType, RecvType]{ + reader: reader, + config: config, + } + for msg := range stream.Channel() { + res, err := r.Read(msg) + if err != nil { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + return err + } + + for _, r := range res { + if err = ch.WriteMessage(r); err != nil { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + return err + } + prometheus.MessageCounter.WithLabelValues("signal", "success").Add(1) + } + + if msg.GetClose() { + return stream.Close(nil) + } + } + return stream.Err() +} + +type signalMessageReader[SendType, RecvType RelaySignalMessage] struct { + seq uint64 + reader SignalMessageReader[RecvType] + config config.SignalRelayConfig +} + +func (r *signalMessageReader[SendType, RecvType]) Read(msg RecvType) ([]proto.Message, error) { + res, err := r.reader.Read(msg) + if err != nil { + return nil, err + } + + if r.seq < msg.GetSeq() { + return nil, ErrSignalMessageDropped + } + if r.seq > msg.GetSeq() { + n := int(r.seq - msg.GetSeq()) + if n > len(res) { + n = len(res) + } + res = res[n:] + } + r.seq += uint64(len(res)) + + return res, nil +} + +type SignalSinkParams[SendType, RecvType RelaySignalMessage] struct { + Stream psrpc.Stream[SendType, RecvType] + Logger logger.Logger + Config config.SignalRelayConfig + Writer SignalMessageWriter[SendType] + CloseOnFailure bool + BlockOnClose bool + ConnectionID livekit.ConnectionID +} + +func NewSignalMessageSink[SendType, RecvType RelaySignalMessage](params SignalSinkParams[SendType, RecvType]) MessageSink { + return &signalMessageSink[SendType, RecvType]{ + SignalSinkParams: params, + } +} + +type signalMessageSink[SendType, RecvType RelaySignalMessage] struct { + SignalSinkParams[SendType, RecvType] + + mu sync.Mutex + seq uint64 + queue []proto.Message + writing bool + draining bool +} + +func (s *signalMessageSink[SendType, RecvType]) Close() { + s.mu.Lock() + s.draining = true + if !s.writing { + s.writing = true + go s.write() + } + s.mu.Unlock() + + // conditionally block while closing to wait for outgoing messages to drain + // + // on media the signal sink shares a goroutine with other signal connection + // attempts from the same participant so blocking delays establishing new + // sessions during reconnect. + // + // on controller closing without waiting for the outstanding messages to + // drain causes leave messages to be dropped from the write queue. when + // this happens other participants in the room aren't notified about the + // departure until the participant times out. + if s.BlockOnClose { + <-s.Stream.Context().Done() + } +} + +func (s *signalMessageSink[SendType, RecvType]) IsClosed() bool { + return s.Stream.Err() != nil +} + +func (s *signalMessageSink[SendType, RecvType]) write() { + interval := s.Config.MinRetryInterval + deadline := time.Now().Add(s.Config.RetryTimeout) + var err error + + s.mu.Lock() + for { + close := s.draining + if (!close && len(s.queue) == 0) || s.IsClosed() { + break + } + msg, n := s.Writer.Write(s.seq, close, s.queue), len(s.queue) + s.mu.Unlock() + + err = s.Stream.Send(msg, psrpc.WithTimeout(interval)) + if err != nil { + if time.Now().After(deadline) { + s.Logger.Warnw("could not send signal message", err) + + s.mu.Lock() + s.seq += uint64(len(s.queue)) + s.queue = nil + break + } + + interval *= 2 + if interval > s.Config.MaxRetryInterval { + interval = s.Config.MaxRetryInterval + } + } + + s.mu.Lock() + if err == nil { + interval = s.Config.MinRetryInterval + deadline = time.Now().Add(s.Config.RetryTimeout) + + s.seq += uint64(n) + s.queue = s.queue[n:] + + if close { + break + } + } + } + + s.writing = false + if s.draining { + s.Stream.Close(nil) + } + if err != nil && s.CloseOnFailure { + s.Stream.Close(ErrSignalWriteFailed) + } + s.mu.Unlock() +} + +func (s *signalMessageSink[SendType, RecvType]) WriteMessage(msg proto.Message) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.Stream.Err(); err != nil { + return err + } else if s.draining { + return psrpc.ErrStreamClosed + } + + s.queue = append(s.queue, msg) + if !s.writing { + s.writing = true + go s.write() + } + return nil +} + +func (s *signalMessageSink[SendType, RecvType]) ConnectionID() livekit.ConnectionID { + return s.SignalSinkParams.ConnectionID } diff --git a/pkg/routing/utils.go b/pkg/routing/utils.go index 8db4f649c..2e11fdbe2 100644 --- a/pkg/routing/utils.go +++ b/pkg/routing/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( @@ -9,7 +23,7 @@ import ( "github.com/livekit/protocol/livekit" ) -func participantKeyLegacy(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { +func ParticipantKeyLegacy(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { return livekit.ParticipantKey(string(roomName) + "|" + string(identity)) } @@ -25,7 +39,7 @@ func parseParticipantKeyLegacy(pkey livekit.ParticipantKey) (roomName livekit.Ro return } -func participantKey(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { +func ParticipantKey(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { return livekit.ParticipantKey(encode(string(roomName), string(identity))) } diff --git a/pkg/routing/utils_test.go b/pkg/routing/utils_test.go index 127d3fd13..8ae1e9b4c 100644 --- a/pkg/routing/utils_test.go +++ b/pkg/routing/utils_test.go @@ -1,15 +1,30 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( "testing" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" ) func TestUtils_ParticipantKey(t *testing.T) { // encode/decode empty - encoded := participantKey("", "") + encoded := ParticipantKey("", "") roomName, identity, err := parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName(""), roomName) @@ -20,28 +35,28 @@ func TestUtils_ParticipantKey(t *testing.T) { require.Error(t, err) // encode/decode without delimiter - encoded = participantKey("room1", "identity1") + encoded = ParticipantKey("room1", "identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) // encode/decode with delimiter in roomName - encoded = participantKey("room1|alter_room1", "identity1") + encoded = ParticipantKey("room1|alter_room1", "identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1|alter_room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) // encode/decode with delimiter in identity - encoded = participantKey("room1", "identity1|alter-identity1") + encoded = ParticipantKey("room1", "identity1|alter-identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1|alter-identity1"), identity) // encode/decode with delimiter in both and multiple delimiters in both - encoded = participantKey("room1|alter_room1|again_room1", "identity1|alter-identity1|again-identity1") + encoded = ParticipantKey("room1|alter_room1|again_room1", "identity1|alter-identity1|again-identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1|alter_room1|again_room1"), roomName) diff --git a/pkg/rtc/clientinfo.go b/pkg/rtc/clientinfo.go index efef60bef..7912968b0 100644 --- a/pkg/rtc/clientinfo.go +++ b/pkg/rtc/clientinfo.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index f5c885603..efe2a6a1f 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -1,43 +1,40 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( - "context" - "errors" - "fmt" - "math/rand" - "net" - "strings" - "sync" - "time" - - "github.com/pion/ice/v2" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "github.com/livekit/livekit-server/pkg/config" - logging "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" - "github.com/livekit/protocol/logger" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" ) const ( - minUDPBufferSize = 5_000_000 - defaultUDPBufferSize = 16_777_216 - frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" + frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" ) type WebRTCConfig struct { - Configuration webrtc.Configuration - SettingEngine webrtc.SettingEngine - Receiver ReceiverConfig - BufferFactory *buffer.Factory - UDPMux ice.UDPMux - TCPMuxListener *net.TCPListener - Publisher DirectionConfig - Subscriber DirectionConfig - NAT1To1IPs []string - UseMDNS bool + rtcconfig.WebRTCConfig + + BufferFactory *buffer.Factory + Receiver ReceiverConfig + Publisher DirectionConfig + Subscriber DirectionConfig } type ReceiverConfig struct { @@ -60,138 +57,21 @@ type DirectionConfig struct { StrictACKs bool } -const ( - // number of packets to buffer up - readBufferSize = 50 - - writeBufferSizeInBytes = 4 * 1024 * 1024 -) - -func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, error) { +func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { rtcConf := conf.RTC - c := webrtc.Configuration{ - SDPSemantics: webrtc.SDPSemanticsUnifiedPlan, - } - s := webrtc.SettingEngine{ - LoggerFactory: logging.NewLoggerFactory(logger.GetLogger()), + + webRTCConfig, err := rtcconfig.NewWebRTCConfig(&rtcConf.RTCConfig, conf.Development) + if err != nil { + return nil, err } - var ifFilter func(string) bool - if len(rtcConf.Interfaces.Includes) != 0 || len(rtcConf.Interfaces.Excludes) != 0 { - ifFilter = InterfaceFilterFromConf(rtcConf.Interfaces) - s.SetInterfaceFilter(ifFilter) - } - - var ipFilter func(net.IP) bool - if len(rtcConf.IPs.Includes) != 0 || len(rtcConf.IPs.Excludes) != 0 { - filter, err := IPFilterFromConf(rtcConf.IPs) - if err != nil { - return nil, err - } - ipFilter = filter - s.SetIPFilter(filter) - } - - if !rtcConf.UseMDNS { - s.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) - } - - var nat1to1IPs []string - // force it to the node IPs that the user has set - if externalIP != "" && (conf.RTC.UseExternalIP || (conf.RTC.NodeIP != "" && !conf.RTC.NodeIPAutoGenerated)) { - if conf.RTC.UseExternalIP { - ips, err := getNAT1to1IPsForConf(conf, ipFilter) - if err != nil { - return nil, err - } - if len(ips) == 0 { - logger.Infow("no external IPs found, using node IP for NAT1To1Ips", "ip", externalIP) - s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost) - } else { - logger.Infow("using external IPs", "ips", ips) - s.SetNAT1To1IPs(ips, webrtc.ICECandidateTypeHost) - } - nat1to1IPs = ips - } else { - s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost) - } - } + // we don't want to use active TCP on a server, clients should be dialing + webRTCConfig.SettingEngine.DisableActiveTCP(true) if rtcConf.PacketBufferSize == 0 { rtcConf.PacketBufferSize = 500 } - var udpMux ice.UDPMux - var err error - networkTypes := make([]webrtc.NetworkType, 0, 4) - - if !rtcConf.ForceTCP { - networkTypes = append(networkTypes, - webrtc.NetworkTypeUDP4, webrtc.NetworkTypeUDP6, - ) - if rtcConf.ICEPortRangeStart != 0 && rtcConf.ICEPortRangeEnd != 0 { - if err := s.SetEphemeralUDPPortRange(uint16(rtcConf.ICEPortRangeStart), uint16(rtcConf.ICEPortRangeEnd)); err != nil { - return nil, err - } - } else if rtcConf.UDPPort != 0 { - opts := []ice.UDPMuxFromPortOption{ - ice.UDPMuxFromPortWithReadBufferSize(defaultUDPBufferSize), - ice.UDPMuxFromPortWithWriteBufferSize(defaultUDPBufferSize), - ice.UDPMuxFromPortWithLogger(s.LoggerFactory.NewLogger("udp_mux")), - } - if rtcConf.EnableLoopbackCandidate { - opts = append(opts, ice.UDPMuxFromPortWithLoopback()) - } - if ipFilter != nil { - opts = append(opts, ice.UDPMuxFromPortWithIPFilter(ipFilter)) - } - if ifFilter != nil { - opts = append(opts, ice.UDPMuxFromPortWithInterfaceFilter(ifFilter)) - } - udpMux, err := ice.NewMultiUDPMuxFromPort(int(rtcConf.UDPPort), opts...) - if err != nil { - return nil, err - } - - s.SetICEUDPMux(udpMux) - if !conf.Development { - checkUDPReadBuffer() - } - } - } - - // use TCP mux when it's set - var tcpListener *net.TCPListener - if rtcConf.TCPPort != 0 { - networkTypes = append(networkTypes, - webrtc.NetworkTypeTCP4, webrtc.NetworkTypeTCP6, - ) - tcpListener, err = net.ListenTCP("tcp", &net.TCPAddr{ - Port: int(rtcConf.TCPPort), - }) - if err != nil { - return nil, err - } - - tcpMux := ice.NewTCPMuxDefault(ice.TCPMuxParams{ - Logger: s.LoggerFactory.NewLogger("tcp_mux"), - Listener: tcpListener, - ReadBufferSize: readBufferSize, - WriteBufferSize: writeBufferSizeInBytes, - }) - - s.SetICETCPMux(tcpMux) - } - - if len(networkTypes) == 0 { - return nil, errors.New("TCP is forced but not configured") - } - s.SetNetworkTypes(networkTypes) - - if rtcConf.EnableLoopbackCandidate { - s.SetIncludeLoopbackCandidate(true) - } - // publisher configuration publisherConfig := DirectionConfig{ StrictACKs: true, // publisher is dialed, and will always reply with ACK @@ -244,32 +124,13 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err subscriberConfig.RTCPFeedback.Video = append(subscriberConfig.RTCPFeedback.Video, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBGoogREMB}) } - if rtcConf.UseICELite { - s.SetLite(true) - } else if rtcConf.NodeIP == "" && !rtcConf.UseExternalIP { - // use STUN servers for server to support NAT - // when deployed in production, we expect UseExternalIP to be used, and ports accessible - // this is not compatible with ICE Lite - // Do not automatically add STUN servers if nodeIP is set - if len(rtcConf.STUNServers) > 0 { - c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(rtcConf.STUNServers)} - } else { - c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(config.DefaultStunServers)} - } - } - return &WebRTCConfig{ - Configuration: c, - SettingEngine: s, + WebRTCConfig: *webRTCConfig, Receiver: ReceiverConfig{ PacketBufferSize: rtcConf.PacketBufferSize, }, - UDPMux: udpMux, - TCPMuxListener: tcpListener, - Publisher: publisherConfig, - Subscriber: subscriberConfig, - NAT1To1IPs: nat1to1IPs, - UseMDNS: rtcConf.UseMDNS, + Publisher: publisherConfig, + Subscriber: subscriberConfig, }, nil } @@ -277,190 +138,3 @@ func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) { c.BufferFactory = factory c.SettingEngine.BufferFactory = factory.GetOrNew } - -func iceServerForStunServers(servers []string) webrtc.ICEServer { - iceServer := webrtc.ICEServer{} - for _, stunServer := range servers { - iceServer.URLs = append(iceServer.URLs, fmt.Sprintf("stun:%s", stunServer)) - } - return iceServer -} - -func getNAT1to1IPsForConf(conf *config.Config, ipFilter func(net.IP) bool) ([]string, error) { - stunServers := conf.RTC.STUNServers - if len(stunServers) == 0 { - stunServers = config.DefaultStunServers - } - localIPs, err := config.GetLocalIPAddresses(conf.RTC.EnableLoopbackCandidate) - if err != nil { - return nil, err - } - type ipmapping struct { - externalIP string - localIP string - } - addrCh := make(chan ipmapping, len(localIPs)) - - var udpPorts []int - if conf.RTC.ICEPortRangeStart != 0 && conf.RTC.ICEPortRangeEnd != 0 { - portRangeStart, portRangeEnd := uint16(conf.RTC.ICEPortRangeStart), uint16(conf.RTC.ICEPortRangeEnd) - for i := 0; i < 5; i++ { - udpPorts = append(udpPorts, rand.Intn(int(portRangeEnd-portRangeStart))+int(portRangeStart)) - } - } else if conf.RTC.UDPPort != 0 { - udpPorts = append(udpPorts, int(conf.RTC.UDPPort)) - } else { - udpPorts = append(udpPorts, 0) - } - - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(context.Background()) - for _, ip := range localIPs { - if ipFilter != nil && !ipFilter(net.ParseIP(ip)) { - continue - } - - wg.Add(1) - go func(localIP string) { - defer wg.Done() - for _, port := range udpPorts { - addr, err := config.GetExternalIP(ctx, stunServers, &net.UDPAddr{IP: net.ParseIP(localIP), Port: port}) - if err != nil { - if strings.Contains(err.Error(), "address already in use") { - logger.Debugw("failed to get external ip, address already in use", "local", localIP, "port", port) - continue - } - logger.Infow("failed to get external ip", "local", localIP, "err", err) - return - } - addrCh <- ipmapping{externalIP: addr, localIP: localIP} - return - } - logger.Infow("failed to get external ip after all ports tried", "local", localIP, "ports", udpPorts) - }(ip) - } - - var firstResloved bool - natMapping := make(map[string]string) - timeout := time.NewTimer(5 * time.Second) - defer timeout.Stop() - -done: - for { - select { - case mapping := <-addrCh: - if !firstResloved { - firstResloved = true - timeout.Reset(1 * time.Second) - } - if local, ok := natMapping[mapping.externalIP]; ok { - logger.Infow("external ip already solved, ignore duplicate", - "external", mapping.externalIP, - "local", local, - "ignore", mapping.localIP) - } else { - natMapping[mapping.externalIP] = mapping.localIP - } - - case <-timeout.C: - break done - } - } - cancel() - wg.Wait() - - if len(natMapping) == 0 { - // no external ip resolved - return nil, nil - } - - // mapping unresolved local ip to itself - for _, local := range localIPs { - var found bool - for _, localIPMapping := range natMapping { - if local == localIPMapping { - found = true - break - } - } - if !found { - natMapping[local] = local - } - } - - nat1to1IPs := make([]string, 0, len(natMapping)) - for external, local := range natMapping { - nat1to1IPs = append(nat1to1IPs, fmt.Sprintf("%s/%s", external, local)) - } - return nat1to1IPs, nil -} - -func InterfaceFilterFromConf(ifs config.InterfacesConfig) func(string) bool { - includes := ifs.Includes - excludes := ifs.Excludes - return func(s string) bool { - // filter by include interfaces - if len(includes) > 0 { - for _, iface := range includes { - if iface == s { - return true - } - } - return false - } - - // filter by exclude interfaces - if len(excludes) > 0 { - for _, iface := range excludes { - if iface == s { - return false - } - } - } - return true - } -} - -func IPFilterFromConf(ips config.IPsConfig) (func(ip net.IP) bool, error) { - var ipnets [2][]*net.IPNet - var err error - for i, ips := range [][]string{ips.Includes, ips.Excludes} { - ipnets[i], err = func(fromIPs []string) ([]*net.IPNet, error) { - var toNets []*net.IPNet - for _, ip := range fromIPs { - _, ipnet, err := net.ParseCIDR(ip) - if err != nil { - return nil, err - } - toNets = append(toNets, ipnet) - } - return toNets, nil - }(ips) - - if err != nil { - return nil, err - } - } - - includes, excludes := ipnets[0], ipnets[1] - - return func(ip net.IP) bool { - if len(includes) > 0 { - for _, ipn := range includes { - if ipn.Contains(ip) { - return true - } - } - return false - } - - if len(excludes) > 0 { - for _, ipn := range excludes { - if ipn.Contains(ip) { - return false - } - } - } - return true - }, nil -} diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index aed49e63f..edaadb4f2 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -88,7 +102,7 @@ func (d *DynacastManager) Close() { } } -// THere are situations like track unmute or streaming from a sifferent node +// THere are situations like track unmute or streaming from a different node // where subscribed quality needs to sent to the provider immediately. // This bypasses any debouncing and forces a subscribed quality update // with immediate effect. diff --git a/pkg/rtc/dynacastmanager_test.go b/pkg/rtc/dynacastmanager_test.go index ae600575a..ee1c97c70 100644 --- a/pkg/rtc/dynacastmanager_test.go +++ b/pkg/rtc/dynacastmanager_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -6,10 +20,11 @@ import ( "testing" "time" - "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/protocol/livekit" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" ) func TestSubscribedMaxQuality(t *testing.T) { diff --git a/pkg/rtc/dynacastquality.go b/pkg/rtc/dynacastquality.go index 5be1e9976..7f0f90495 100644 --- a/pkg/rtc/dynacastquality.go +++ b/pkg/rtc/dynacastquality.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/errors.go b/pkg/rtc/errors.go index dcd31c979..383afde0d 100644 --- a/pkg/rtc/errors.go +++ b/pkg/rtc/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import "errors" @@ -14,9 +28,10 @@ var ( ErrMissingGrants = errors.New("VideoGrant is missing") // Track subscription related - ErrNoTrackPermission = errors.New("participant is not allowed to subscribe to this track") - ErrNoSubscribePermission = errors.New("participant is not given permission to subscribe to tracks") - ErrTrackNotFound = errors.New("track cannot be found") - ErrTrackNotAttached = errors.New("track is not yet attached") - ErrTrackNotBound = errors.New("track not bound") + ErrNoTrackPermission = errors.New("participant is not allowed to subscribe to this track") + ErrNoSubscribePermission = errors.New("participant is not given permission to subscribe to tracks") + ErrTrackNotFound = errors.New("track cannot be found") + ErrTrackNotAttached = errors.New("track is not yet attached") + ErrTrackNotBound = errors.New("track not bound") + ErrSubscriptionLimitExceeded = errors.New("participant has exceeded its subscription limit") ) diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index dff552233..c47b5a658 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -21,7 +35,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro p.StateReturns(livekit.ParticipantInfo_JOINED) p.ProtocolVersionReturns(protocol) p.CanSubscribeReturns(true) - p.CanPublishReturns(!hidden) + p.CanPublishSourceReturns(!hidden) p.CanPublishDataReturns(!hidden) p.HiddenReturns(hidden) p.ToProtoReturns(&livekit.ParticipantInfo{ @@ -31,7 +45,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro IsPublisher: publisher, }) - p.SetMetadataStub = func(m string) { + p.SetMetadataCalls(func(m string) { var f func(participant types.LocalParticipant) if p.OnParticipantUpdateCallCount() > 0 { f = p.OnParticipantUpdateArgsForCall(p.OnParticipantUpdateCallCount() - 1) @@ -39,7 +53,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro if f != nil { f(p) } - } + }) updateTrack := func() { var f func(participant types.LocalParticipant, track types.MediaTrack) if p.OnTrackUpdatedCallCount() > 0 { @@ -50,12 +64,12 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro } } - p.SetTrackMutedStub = func(sid livekit.TrackID, muted bool, fromServer bool) { + p.SetTrackMutedCalls(func(sid livekit.TrackID, muted bool, fromServer bool) { updateTrack() - } - p.AddTrackStub = func(req *livekit.AddTrackRequest) { + }) + p.AddTrackCalls(func(req *livekit.AddTrackRequest) { updateTrack() - } + }) return p } diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index de637e9df..c2866697b 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -16,7 +30,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac opusCodec := opusCodecCapability opusCodec.RTCPFeedback = rtcpFeedback.Audio var opusPayload webrtc.PayloadType - if isCodecEnabled(codecs, opusCodec) { + if IsCodecEnabled(codecs, opusCodec) { opusPayload = 111 if err := me.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: opusCodec, @@ -25,7 +39,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac return err } - if isCodecEnabled(codecs, redCodecCapability) { + if IsCodecEnabled(codecs, redCodecCapability) { if err := me.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: redCodecCapability, PayloadType: 63, @@ -40,16 +54,14 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, PayloadType: 96, }, - /* - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 98, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 100, - }, - */ + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 98, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 100, + }, { RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, PayloadType: 125, @@ -67,7 +79,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac PayloadType: 35, }, } { - if isCodecEnabled(codecs, codec.RTPCodecCapability) { + if IsCodecEnabled(codecs, codec.RTPCodecCapability) { if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { return err } @@ -105,7 +117,7 @@ func createMediaEngine(codecs []*livekit.Codec, config DirectionConfig) (*webrtc return me, nil } -func isCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool { +func IsCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool { for _, codec := range codecs { if !strings.EqualFold(codec.Mime, cap.MimeType) { continue diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index b774f49fa..353f6f966 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -12,15 +26,15 @@ import ( func TestIsCodecEnabled(t *testing.T) { t.Run("empty fmtp requirement should match all", func(t *testing.T) { enabledCodecs := []*livekit.Codec{{Mime: "video/h264"}} - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) }) t.Run("when fmtp is provided, require match", func(t *testing.T) { enabledCodecs := []*livekit.Codec{{Mime: "video/h264", FmtpLine: "special"}} - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) }) } diff --git a/pkg/rtc/medialossproxy.go b/pkg/rtc/medialossproxy.go index 4b25d479a..0ac54b7e6 100644 --- a/pkg/rtc/medialossproxy.go +++ b/pkg/rtc/medialossproxy.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index f492930e3..72f0a6f3a 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -190,6 +204,7 @@ func (t *MediaTrack) SetPendingCodecSid(codecs []*livekit.SimulcastCodec) { } } t.params.TrackInfo = ti + t.MediaTrackReceiver.UpdateTrackInfo(ti) } // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 5def4e0b0..9a0587447 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -98,4 +112,65 @@ func TestGetQualityForDimension(t *testing.T) { require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(800, 500)) require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) }) + + t.Run("highest layer with smallest dimensions", func(t *testing.T) { + mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + Type: livekit.TrackType_VIDEO, + Width: 1080, + Height: 720, + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 1080, + Height: 720, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, + }}) + + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + + mt = NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + Type: livekit.TrackType_VIDEO, + Width: 1080, + Height: 720, + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, + }}) + + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + }) + } diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index a4261abf7..7a9dcf5ff 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -19,6 +33,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/livekit-server/pkg/telemetry" ) @@ -54,7 +69,7 @@ func (m mediaTrackReceiverState) String() string { } } -//----------------------------------------------------- +// ----------------------------------------------------- type simulcastReceiver struct { sfu.TrackReceiver @@ -206,6 +221,15 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority } func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParameters, headers []webrtc.RTPHeaderExtensionParameter) { + // The potential codecs have not published yet, so we can't get the actual Extensions, the client/browser uses same extensions + // for all video codecs so we assume they will have same extensions as the primary codec except for the dependency descriptor + // that is munged in svc codec. + headersWithoutDD := make([]webrtc.RTPHeaderExtensionParameter, 0, len(headers)) + for _, h := range headers { + if h.URI != dependencydescriptor.ExtensionUrl { + headersWithoutDD = append(headersWithoutDD, h) + } + } t.lock.Lock() t.potentialCodecs = codecs for i, c := range codecs { @@ -217,8 +241,12 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete } } if !exist { + extHeaders := headers + if !sfu.IsSvcCodec(c.MimeType) { + extHeaders = headersWithoutDD + } t.receivers = append(t.receivers, &simulcastReceiver{ - TrackReceiver: NewDummyReceiver(livekit.TrackID(t.trackInfo.Sid), string(t.PublisherID()), c, headers), + TrackReceiver: NewDummyReceiver(livekit.TrackID(t.trackInfo.Sid), string(t.PublisherID()), c, extHeaders), priority: i, }) } @@ -240,7 +268,7 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) defer t.lock.Unlock() layer := buffer.RidToSpatialLayer(rid, t.params.TrackInfo) - if layer == sfu.InvalidLayerSpatial { + if layer == buffer.InvalidLayerSpatial { // non-simulcast case will not have `rid` layer = 0 } @@ -667,11 +695,13 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki }) } - // finds the lowest layer that could satisfy client demands + // finds the highest layer with smallest dimensions that still satisfy client demands requestedSize = uint32(float32(requestedSize) * layerSelectionTolerance) for i, s := range layerSizes { quality = livekit.VideoQuality(i) - if s >= requestedSize { + if i == len(layerSizes)-1 { + break + } else if s >= requestedSize && s != layerSizes[i+1] { break } } @@ -788,4 +818,11 @@ func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(spatial int32, fps ui return buffer.DefaultMaxLayerTemporal } +func (t *MediaTrackReceiver) IsEncrypted() bool { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.trackInfo.Encryption != livekit.Encryption_NONE +} + // --------------------------- diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index af82fe224..51499a5ba 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -98,12 +112,18 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * for _, c := range codecs { c.RTCPFeedback = rtcpFeedback } + var trailer []byte + if t.params.MediaTrack.IsEncrypted() { + trailer = sub.GetTrailer() + } downTrack, err := sfu.NewDownTrack( codecs, wr, sub.GetBufferFactory(), subscriberID, t.params.ReceiverConfig.PacketBufferSize, + sub.GetPacer(), + trailer, LoggerWithTrack(sub.GetLogger(), trackID, t.params.IsRelayed), ) if err != nil { @@ -127,7 +147,11 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * // Bind callback can happen from replaceTrack, so set it up early var reusingTransceiver atomic.Bool var dtState sfu.DownTrackState - downTrack.OnBinding(func() { + downTrack.OnBinding(func(err error) { + if err != nil { + go subTrack.Bound(err) + return + } wr.DetermineReceiver(downTrack.Codec()) if reusingTransceiver.Load() { downTrack.SeedState(dtState) @@ -141,7 +165,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * ) } - go subTrack.Bound() + go subTrack.Bound(nil) subTrack.SetPublisherMuted(t.params.MediaTrack.IsMuted()) }) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 101eeecac..aea4c9e94 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1,8 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( "context" - "io" "os" "strconv" "strings" @@ -24,7 +37,10 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -33,7 +49,7 @@ import ( ) const ( - sdBatchSize = 20 + sdBatchSize = 30 rttUpdateInterval = 5 * time.Second disconnectCleanupDuration = 15 * time.Second @@ -66,6 +82,7 @@ type ParticipantParams struct { VideoConfig config.VideoConfig ProtocolVersion types.ProtocolVersion Telemetry telemetry.TelemetryService + Trailer []byte PLIThrottleConfig config.PLIThrottleConfig CongestionControlConfig config.CongestionControlConfig EnabledCodecs []*livekit.Codec @@ -80,6 +97,7 @@ type ParticipantParams struct { AdaptiveStream bool AllowTCPFallback bool TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool TURNSEnabled bool GetParticipantInfo func(pID livekit.ParticipantID) *livekit.ParticipantInfo ReconnectOnPublicationError bool @@ -87,17 +105,20 @@ type ParticipantParams struct { VersionGenerator utils.TimedVersionGenerator TrackResolver types.MediaTrackResolver DisableDynacast bool + SubscriberAllowPause bool + SubscriptionLimitAudio int32 + SubscriptionLimitVideo int32 } type ParticipantImpl struct { params ParticipantParams - isClosed atomic.Bool - state atomic.Value // livekit.ParticipantInfo_State - resSink atomic.Value // routing.MessageSink - resSinkValid atomic.Bool - grants *auth.ClaimGrants - isPublisher atomic.Bool + isClosed atomic.Bool + state atomic.Value // livekit.ParticipantInfo_State + resSinkMu sync.Mutex + resSink routing.MessageSink + grants *auth.ClaimGrants + isPublisher atomic.Bool // when first connected connectedAt time.Time @@ -121,11 +142,10 @@ type ParticipantImpl struct { *UpTrackManager *SubscriptionManager - // tracks and participants that this participant isn't allowed to subscribe to - disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID // trackID -> publisherID // keeps track of unpublished tracks in order to reuse trackID unpublishedTracks []*livekit.TrackInfo + requireBroadcast bool // queued participant updates before join response is sent // guarded by updateLock queuedUpdates []*livekit.ParticipantInfo @@ -139,9 +159,12 @@ type ParticipantImpl struct { rttUpdatedAt time.Time lastRTT uint32 - lock utils.RWMutex - once sync.Once - version atomic.Uint32 + lock utils.RWMutex + once sync.Once + + dirty atomic.Bool + version atomic.Uint32 + timedVersion utils.TimedVersion // callbacks & handlers onTrackPublished func(types.LocalParticipant, types.MediaTrack) @@ -154,13 +177,15 @@ type ParticipantImpl struct { migrateState atomic.Value // types.MigrateState - onClose func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + onClose func(types.LocalParticipant) onClaimsChanged func(participant types.LocalParticipant) onICEConfigChanged func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) cachedDownTracks map[livekit.TrackID]*downTrackState supervisor *supervisor.ParticipantSupervisor + + tracksQuality map[livekit.TrackID]livekit.ConnectionQuality } func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { @@ -178,7 +203,6 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { rtcpCh: make(chan []rtcp.Packet, 100), pendingTracks: make(map[string]*pendingTrackInfo), pendingPublishingTracks: make(map[livekit.TrackID]*pendingTrackInfo), - disallowedSubscriptions: make(map[livekit.TrackID]livekit.ParticipantID), connectedAt: time.Now(), rttUpdatedAt: time.Now(), cachedDownTracks: make(map[livekit.TrackID]*downTrackState), @@ -186,9 +210,11 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeData, params.SID), params.SID, params.Telemetry), - supervisor: supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}), + supervisor: supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}), + tracksQuality: make(map[livekit.TrackID]livekit.ConnectionQuality), } p.version.Store(params.InitialVersion) + p.timedVersion.Update(params.VersionGenerator.New()) p.migrateState.Store(types.MigrateStateInit) p.state.Store(livekit.ParticipantInfo_JOINING) p.grants = params.Grants @@ -213,6 +239,12 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { return p, nil } +func (p *ParticipantImpl) GetTrailer() []byte { + trailer := make([]byte, len(p.params.Trailer)) + copy(trailer, p.params.Trailer) + return trailer +} + func (p *ParticipantImpl) GetLogger() logger.Logger { return p.params.Logger } @@ -221,6 +253,10 @@ func (p *ParticipantImpl) GetAdaptiveStream() bool { return p.params.AdaptiveStream } +func (p *ParticipantImpl) GetPacer() pacer.Pacer { + return p.TransportManager.GetSubscriberPacer() +} + func (p *ParticipantImpl) ID() livekit.ParticipantID { return p.params.SID } @@ -269,6 +305,12 @@ func (p *ParticipantImpl) ConnectedAt() time.Time { return p.connectedAt } +func (p *ParticipantImpl) GetClientInfo() *livekit.ClientInfo { + p.lock.RLock() + defer p.lock.RUnlock() + return p.params.ClientInfo.ClientInfo +} + func (p *ParticipantImpl) GetClientConfiguration() *livekit.ClientConfiguration { p.lock.RLock() defer p.lock.RUnlock() @@ -286,16 +328,18 @@ func (p *ParticipantImpl) GetBufferFactory() *buffer.Factory { // SetName attaches name to the participant func (p *ParticipantImpl) SetName(name string) { p.lock.Lock() - changed := p.grants.Name != name + if p.grants.Name == name { + p.lock.Unlock() + return + } + p.grants.Name = name + p.dirty.Store(true) + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if !changed { - return - } - if onParticipantUpdate != nil { onParticipantUpdate(p) } @@ -307,16 +351,19 @@ func (p *ParticipantImpl) SetName(name string) { // SetMetadata attaches metadata to the participant func (p *ParticipantImpl) SetMetadata(metadata string) { p.lock.Lock() - changed := p.grants.Metadata != metadata + if p.grants.Metadata == metadata { + p.lock.Unlock() + return + } + p.grants.Metadata = metadata + p.requireBroadcast = p.requireBroadcast || metadata != "" + p.dirty.Store(true) + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if !changed { - return - } - if onParticipantUpdate != nil { onParticipantUpdate(p) } @@ -338,53 +385,46 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } p.lock.Lock() video := p.grants.Video - hasChanged := video.GetCanSubscribe() != permission.CanSubscribe || - video.GetCanPublish() != permission.CanPublish || - video.GetCanPublishData() != permission.CanPublishData || - video.Hidden != permission.Hidden || - video.Recorder != permission.Recorder - if !hasChanged { + if video.MatchesPermission(permission) { p.lock.Unlock() return false } - video.SetCanSubscribe(permission.CanSubscribe) - video.SetCanPublish(permission.CanPublish) - video.SetCanPublishData(permission.CanPublishData) - video.Hidden = permission.Hidden - video.Recorder = permission.Recorder + p.params.Logger.Infow("updating participant permission", "permission", permission) + + video.UpdateFromPermission(permission) + p.dirty.Store(true) canPublish := video.GetCanPublish() canSubscribe := video.GetCanSubscribe() + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged + + isPublisher := canPublish && p.TransportManager.IsPublisherEstablished() + p.requireBroadcast = p.requireBroadcast || isPublisher p.lock.Unlock() - // publish permission has been revoked then remove all published tracks - if !canPublish { - for _, track := range p.GetPublishedTracks() { - p.RemovePublishedTrack(track, false, false) - if p.ProtocolVersion().SupportsUnpublish() { - p.sendTrackUnpublished(track.ID()) - } else { - // for older clients that don't support unpublish, mute to avoid them sending data - p.sendTrackMuted(track.ID(), true) - } + // publish permission has been revoked then remove offending tracks + for _, track := range p.GetPublishedTracks() { + if !video.GetCanPublishSource(track.Source()) { + p.removePublishedTrack(track) } } + if canSubscribe { // reconcile everything p.SubscriptionManager.queueReconcile("") } else { // revoke all subscriptions - for _, st := range p.GetSubscribedTracks() { + for _, st := range p.SubscriptionManager.GetSubscribedTracks() { st.MediaTrack().RemoveSubscriber(p.ID(), false) } } // update isPublisher attribute - p.isPublisher.Store(canPublish && p.TransportManager.IsPublisherEstablished()) + p.isPublisher.Store(isPublisher) if onParticipantUpdate != nil { onParticipantUpdate(p) @@ -395,24 +435,43 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return true } -func (p *ParticipantImpl) ToProto() *livekit.ParticipantInfo { +func (p *ParticipantImpl) CanSkipBroadcast() bool { p.lock.RLock() - info := &livekit.ParticipantInfo{ + defer p.lock.RUnlock() + return !p.requireBroadcast +} + +func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { + v := p.version.Load() + piv := p.timedVersion.Load() + if p.dirty.Swap(false) { + v = p.version.Inc() + piv = p.params.VersionGenerator.Next() + p.timedVersion.Update(&piv) + } + + p.lock.RLock() + pi := &livekit.ParticipantInfo{ Sid: string(p.params.SID), Identity: string(p.params.Identity), Name: p.grants.Name, State: p.State(), JoinedAt: p.ConnectedAt().Unix(), - Version: p.version.Inc(), + Version: v, Permission: p.grants.Video.ToPermission(), Metadata: p.grants.Metadata, Region: p.params.Region, IsPublisher: p.IsPublisher(), } p.lock.RUnlock() - info.Tracks = p.UpTrackManager.ToProto() + pi.Tracks = p.UpTrackManager.ToProto() - return info + return pi, piv +} + +func (p *ParticipantImpl) ToProto() *livekit.ParticipantInfo { + pi, _ := p.ToProtoWithVersion() + return pi } // callbacks for clients @@ -466,7 +525,7 @@ func (p *ParticipantImpl) OnDataPacket(callback func(types.LocalParticipant, *li p.lock.Unlock() } -func (p *ParticipantImpl) OnClose(callback func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) { +func (p *ParticipantImpl) OnClose(callback func(types.LocalParticipant)) { p.lock.Lock() p.onClose = callback p.lock.Unlock() @@ -478,9 +537,18 @@ func (p *ParticipantImpl) OnClaimsChanged(callback func(types.LocalParticipant)) p.lock.Unlock() } +func (p *ParticipantImpl) HandleSignalSourceClose() { + p.TransportManager.SetSignalSourceValid(false) + + if !p.TransportManager.HasPublisherEverConnected() && !p.TransportManager.HasSubscriberEverConnected() { + p.params.Logger.Infow("closing disconnected participant") + _ = p.Close(false, types.ParticipantCloseReasonJoinFailed, false) + } +} + // HandleOffer an offer from remote participant, used when clients make the initial connection func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) { - p.params.Logger.Infow("received offer", "transport", livekit.SignalTarget_PUBLISHER) + p.params.Logger.Debugw("received offer", "transport", livekit.SignalTarget_PUBLISHER) shouldPend := false if p.MigrateState() == types.MigrateStateInit { shouldPend = true @@ -494,7 +562,7 @@ func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) { // HandleAnswer handles a client answer response, with subscriber PC, server initiates the // offer and client answers func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { - p.params.Logger.Infow("received answer", "transport", livekit.SignalTarget_SUBSCRIBER) + p.params.Logger.Debugw("received answer", "transport", livekit.SignalTarget_SUBSCRIBER) /* from server received join request to client answer * 1. server send join response & offer @@ -508,7 +576,11 @@ func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { } func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) error { - p.params.Logger.Infow("sending answer", "transport", livekit.SignalTarget_PUBLISHER) + if p.IsClosed() || p.IsDisconnected() { + return nil + } + + p.params.Logger.Debugw("sending answer", "transport", livekit.SignalTarget_PUBLISHER) answer = p.configurePublisherAnswer(answer) if err := p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Answer{ @@ -518,10 +590,6 @@ func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) er return err } - // received an offer from the client, if publishing is allowed, mark this - // participant as a publisher - p.setIsPublisher(p.CanPublish()) - if p.MigrateState() == types.MigrateStateSync { go p.handleMigrateMutedTrack() } @@ -538,11 +606,11 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { } if len(pti.trackInfos) > 1 { - p.params.Logger.Warnw("too many pending migrated tracks", nil, "count", len(pti.trackInfos), "cid", cid) + p.params.Logger.Warnw("too many pending migrated tracks", nil, "trackID", pti.trackInfos[0].Sid, "count", len(pti.trackInfos), "cid", cid) } ti := pti.trackInfos[0] - if ti.Muted && ti.Type == livekit.TrackType_VIDEO { + if ti.Muted { mt := p.addMigrateMutedTrack(cid, ti) if mt != nil { addedTracks = append(addedTracks, mt) @@ -552,6 +620,10 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { } } p.mutedTrackNotFired = append(p.mutedTrackNotFired, addedTracks...) + + if len(addedTracks) != 0 { + p.dirty.Store(true) + } p.pendingTracksLock.Unlock() // launch callbacks in goroutine since they could block. @@ -578,14 +650,13 @@ func (p *ParticipantImpl) removeMutedTrackNotFired(mt *MediaTrack) { // AddTrack is called when client intends to publish track. // records track details and lets client know it's ok to proceed func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { - p.lock.Lock() - defer p.lock.Unlock() - - if !p.grants.Video.GetCanPublish() { + if !p.CanPublishSource(req.Source) { p.params.Logger.Warnw("no permission to publish track", nil) return } + p.lock.Lock() + defer p.lock.Unlock() ti := p.addPendingTrackLocked(req) if ti == nil { return @@ -607,6 +678,7 @@ func (p *ParticipantImpl) SetMigrateInfo( p.supervisor.SetPublicationMute(livekit.TrackID(ti.Sid), ti.Muted) p.pendingTracks[t.GetCid()] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}, migrated: true} + p.params.Logger.Infow("pending track added (migration)", "trackID", ti.Sid, "track", ti.String()) } p.pendingTracksLock.Unlock() @@ -619,13 +691,13 @@ func (p *ParticipantImpl) Start() { }) } -func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseReason) error { +func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseReason, isExpectedToResume bool) error { if p.isClosed.Swap(true) { // already closed return nil } - p.params.Logger.Infow("participant closing", "sendLeave", sendLeave, "reason", reason.String()) + p.params.Logger.Infow("participant closing", "sendLeave", sendLeave, "reason", reason.String(), "isExpectedToResume", isExpectedToResume) p.clearDisconnectTimer() p.clearMigrationTimer() @@ -649,33 +721,26 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.pendingTracksLock.Unlock() for _, t := range closeMutedTrack { - t.Close(!sendLeave) + t.Close(isExpectedToResume) } - p.UpTrackManager.Close(!sendLeave) - - p.lock.Lock() - disallowedSubscriptions := make(map[livekit.TrackID]livekit.ParticipantID) - for trackID, publisherID := range p.disallowedSubscriptions { - disallowedSubscriptions[trackID] = publisherID - } - p.lock.Unlock() + p.UpTrackManager.Close(isExpectedToResume) p.updateState(livekit.ParticipantInfo_DISCONNECTED) // ensure this is synchronized - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) p.lock.RLock() onClose := p.onClose p.lock.RUnlock() if onClose != nil { - onClose(p, disallowedSubscriptions) + onClose(p) } // Close peer connections without blocking participant Close. If peer connections are gathering candidates // Close will block. go func() { - p.SubscriptionManager.Close(!sendLeave) + p.SubscriptionManager.Close(isExpectedToResume) p.TransportManager.Close() }() @@ -687,7 +752,7 @@ func (p *ParticipantImpl) IsClosed() bool { return p.isClosed.Load() } -// Negotiate subscriber SDP with client, if force is true, will cencel pending +// Negotiate subscriber SDP with client, if force is true, will cancel pending // negotiate task and negotiate immediately func (p *ParticipantImpl) Negotiate(force bool) { if p.MigrateState() != types.MigrateStateInit { @@ -705,7 +770,11 @@ func (p *ParticipantImpl) clearMigrationTimer() { } func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { - if !force && !p.TransportManager.HaveAllTransportEverConnected() { + allTransportConnected := p.TransportManager.HasSubscriberEverConnected() + if p.IsPublisher() { + allTransportConnected = allTransportConnected && p.TransportManager.HasPublisherEverConnected() + } + if !force && !allTransportConnected { return false } @@ -713,7 +782,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { onStart() } - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonMigration) // // On subscriber peer connection, remote side will try ICE on both @@ -732,7 +801,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { p.migrationTimer = time.AfterFunc(migrationWaitDuration, func() { p.clearMigrationTimer() - if p.isClosed.Load() || p.IsDisconnected() { + if p.IsClosed() || p.IsDisconnected() { return } // TODO: change to debug once we are confident @@ -759,8 +828,9 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { return } - p.params.Logger.Debugw("SetMigrateState", "state", s) + p.params.Logger.Infow("SetMigrateState", "state", s) p.migrateState.Store(s) + p.dirty.Store(true) processPendingOffer := false if s == types.MigrateStateSync { @@ -785,7 +855,7 @@ func (p *ParticipantImpl) MigrateState() types.MigrateState { } // ICERestart restarts subscriber ICE connections -func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) { +func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig) { p.clearDisconnectTimer() p.clearMigrationTimer() @@ -793,7 +863,7 @@ func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig, reason liveki t.(types.LocalMediaTrack).Restart() } - p.TransportManager.ICERestart(iceConfig, reason == livekit.ReconnectReason_RR_PUBLISHER_FAILED || reason == livekit.ReconnectReason_RR_SUBSCRIBER_FAILED) + p.TransportManager.ICERestart(iceConfig) } func (p *ParticipantImpl) OnICEConfigChanged(f func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) { @@ -826,7 +896,11 @@ func (p *ParticipantImpl) GetAudioLevel() (level float64, active bool) { func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo { numTracks := 0 minQuality := livekit.ConnectionQuality_EXCELLENT - minScore := float32(0.0) + minScore := connectionquality.MaxMOS + numUpDrops := 0 + numDownDrops := 0 + + availableTracks := make(map[livekit.TrackID]bool) for _, pt := range p.GetPublishedTracks() { numTracks++ @@ -839,6 +913,19 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo } else if quality == minQuality && score < minScore { minScore = score } + + p.lock.Lock() + trackID := pt.ID() + if prevQuality, ok := p.tracksQuality[trackID]; ok { + // WARNING NOTE: comparing protobuf enums directly + if prevQuality > quality { + numUpDrops++ + } + } + p.tracksQuality[trackID] = quality + p.lock.Unlock() + + availableTracks[trackID] = true } subscribedTracks := p.SubscriptionManager.GetSubscribedTracks() @@ -853,12 +940,31 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo } else if quality == minQuality && score < minScore { minScore = score } + + p.lock.Lock() + trackID := subTrack.ID() + if prevQuality, ok := p.tracksQuality[trackID]; ok { + // WARNING NOTE: comparing protobuf enums directly + if prevQuality > quality { + numDownDrops++ + } + } + p.tracksQuality[trackID] = quality + p.lock.Unlock() + + availableTracks[trackID] = true } - if numTracks == 0 { - minQuality = livekit.ConnectionQuality_EXCELLENT - minScore = connectionquality.MaxMOS + prometheus.RecordQuality(minQuality, minScore, numUpDrops, numDownDrops) + + // remove unavailable tracks from track quality cache + p.lock.Lock() + for trackID := range p.tracksQuality { + if !availableTracks[trackID] { + delete(p.tracksQuality, trackID) + } } + p.lock.Unlock() return &livekit.ConnectionQualityInfo{ ParticipantSid: string(p.ID()), @@ -871,11 +977,10 @@ func (p *ParticipantImpl) IsPublisher() bool { return p.isPublisher.Load() } -func (p *ParticipantImpl) CanPublish() bool { +func (p *ParticipantImpl) CanPublishSource(source livekit.TrackSource) bool { p.lock.RLock() defer p.lock.RUnlock() - - return p.grants.Video.GetCanPublish() + return p.grants.Video.GetCanPublishSource(source) } func (p *ParticipantImpl) CanSubscribe() bool { @@ -928,7 +1033,10 @@ func (p *ParticipantImpl) onTrackSubscribed(subTrack types.SubscribedTrack) { subTrack.DownTrack().SetActivePaddingOnMuteUpTrack() } - subTrack.AddOnBind(func() { + subTrack.AddOnBind(func(err error) { + if err != nil { + return + } if p.TransportManager.HasSubscriberEverConnected() { subTrack.DownTrack().SetConnected() } @@ -942,14 +1050,6 @@ func (p *ParticipantImpl) onTrackUnsubscribed(subTrack types.SubscribedTrack) { } func (p *ParticipantImpl) SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) { - p.lock.Lock() - if allowed { - delete(p.disallowedSubscriptions, trackID) - } else { - p.disallowedSubscriptions[trackID] = publisherID - } - p.lock.Unlock() - p.params.Logger.Debugw("sending subscription permission update", "publisherID", publisherID, "trackID", trackID, "allowed", allowed) err := p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_SubscriptionPermissionUpdate{ @@ -988,20 +1088,21 @@ func (p *ParticipantImpl) setupTransportManager() error { SID: p.params.SID, // primary connection does not change, canSubscribe can change if permission was updated // after the participant has joined - SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), - Config: p.params.Config, - ProtocolVersion: p.params.ProtocolVersion, - Telemetry: p.params.Telemetry, - CongestionControlConfig: p.params.CongestionControlConfig, - EnabledCodecs: p.params.EnabledCodecs, - SimTracks: p.params.SimTracks, - ClientConf: p.params.ClientConf, - ClientInfo: p.params.ClientInfo, - Migration: p.params.Migration, - AllowTCPFallback: p.params.AllowTCPFallback, - TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, - TURNSEnabled: p.params.TURNSEnabled, - Logger: p.params.Logger, + SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), + Config: p.params.Config, + ProtocolVersion: p.params.ProtocolVersion, + Telemetry: p.params.Telemetry, + CongestionControlConfig: p.params.CongestionControlConfig, + EnabledCodecs: p.params.EnabledCodecs, + SimTracks: p.params.SimTracks, + ClientConf: p.params.ClientConf, + ClientInfo: p.params.ClientInfo, + Migration: p.params.Migration, + AllowTCPFallback: p.params.AllowTCPFallback, + TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, + AllowUDPUnstableFallback: p.params.AllowUDPUnstableFallback, + TURNSEnabled: p.params.TURNSEnabled, + Logger: p.params.Logger, }) if err != nil { return err @@ -1047,6 +1148,8 @@ func (p *ParticipantImpl) setupTransportManager() error { tm.OnAnyTransportNegotiationFailed(p.onAnyTransportNegotiationFailed) tm.OnDataMessage(p.onDataMessage) + + tm.SetSubscriberAllowPause(p.params.SubscriberAllowPause) p.TransportManager = tm return nil } @@ -1062,6 +1165,8 @@ func (p *ParticipantImpl) setupUpTrackManager() { p.lock.RLock() onTrackUpdated := p.onTrackUpdated p.lock.RUnlock() + + p.dirty.Store(true) if onTrackUpdated != nil { onTrackUpdated(p, track) } @@ -1072,13 +1177,15 @@ func (p *ParticipantImpl) setupUpTrackManager() { func (p *ParticipantImpl) setupSubscriptionManager() { p.SubscriptionManager = NewSubscriptionManager(SubscriptionManagerParams{ - Participant: p, - Logger: p.params.Logger.WithoutSampler(), - TrackResolver: p.params.TrackResolver, - Telemetry: p.params.Telemetry, - OnTrackSubscribed: p.onTrackSubscribed, - OnTrackUnsubscribed: p.onTrackUnsubscribed, - OnSubscriptionError: p.onSubscriptionError, + Participant: p, + Logger: p.params.Logger.WithoutSampler(), + TrackResolver: p.params.TrackResolver, + Telemetry: p.params.Telemetry, + OnTrackSubscribed: p.onTrackSubscribed, + OnTrackUnsubscribed: p.onTrackUnsubscribed, + OnSubscriptionError: p.onSubscriptionError, + SubscriptionLimitVideo: p.params.SubscriptionLimitVideo, + SubscriptionLimitAudio: p.params.SubscriptionLimitAudio, }) } @@ -1087,8 +1194,11 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { if state == oldState { return } - p.state.Store(state) + p.params.Logger.Debugw("updating participant state", "state", state.String()) + p.state.Store(state) + p.dirty.Store(true) + p.lock.RLock() onStateChange := p.onStateChange p.lock.RUnlock() @@ -1105,23 +1215,31 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { } func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { - if p.isPublisher.Swap(isPublisher) != isPublisher { - // trigger update as well if participant is already fully connected - if p.State() == livekit.ParticipantInfo_ACTIVE { - p.lock.RLock() - onParticipantUpdate := p.onParticipantUpdate - p.lock.RUnlock() + if p.isPublisher.Swap(isPublisher) == isPublisher { + return + } - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.lock.Lock() + p.requireBroadcast = true + p.lock.Unlock() + + p.dirty.Store(true) + + // trigger update as well if participant is already fully connected + if p.State() == livekit.ParticipantInfo_ACTIVE { + p.lock.RLock() + onParticipantUpdate := p.onParticipantUpdate + p.lock.RUnlock() + + if onParticipantUpdate != nil { + onParticipantUpdate(p) } } } // when the server has an offer for participant func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) error { - p.params.Logger.Infow("sending offer", "transport", livekit.SignalTarget_SUBSCRIBER) + p.params.Logger.Debugw("sending offer", "transport", livekit.SignalTarget_SUBSCRIBER) return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Offer{ Offer: ToProtoSessionDescription(offer), @@ -1129,28 +1247,24 @@ func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) err }) } +func (p *ParticipantImpl) removePublishedTrack(track types.MediaTrack) { + p.RemovePublishedTrack(track, false, false) + if p.ProtocolVersion().SupportsUnpublish() { + p.sendTrackUnpublished(track.ID()) + } else { + // for older clients that don't support unpublish, mute to avoid them sending data + p.sendTrackMuted(track.ID(), true) + } +} + // when a new remoteTrack is created, creates a Track and adds it to room func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { if p.IsDisconnected() { return } - if !p.CanPublish() { - p.params.Logger.Warnw("no permission to publish mediaTrack", nil) - return - } - publishedTrack, isNewTrack := p.mediaTrackReceived(track, rtpReceiver) - - if publishedTrack != nil { - p.params.Logger.Infow("mediaTrack published", - "kind", track.Kind().String(), - "trackID", publishedTrack.ID(), - "rid", track.RID(), - "SSRC", track.SSRC(), - "mime", track.Codec().MimeType, - ) - } else { + if publishedTrack == nil { p.params.Logger.Warnw("webrtc Track published but can't find MediaTrack", nil, "kind", track.Kind().String(), "webrtcTrackID", track.ID(), @@ -1158,9 +1272,30 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w "SSRC", track.SSRC(), "mime", track.Codec().MimeType, ) + return } - if !isNewTrack && publishedTrack != nil && !publishedTrack.HasPendingCodec() && p.IsReady() { + if !p.CanPublishSource(publishedTrack.Source()) { + p.params.Logger.Warnw("no permission to publish mediaTrack", nil, + "source", publishedTrack.Source(), + ) + p.removePublishedTrack(publishedTrack) + return + } + + p.setIsPublisher(true) + p.dirty.Store(true) + + p.params.Logger.Infow("mediaTrack published", + "kind", track.Kind().String(), + "trackID", publishedTrack.ID(), + "webrtcTrackID", track.ID(), + "rid", track.RID(), + "SSRC", track.SSRC(), + "mime", track.Codec().MimeType, + ) + + if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { p.lock.RLock() onTrackUpdated := p.onTrackUpdated p.lock.RUnlock() @@ -1200,13 +1335,11 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt p.params.Logger.Warnw("received unsupported data packet", nil, "payload", payload) } - if !p.IsPublisher() { - p.setIsPublisher(true) - } + p.setIsPublisher(true) } func (p *ParticipantImpl) onICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { - if c == nil || p.IsDisconnected() { + if c == nil || p.IsDisconnected() || p.IsClosed() { return nil } @@ -1254,18 +1387,18 @@ func (p *ParticipantImpl) setupDisconnectTimer() { p.disconnectTimer = time.AfterFunc(disconnectCleanupDuration, func() { p.clearDisconnectTimer() - if p.isClosed.Load() || p.IsDisconnected() { + if p.IsClosed() || p.IsDisconnected() { return } p.params.Logger.Infow("closing disconnected participant") - _ = p.Close(true, types.ParticipantCloseReasonPeerConnectionDisconnected) + _ = p.Close(true, types.ParticipantCloseReasonPeerConnectionDisconnected, false) }) p.lock.Unlock() } func (p *ParticipantImpl) onAnyTransportFailed() { // clients support resuming of connections when websocket becomes disconnected - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonTransportFailure) // detect when participant has actually left. p.setupDisconnectTimer() @@ -1284,61 +1417,56 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { return } - var srs []rtcp.Packet - var sd []rtcp.SourceDescriptionChunk subscribedTracks := p.SubscriptionManager.GetSubscribedTracks() - p.lock.RLock() + + // send in batches of sdBatchSize + batchSize := 0 + var pkts []rtcp.Packet + var sd []rtcp.SourceDescriptionChunk for _, subTrack := range subscribedTracks { sr := subTrack.DownTrack().CreateSenderReport() chunks := subTrack.DownTrack().CreateSourceDescriptionChunks() if sr == nil || chunks == nil { continue } - srs = append(srs, sr) + + pkts = append(pkts, sr) sd = append(sd, chunks...) - } - p.lock.RUnlock() - - // now send in batches of sdBatchSize - var batch []rtcp.SourceDescriptionChunk - var pkts []rtcp.Packet - batchSize := 0 - for len(sd) > 0 || len(srs) > 0 { - numSRs := len(srs) - if numSRs > 0 { - if numSRs > sdBatchSize { - numSRs = sdBatchSize + batchSize = batchSize + 1 + len(chunks) + if batchSize >= sdBatchSize { + if len(sd) != 0 { + pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) } - pkts = append(pkts, srs[:numSRs]...) - srs = srs[numSRs:] - } - - size := len(sd) - spaceRemain := sdBatchSize - batchSize - if spaceRemain > 0 && size > 0 { - if size > spaceRemain { - size = spaceRemain - } - batch = sd[:size] - sd = sd[size:] - pkts = append(pkts, &rtcp.SourceDescription{Chunks: batch}) if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { - if err == io.EOF || err == io.ErrClosedPipe { + if IsEOF(err) { return } p.params.Logger.Errorw("could not send down track reports", err) } - } - pkts = pkts[:0] - batchSize = 0 + pkts = pkts[:0] + sd = sd[:0] + batchSize = 0 + } } - time.Sleep(5 * time.Second) + if len(pkts) != 0 || len(sd) != 0 { + if len(sd) != 0 { + pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) + } + if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { + if IsEOF(err) { + return + } + p.params.Logger.Errorw("could not send down track reports", err) + } + } + + time.Sleep(3 * time.Second) } } -func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) error { +func (p *ParticipantImpl) onStreamStateChange(update *streamallocator.StreamStateUpdate) error { if len(update.StreamStates) == 0 { return nil } @@ -1346,7 +1474,7 @@ func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) err streamStateUpdate := &livekit.StreamStateUpdate{} for _, streamStateInfo := range update.StreamStates { state := livekit.StreamState_ACTIVE - if streamStateInfo.State == sfu.StreamStatePaused { + if streamStateInfo.State == streamallocator.StreamStatePaused { state = livekit.StreamState_PAUSED } streamStateUpdate.StreamStates = append(streamStateUpdate.StreamStates, &livekit.StreamStateInfo{ @@ -1363,7 +1491,7 @@ func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) err }) } -func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualites []types.SubscribedCodecQuality) error { +func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) error { if p.params.DisableDynacast { return nil } @@ -1394,7 +1522,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, } } - for _, maxSubscribedQuality := range maxSubscribedQualites { + for _, maxSubscribedQuality := range maxSubscribedQualities { ti := &livekit.TrackInfo{ Sid: string(trackID), Type: livekit.TrackType_VIDEO, @@ -1417,7 +1545,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, "sending max subscribed quality", "trackID", trackID, "qualities", subscribedQualities, - "max", maxSubscribedQualites, + "max", maxSubscribedQualities, ) return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_SubscribedQualityUpdate{ @@ -1463,10 +1591,12 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { mime = "audio/" + mime } - ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mime, - Cid: codec.Cid, - }) + if IsCodecEnabled(p.params.EnabledCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + MimeType: mime, + Cid: codec.Cid, + }) + } } p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), ti) @@ -1509,6 +1639,7 @@ func (p *ParticipantImpl) SetTrackMuted(trackID livekit.TrackID, muted bool, fro } func (p *ParticipantImpl) setTrackMuted(trackID livekit.TrackID, muted bool) { + p.dirty.Store(true) p.supervisor.SetPublicationMute(trackID, muted) track := p.UpTrackManager.SetPublishedTrackMuted(trackID, muted) @@ -1573,6 +1704,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei ti.MimeType = track.Codec().MimeType mt = p.addMediaTrack(signalCid, track.ID(), ti) newTrack = true + p.dirty.Store(true) } ssrc := uint32(track.SSRC()) @@ -1595,10 +1727,10 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { - p.params.Logger.Debugw("add migrate muted track", "cid", cid, "track", ti.String()) + p.params.Logger.Infow("add migrate muted track", "cid", cid, "trackID", ti.Sid, "track", ti.String()) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { - p.params.Logger.Errorw("could not find receiver for migrated track", nil, "track", ti.Sid) + p.params.Logger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid) return nil } @@ -1614,6 +1746,24 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo } } } + // check for mime_type for tracks that do not have simulcast_codecs set + if ti.MimeType != "" { + for _, nc := range parameters.Codecs { + if strings.EqualFold(nc.MimeType, ti.MimeType) { + alreadyAdded := false + for _, pc := range potentialCodecs { + if strings.EqualFold(pc.MimeType, ti.MimeType) { + alreadyAdded = true + break + } + } + if !alreadyAdded { + potentialCodecs = append(potentialCodecs, nc) + } + break + } + } + } mt.SetPotentialCodecs(potentialCodecs, parameters.HeaderExtensions) for _, codec := range ti.Codecs { @@ -1693,6 +1843,8 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv } p.pendingTracksLock.Unlock() + p.dirty.Store(true) + if !p.IsClosed() { // unpublished events aren't necessary when participant is closed p.params.Logger.Infow("unpublished track", "trackID", ti.Sid, "trackInfo", ti) @@ -1763,12 +1915,6 @@ func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackTyp if pendingInfo == nil { track_loop: for cid, pti := range p.pendingTracks { - if cid == clientId { - pendingInfo = pti - signalCid = cid - break - } - ti := pti.trackInfos[0] for _, c := range ti.Codecs { if c.Cid == clientId { @@ -1894,12 +2040,14 @@ func (p *ParticipantImpl) publisherRTCPWorker() { // read from rtcpChan for pkts := range p.rtcpCh { if pkts == nil { - p.params.Logger.Infow("exiting publisher RTCP worker") + p.params.Logger.Debugw("exiting publisher RTCP worker") return } if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil { - p.params.Logger.Errorw("could not write RTCP to participant", err) + if !IsEOF(err) { + p.params.Logger.Errorw("could not write RTCP to participant", err) + } } } } @@ -1940,7 +2088,7 @@ func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { } func (p *ParticipantImpl) setDowntracksConnected() { - for _, t := range p.GetSubscribedTracks() { + for _, t := range p.SubscriptionManager.GetSubscribedTracks() { if dt := t.DownTrack(); dt != nil { dt.SetConnected() } @@ -1988,10 +2136,20 @@ func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason }, }, }) - p.CloseSignalConnection() - // on a full reconnect, no need to supervise this participant anymore - p.supervisor.Stop() + scr := types.SignallingCloseReasonUnknown + switch reason { + case types.ParticipantCloseReasonPublicationError: + scr = types.SignallingCloseReasonFullReconnectPublicationError + case types.ParticipantCloseReasonSubscriptionError: + scr = types.SignallingCloseReasonFullReconnectSubscriptionError + case types.ParticipantCloseReasonNegotiateFailed: + scr = types.SignallingCloseReasonFullReconnectNegotiateFailed + } + p.CloseSignalConnection(scr) + + // a full reconnect == client should connect back with a new session, close current one + p.Close(false, reason, false) } func (p *ParticipantImpl) onPublicationError(trackID livekit.TrackID) { @@ -2001,10 +2159,27 @@ func (p *ParticipantImpl) onPublicationError(trackID livekit.TrackID) { } } -func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID) { - if p.params.ReconnectOnSubscriptionError { +func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal bool, err error) { + signalErr := livekit.SubscriptionError_SE_UNKNOWN + switch { + case errors.Is(err, webrtc.ErrUnsupportedCodec): + signalErr = livekit.SubscriptionError_SE_CODEC_UNSUPPORTED + case errors.Is(err, ErrTrackNotFound): + signalErr = livekit.SubscriptionError_SE_TRACK_NOTFOUND + } + + _ = p.writeMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscriptionResponse{ + SubscriptionResponse: &livekit.SubscriptionResponse{ + TrackSid: string(trackID), + Err: signalErr, + }, + }, + }) + + if p.params.ReconnectOnSubscriptionError && fatal { p.params.Logger.Infow("issuing full reconnect on subscription error", "trackID", trackID) - p.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) + p.IssueFullReconnect(types.ParticipantCloseReasonSubscriptionError) } } diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 509282d03..55f773879 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -1,6 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( + "fmt" "strings" "testing" "time" @@ -175,6 +190,32 @@ func TestTrackPublishing(t *testing.T) { // check SID is the same require.Equal(t, p.pendingTracks["cid"].trackInfos[0].Sid, p.pendingTracks["cid"].trackInfos[1].Sid) }) + + t.Run("should not allow adding disallowed sources", func(t *testing.T) { + p := newParticipantForTest("test") + p.SetPermission(&livekit.ParticipantPermission{ + CanPublish: true, + CanPublishSources: []livekit.TrackSource{ + livekit.TrackSource_CAMERA, + }, + }) + sink := p.params.Sink.(*routingfakes.FakeMessageSink) + p.AddTrack(&livekit.AddTrackRequest{ + Cid: "cid", + Name: "webcam", + Source: livekit.TrackSource_CAMERA, + Type: livekit.TrackType_VIDEO, + }) + require.Equal(t, 1, sink.WriteMessageCallCount()) + + p.AddTrack(&livekit.AddTrackRequest{ + Cid: "cid2", + Name: "rejected source", + Type: livekit.TrackType_AUDIO, + Source: livekit.TrackSource_MICROPHONE, + }) + require.Equal(t, 1, sink.WriteMessageCallCount()) + }) } func TestOutOfOrderUpdates(t *testing.T) { @@ -202,7 +243,7 @@ func TestOutOfOrderUpdates(t *testing.T) { func TestDisconnectTiming(t *testing.T) { t.Run("Negotiate doesn't panic after channel closed", func(t *testing.T) { p := newParticipantForTest("test") - msg := routing.NewMessageChannel(routing.DefaultMessageChannelSize) + msg := routing.NewMessageChannel(livekit.ConnectionID("test"), routing.DefaultMessageChannelSize) p.params.Sink = msg go func() { for msg := range msg.ReadChan() { @@ -361,7 +402,7 @@ func TestSetStableTrackID(t *testing.T) { } func TestDisableCodecs(t *testing.T) { - participant := newParticipantForTestWithOpts(livekit.ParticipantIdentity("123"), &participantOpts{ + participant := newParticipantForTestWithOpts("123", &participantOpts{ publisher: false, clientConf: &livekit.ClientConfiguration{ DisabledCodecs: &livekit.DisabledCodecs{ @@ -395,7 +436,7 @@ func TestDisableCodecs(t *testing.T) { participant.SetResponseSink(sink) var answer webrtc.SessionDescription var answerReceived atomic.Bool - sink.WriteMessageStub = func(msg proto.Message) error { + sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { answer = FromProtoSessionDescription(res.GetAnswer()) @@ -403,7 +444,7 @@ func TestDisableCodecs(t *testing.T) { } } return nil - } + }) participant.HandleOffer(sdp) testutils.WithTimeout(t, func() string { @@ -425,6 +466,184 @@ func TestDisableCodecs(t *testing.T) { require.False(t, found264) } +func TestPreferVideoCodecForPublisher(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) + + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for i := 0; i < 2; i++ { + // publish h264 track without client preferred codec + trackCid := fmt.Sprintf("preferh264video%d", i) + participant.AddTrack(&livekit.AddTrackRequest{ + Type: livekit.TrackType_VIDEO, + Name: "video", + Width: 1280, + Height: 720, + Source: livekit.TrackSource_CAMERA, + SimulcastCodecs: []*livekit.SimulcastCodec{ + { + Codec: "h264", + Cid: trackCid, + }, + }, + }) + + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + pc.SetLocalDescription(sdp) + codecs := transceiver.Receiver().GetParameters().Codecs + + // h264 should not be preferred + require.NotEqual(t, codecs[0].MimeType, "video/h264") + + sink := &routingfakes.FakeMessageSink{} + participant.SetResponseSink(sink) + var answer webrtc.SessionDescription + var answerReceived atomic.Bool + sink.WriteMessageCalls(func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer = FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + } + } + return nil + }) + participant.HandleOffer(sdp) + + require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + + var h264Preferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var videoSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == "video" { + if videoSectionIndex == i { + codecs, err := codecsFromMediaDescription(m) + require.NoError(t, err) + if strings.EqualFold(codecs[0].Name, "h264") { + h264Preferred = true + break + } + } + videoSectionIndex++ + } + } + + require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP) + } +} + +func TestPreferAudioCodecForRed(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) + + me := webrtc.MediaEngine{} + me.RegisterDefaultCodecs() + require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: redCodecCapability, + PayloadType: 63, + }, webrtc.RTPCodecTypeAudio)) + + api := webrtc.NewAPI(webrtc.WithMediaEngine(&me)) + pc, err := api.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for i, disableRed := range []bool{false, true} { + t.Run(fmt.Sprintf("disableRed=%v", disableRed), func(t *testing.T) { + trackCid := fmt.Sprintf("audiotrack%d", i) + participant.AddTrack(&livekit.AddTrackRequest{ + Type: livekit.TrackType_AUDIO, + DisableRed: disableRed, + Cid: trackCid, + }) + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "audio/opus"}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + codecs := transceiver.Sender().GetParameters().Codecs + for i, c := range codecs { + if c.MimeType == "audio/opus" && i != 0 { + codecs[0], codecs[i] = codecs[i], codecs[0] + break + } + } + transceiver.SetCodecPreferences(codecs) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + pc.SetLocalDescription(sdp) + // opus should be preferred + require.Equal(t, codecs[0].MimeType, "audio/opus", sdp) + + sink := &routingfakes.FakeMessageSink{} + participant.SetResponseSink(sink) + var answer webrtc.SessionDescription + var answerReceived atomic.Bool + sink.WriteMessageCalls(func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer = FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + } + } + return nil + }) + participant.HandleOffer(sdp) + + require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + + var redPreferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var audioSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == "audio" { + if audioSectionIndex == i { + codecs, err := codecsFromMediaDescription(m) + require.NoError(t, err) + // nack is always enabled. if red is preferred, server will not generate nack request + var nackEnabled bool + for _, c := range codecs { + if c.Name == "opus" { + for _, fb := range c.RTCPFeedback { + if strings.Contains(fb, "nack") { + nackEnabled = true + break + } + } + } + } + require.True(t, nackEnabled, "nack should be enabled for opus") + + if strings.EqualFold(codecs[0].Name, "red") { + redPreferred = true + break + } + } + audioSectionIndex++ + } + } + require.Equalf(t, !disableRed, redPreferred, "offer : \n%s\nanswer sdp: \n%s", sdp, answer.SDP) + }) + } + +} + type participantOpts struct { permissions *livekit.ParticipantPermission protocolVersion types.ProtocolVersion @@ -444,7 +663,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p // disable mux, it doesn't play too well with unit test conf.RTC.UDPPort = 0 conf.RTC.TCPPort = 0 - rtcConf, err := NewWebRTCConfig(conf, "") + rtcConf, err := NewWebRTCConfig(conf) if err != nil { panic(err) } @@ -478,6 +697,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p ClientInfo: ClientInfo{ClientInfo: opts.clientInfo}, Logger: LoggerWithParticipant(logger.GetLogger(), identity, sid, false), Telemetry: &telemetryfakes.FakeTelemetryService{}, + VersionGenerator: utils.NewDefaultTimedVersionGenerator(), }) p.isPublisher.Store(opts.publisher) p.updateState(livekit.ParticipantInfo_ACTIVE) diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 6ae4b8715..01a50b6b0 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -112,8 +126,15 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess continue } + var info *livekit.TrackInfo p.pendingTracksLock.RLock() - _, info := p.getPendingTrack(streamID, livekit.TrackType_VIDEO) + mt := p.getPublishedTrackBySdpCid(streamID) + if mt != nil { + info = mt.ToProto() + } else { + _, info = p.getPendingTrack(streamID, livekit.TrackType_VIDEO) + } + if info == nil { p.pendingTracksLock.RUnlock() continue @@ -131,8 +152,8 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess p.pendingTracksLock.RUnlock() mime = strings.ToUpper(mime) - // remove dd extension if av1 not preferred - if !strings.Contains(mime, "AV1") { + // remove dd extension if av1/vp9 not preferred + if !strings.Contains(strings.ToLower(mime), "av1") && !strings.Contains(strings.ToLower(mime), "vp9") { for i, attr := range unmatchVideo.Attributes { if strings.Contains(attr.Value, dd.ExtensionUrl) { unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 60c5e4e79..a9ae4dfab 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -1,33 +1,43 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( + "errors" "fmt" "time" "github.com/pion/webrtc/v3" "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" ) func (p *ParticipantImpl) getResponseSink() routing.MessageSink { - if !p.resSinkValid.Load() { - return nil - } - sink := p.resSink.Load() - if s, ok := sink.(routing.MessageSink); ok { - return s - } - return nil + p.resSinkMu.Lock() + defer p.resSinkMu.Unlock() + return p.resSink } func (p *ParticipantImpl) SetResponseSink(sink routing.MessageSink) { - p.resSinkValid.Store(sink != nil) - if sink != nil { - // cannot store nil into atomic.Value - p.resSink.Store(sink) - } + p.resSinkMu.Lock() + defer p.resSinkMu.Unlock() + p.resSink = sink } func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) error { @@ -90,6 +100,10 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. isValid = false } } + if pi.Permission != nil && pi.Permission.Hidden && pi.Sid != string(p.params.SID) { + p.params.Logger.Debugw("skipping hidden participant update", "otherParticipant", pi.Identity) + isValid = false + } if isValid { p.updateCache.Add(pID, participantUpdateInfo{version: pi.Version, state: pi.State, updatedAt: time.Now()}) validUpdates = append(validUpdates, pi) @@ -179,7 +193,9 @@ func (p *ParticipantImpl) SendRefreshToken(token string) error { }) } -func (p *ParticipantImpl) SendReconnectResponse(reconnectResponse *livekit.ReconnectResponse) error { +func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error { + p.TransportManager.HandleClientReconnect(reconnectReason) + if !p.params.ClientInfo.CanHandleReconnectResponse() { return nil } @@ -265,12 +281,16 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { sink := p.getResponseSink() if sink == nil { - p.params.Logger.Infow("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message)) + p.params.Logger.Debugw("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message)) return nil } err := sink.WriteMessage(msg) - if err != nil { + if errors.Is(err, psrpc.Canceled) { + p.params.Logger.Debugw("could not send message to participant", + "error", err, "messageType", fmt.Sprintf("%T", msg.Message)) + return nil + } else if err != nil { p.params.Logger.Warnw("could not send message to participant", err, "messageType", fmt.Sprintf("%T", msg.Message)) return err @@ -279,10 +299,10 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { } // closes signal connection to notify client to resume/reconnect -func (p *ParticipantImpl) CloseSignalConnection() { +func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { sink := p.getResponseSink() if sink != nil { - p.params.Logger.Infow("closing signal connection") + p.params.Logger.Infow("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) sink.Close() p.SetResponseSink(nil) } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index f319010e4..c0fce9d25 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -27,14 +41,19 @@ import ( const ( DefaultEmptyTimeout = 5 * 60 // 5m - DefaultRoomDepartureGrace = 20 - AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal + AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal invAudioLevelQuantization = 1.0 / AudioLevelQuantization subscriberUpdateInterval = 3 * time.Second dataForwardLoadBalanceThreshold = 20 ) +var ( + // var to allow unit test override + RoomDepartureGrace uint32 = 20 + roomUpdateInterval = 5 * time.Second // frequency to update room participant counts +) + type broadcastOptions struct { skipSource bool immediate bool @@ -43,9 +62,10 @@ type broadcastOptions struct { type Room struct { lock sync.RWMutex - protoRoom *livekit.Room - internal *livekit.RoomInternal - Logger logger.Logger + protoRoom *livekit.Room + internal *livekit.RoomInternal + protoProxy *utils.ProtoProxy[*livekit.Room] + Logger logger.Logger config WebRTCConfig audioConfig *config.AudioConfig @@ -71,8 +91,10 @@ type Room struct { leftAt atomic.Int64 closed chan struct{} + trailer []byte + onParticipantChanged func(p types.LocalParticipant) - onMetadataUpdate func(metadata string) + onRoomUpdated func() onClose func() } @@ -105,7 +127,9 @@ func NewRoom( bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), closed: make(chan struct{}), + trailer: []byte(utils.RandomSecret()), } + r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) if r.protoRoom.EmptyTimeout == 0 { r.protoRoom.EmptyTimeout = DefaultEmptyTimeout } @@ -115,16 +139,13 @@ func NewRoom( go r.audioUpdateWorker() go r.connectionQualityWorker() - go r.subscriberBroadcastWorker() + go r.changeUpdateWorker() return r } func (r *Room) ToProto() *livekit.Room { - r.lock.RLock() - defer r.lock.RUnlock() - - return proto.Clone(r.protoRoom).(*livekit.Room) + return r.protoProxy.Get() } func (r *Room) Name() livekit.RoomName { @@ -135,6 +156,15 @@ func (r *Room) ID() livekit.RoomID { return livekit.RoomID(r.protoRoom.Sid) } +func (r *Room) Trailer() []byte { + r.lock.RLock() + defer r.lock.RUnlock() + + trailer := make([]byte, len(r.trailer)) + copy(trailer, r.trailer) + return trailer +} + func (r *Room) GetParticipant(identity livekit.ParticipantIdentity) types.LocalParticipant { r.lock.RLock() defer r.lock.RUnlock() @@ -240,14 +270,13 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me } if r.protoRoom.MaxParticipants > 0 && !participant.IsRecorder() { - participantCount := 0 + numParticipants := uint32(0) for _, p := range r.participants { if !p.IsRecorder() { - participantCount++ + numParticipants++ } } - - if participantCount >= int(r.protoRoom.MaxParticipants) { + if numParticipants >= r.protoRoom.MaxParticipants { return ErrMaxParticipantsExceeded } } @@ -255,9 +284,6 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if r.FirstJoinedAt() == 0 { r.joinedAt.Store(time.Now().Unix()) } - if !participant.Hidden() { - r.protoRoom.NumParticipants++ - } // it's important to set this before connection, we don't want to miss out on any published tracks participant.OnTrackPublished(r.onTrackPublished) @@ -280,10 +306,15 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me // start the workers once connectivity is established p.Start() - r.telemetry.ParticipantActive(context.Background(), r.ToProto(), p.ToProto(), &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), - ConnectionType: string(p.GetICEConnectionType()), - }) + r.telemetry.ParticipantActive(context.Background(), + r.ToProto(), + p.ToProto(), + &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), + ConnectionType: string(p.GetICEConnectionType()), + }, + false, + ) } else if state == livekit.ParticipantInfo_DISCONNECTED { // remove participant from room go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) @@ -336,7 +367,9 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if participant.IsRecorder() && !r.protoRoom.ActiveRecording { r.protoRoom.ActiveRecording = true - r.sendRoomUpdateLocked() + r.protoProxy.MarkDirty(true) + } else { + r.protoProxy.MarkDirty(false) } r.participants[participant.Identity()] = participant @@ -398,28 +431,26 @@ func (r *Room) GetParticipantRequestSource(identity livekit.ParticipantIdentity) func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing.MessageSource, responseSink routing.MessageSink, iceServers []*livekit.ICEServer, reason livekit.ReconnectReason) error { r.ReplaceParticipantRequestSource(p.Identity(), requestSource) // close previous sink, and link to new one - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonResume) p.SetResponseSink(responseSink) p.SetSignalSourceValid(true) - if err := p.SendReconnectResponse(&livekit.ReconnectResponse{ + if err := p.HandleReconnectAndSendResponse(reason, &livekit.ReconnectResponse{ IceServers: iceServers, ClientConfiguration: p.GetClientConfiguration(), }); err != nil { return err } - updates := ToProtoParticipants(r.GetParticipants()) + // include the local participant's info as well, since metadata could have been changed + updates := r.getOtherParticipantInfo("") if err := p.SendParticipantUpdate(updates); err != nil { return err } - r.lock.RLock() - p.SendRoomUpdate(r.protoRoom) - r.lock.RUnlock() - - p.ICERestart(nil, reason) + _ = p.SendRoomUpdate(r.ToProto()) + p.ICERestart(nil) return nil } @@ -441,6 +472,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } } + immediateChange := false if (p != nil && p.IsRecorder()) || r.protoRoom.ActiveRecording { activeRecording := false for _, op := range r.participants { @@ -452,10 +484,12 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek if r.protoRoom.ActiveRecording != activeRecording { r.protoRoom.ActiveRecording = activeRecording - r.sendRoomUpdateLocked() + immediateChange = true + } } r.lock.Unlock() + r.protoProxy.MarkDirty(immediateChange) if !ok { return @@ -464,6 +498,11 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek // send broadcast only if it's not already closed sendUpdates := !p.IsDisconnected() + // remove all published tracks + for _, t := range p.GetPublishedTracks() { + r.trackManager.RemoveTrack(t) + } + p.OnTrackUpdated(nil) p.OnTrackPublished(nil) p.OnTrackUnpublished(nil) @@ -474,13 +513,9 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek // close participant as well r.Logger.Debugw("closing participant for removal", "pID", p.ID(), "participant", p.Identity()) - _ = p.Close(true, reason) + _ = p.Close(true, reason, false) - r.lock.RLock() - if len(r.participants) == 0 { - r.leftAt.Store(time.Now().Unix()) - } - r.lock.RUnlock() + r.leftAt.Store(time.Now().Unix()) if sendUpdates { if r.onParticipantChanged != nil { @@ -517,11 +552,51 @@ func (r *Room) UpdateSubscriptions( } func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.SyncState) error { + pLogger := participant.GetLogger() + pLogger.Infow("setting sync state", "state", state) + + shouldReconnect := false + pubTracks := state.GetPublishTracks() + existingPubTracks := participant.GetPublishedTracks() + for _, pubTrack := range pubTracks { + // client may not have sent TrackInfo for each published track + ti := pubTrack.Track + if ti == nil { + pLogger.Warnw("TrackInfo not sent during resume", nil) + shouldReconnect = true + break + } + + found := false + for _, existingPubTrack := range existingPubTracks { + if existingPubTrack.ID() == livekit.TrackID(ti.Sid) { + found = true + break + } + } + if !found { + pLogger.Warnw("unknown track during resume", nil, "trackID", ti.Sid) + shouldReconnect = true + break + } + } + if shouldReconnect { + pLogger.Warnw("unable to resume due to missing published tracks, starting full reconnect", nil) + participant.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) + return nil + } + + r.UpdateSubscriptions( + participant, + livekit.StringsAsTrackIDs(state.Subscription.TrackSids), + state.Subscription.ParticipantTracks, + state.Subscription.Subscribe, + ) return nil } func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { - if err := participant.UpdateSubscriptionPermission(subscriptionPermission, nil, r.GetParticipant, r.GetParticipantByID); err != nil { + if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion{}, r.GetParticipant, r.GetParticipantByID); err != nil { return err } for _, track := range participant.GetPublishedTracks() { @@ -530,20 +605,6 @@ func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, return nil } -func (r *Room) RemoveDisallowedSubscriptions(sub types.LocalParticipant, disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID) { - for trackID, publisherID := range disallowedSubscriptions { - pub := r.GetParticipantByID(publisherID) - if pub == nil { - continue - } - - track := pub.GetPublishedTrack(trackID) - if track != nil { - track.RemoveSubscriber(sub.ID(), false) - } - } -} - func (r *Room) UpdateVideoLayers(participant types.Participant, updateVideoLayers *livekit.UpdateVideoLayers) error { return participant.UpdateVideoLayers(updateVideoLayers) } @@ -597,16 +658,15 @@ func (r *Room) CloseIfEmpty() { } } - timeout := r.protoRoom.EmptyTimeout + var timeout uint32 var elapsed int64 - if r.FirstJoinedAt() > 0 { - // exit 20s after + if r.FirstJoinedAt() > 0 && r.LastLeftAt() > 0 { elapsed = time.Now().Unix() - r.LastLeftAt() - if timeout > DefaultRoomDepartureGrace { - timeout = DefaultRoomDepartureGrace - } + // need to give time in case participant is reconnecting + timeout = RoomDepartureGrace } else { elapsed = time.Now().Unix() - r.protoRoom.CreationTime + timeout = r.protoRoom.EmptyTimeout } r.lock.Unlock() @@ -628,8 +688,9 @@ func (r *Room) Close() { r.lock.Unlock() r.Logger.Infow("closing room") for _, p := range r.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomClose) + _ = p.Close(true, types.ParticipantCloseReasonRoomClose, false) } + r.protoProxy.Stop() if r.onClose != nil { r.onClose() } @@ -657,38 +718,43 @@ func (r *Room) SetMetadata(metadata string) { r.lock.Lock() r.protoRoom.Metadata = metadata r.lock.Unlock() + r.protoProxy.MarkDirty(true) +} - r.lock.RLock() - r.sendRoomUpdateLocked() - r.lock.RUnlock() - - if r.onMetadataUpdate != nil { - r.onMetadataUpdate(metadata) +func (r *Room) UpdateParticipantMetadata(participant types.LocalParticipant, name string, metadata string) { + if metadata != "" { + participant.SetMetadata(metadata) + } + if name != "" { + participant.SetName(name) } } -func (r *Room) sendRoomUpdateLocked() { +func (r *Room) sendRoomUpdate() { + roomInfo := r.ToProto() // Send update to participants - for _, p := range r.participants { - if !p.IsReady() { + for _, p := range r.GetParticipants() { + // new participants receive the update as part of JoinResponse + // skip inactive participants + if p.State() != livekit.ParticipantInfo_ACTIVE { continue } - err := p.SendRoomUpdate(r.protoRoom) + err := p.SendRoomUpdate(roomInfo) if err != nil { r.Logger.Warnw("failed to send room update", err, "participant", p.Identity()) } } } -func (r *Room) OnMetadataUpdate(f func(metadata string)) { - r.onMetadataUpdate = f +func (r *Room) OnRoomUpdated(f func()) { + r.onRoomUpdated = f } func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { switch scenario := simulateScenario.Scenario.(type) { case *livekit.SimulateScenario_SpeakerUpdate: - r.Logger.Infow("simulating speaker update", "participant", participant.Identity()) + r.Logger.Infow("simulating speaker update", "participant", participant.Identity(), "duration", scenario.SpeakerUpdate) go func() { <-time.After(time.Duration(scenario.SpeakerUpdate) * time.Second) r.sendSpeakerChanges([]*livekit.SpeakerInfo{{ @@ -705,31 +771,49 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen case *livekit.SimulateScenario_Migration: r.Logger.Infow("simulating migration", "participant", participant.Identity()) // drop participant without necessarily cleaning up - if err := participant.Close(false, types.ParticipantCloseReasonSimulateMigration); err != nil { + if err := participant.Close(false, types.ParticipantCloseReasonSimulateMigration, true); err != nil { return err } case *livekit.SimulateScenario_NodeFailure: r.Logger.Infow("simulating node failure", "participant", participant.Identity()) // drop participant without necessarily cleaning up - if err := participant.Close(false, types.ParticipantCloseReasonSimulateNodeFailure); err != nil { + if err := participant.Close(false, types.ParticipantCloseReasonSimulateNodeFailure, true); err != nil { return err } case *livekit.SimulateScenario_ServerLeave: r.Logger.Infow("simulating server leave", "participant", participant.Identity()) - if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave); err != nil { + if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave, false); err != nil { return err } - case *livekit.SimulateScenario_SwitchCandidateProtocol: r.Logger.Infow("simulating switch candidate protocol", "participant", participant.Identity()) participant.ICERestart(&livekit.ICEConfig{ PreferenceSubscriber: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), PreferencePublisher: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), - }, livekit.ReconnectReason_RR_SWITCH_CANDIDATE) + }) + case *livekit.SimulateScenario_SubscriberBandwidth: + if scenario.SubscriberBandwidth > 0 { + r.Logger.Infow("simulating subscriber bandwidth start", "participant", participant.Identity(), "bandwidth", scenario.SubscriberBandwidth) + } else { + r.Logger.Infow("simulating subscriber bandwidth end", "participant", participant.Identity()) + } + participant.SetSubscriberChannelCapacity(scenario.SubscriberBandwidth) } return nil } +func (r *Room) getOtherParticipantInfo(identity livekit.ParticipantIdentity) []*livekit.ParticipantInfo { + participants := r.GetParticipants() + pi := make([]*livekit.ParticipantInfo, 0, len(participants)) + for _, p := range participants { + if !p.Hidden() && p.Identity() != identity { + pi = append(pi, p.ToProto()) + } + } + + return pi +} + // checks if participant should be autosubscribed to new tracks, assumes lock is already acquired func (r *Room) autoSubscribe(participant types.LocalParticipant) bool { opts := r.participantOpts[participant.Identity()] @@ -750,19 +834,20 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS } return &livekit.JoinResponse{ - Room: r.protoRoom, + Room: r.ToProto(), Participant: participant.ToProto(), OtherParticipants: otherParticipants, - ServerVersion: r.serverInfo.Version, - ServerRegion: r.serverInfo.Region, IceServers: iceServers, // indicates both server and client support subscriber as primary SubscriberPrimary: participant.SubscriberAsPrimary(), ClientConfiguration: participant.GetClientConfiguration(), // sane defaults for ping interval & timeout - PingInterval: 10, - PingTimeout: 20, - ServerInfo: r.serverInfo, + PingInterval: 10, + PingTimeout: 20, + ServerInfo: r.serverInfo, + ServerVersion: r.serverInfo.Version, + ServerRegion: r.serverInfo.Region, + SifTrailer: r.trailer, } } @@ -838,6 +923,7 @@ func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTra } func (r *Room) onParticipantUpdate(p types.LocalParticipant) { + r.protoProxy.MarkDirty(false) // immediately notify when permissions or metadata changed r.broadcastParticipantState(p, broadcastOptions{immediate: true}) if r.onParticipantChanged != nil { @@ -993,15 +1079,39 @@ func (r *Room) pushAndDequeueUpdates(pi *livekit.ParticipantInfo, isImmediate bo return updates } -func (r *Room) subscriberBroadcastWorker() { - ticker := time.NewTicker(subscriberUpdateInterval) - defer ticker.Stop() +func (r *Room) updateProto() *livekit.Room { + r.lock.RLock() + room := proto.Clone(r.protoRoom).(*livekit.Room) + r.lock.RUnlock() + + room.NumPublishers = 0 + room.NumParticipants = 0 + for _, p := range r.GetParticipants() { + if !p.IsRecorder() { + room.NumParticipants++ + } + if p.IsPublisher() { + room.NumPublishers++ + } + } + + return room +} + +func (r *Room) changeUpdateWorker() { + subTicker := time.NewTicker(subscriberUpdateInterval) + defer subTicker.Stop() for !r.IsClosed() { select { case <-r.closed: return - case <-ticker.C: + case <-r.protoProxy.Updated(): + if r.onRoomUpdated != nil { + r.onRoomUpdated() + } + r.sendRoomUpdate() + case <-subTicker.C: r.batchedUpdatesMu.Lock() updatesMap := r.batchedUpdates r.batchedUpdates = make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo) @@ -1162,11 +1272,11 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp var dpData []byte participants := r.GetLocalParticipants() - cap := len(dest) - if cap == 0 { - cap = len(participants) + capacity := len(dest) + if capacity == 0 { + capacity = len(participants) } - destParticpants := make([]types.LocalParticipant, 0, cap) + destParticipants := make([]types.LocalParticipant, 0, capacity) for _, op := range participants { if op.State() != livekit.ParticipantInfo_ACTIVE { @@ -1195,10 +1305,10 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp return } } - destParticpants = append(destParticpants, op) + destParticipants = append(destParticipants, op) } - utils.ParallelExec(destParticpants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { + utils.ParallelExec(destParticipants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { err := op.SendDataPacket(dp, dpData) if err != nil && !errors.Is(err, io.ErrClosedPipe) { op.GetLogger().Infow("send data packet error", "error", err) diff --git a/pkg/rtc/room_egress.go b/pkg/rtc/room_egress.go index 989270713..d632a007b 100644 --- a/pkg/rtc/room_egress.go +++ b/pkg/rtc/room_egress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 2452eb6cc..0fb46b0af 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -14,7 +28,6 @@ import ( "github.com/livekit/protocol/webhook" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" "github.com/livekit/livekit-server/pkg/sfu/audio" @@ -30,9 +43,12 @@ const ( ) func init() { - serverlogger.InitFromConfig(config.LoggingConfig{ + config.InitLoggerFromConfig(config.LoggingConfig{ Config: logger.Config{Level: "debug"}, }) + // allow immediate closure in testing + RoomDepartureGrace = 1 + roomUpdateInterval = defaultDelay } var iceServersForRoom = []*livekit.ICEServer{{Urls: []string{"stun:stun.l.google.com:19302"}}} @@ -59,11 +75,11 @@ func TestJoinedState(t *testing.T) { require.LessOrEqual(t, s, rm.LastLeftAt()) }) - t.Run("LastLeftAt should not be set when there are still participants in the room", func(t *testing.T) { + t.Run("LastLeftAt should be set when there are still participants in the room", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) p0 := rm.GetParticipants()[0] rm.RemoveParticipant(p0.Identity(), p0.ID(), types.ParticipantCloseReasonClientRequestLeave) - require.EqualValues(t, 0, rm.LastLeftAt()) + require.Greater(t, rm.LastLeftAt(), int64(0)) }) } @@ -138,7 +154,9 @@ func TestRoomJoin(t *testing.T) { t.Run("cannot exceed max participants", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) + rm.lock.Lock() rm.protoRoom.MaxParticipants = 1 + rm.lock.Unlock() p := newMockParticipant("second", types.ProtocolVersion(0), false, false) err := rm.Join(p, nil, nil, iceServersForRoom) @@ -345,11 +363,13 @@ func TestRoomClosure(t *testing.T) { isClosed = true }) p := rm.GetParticipants()[0] + rm.lock.Lock() // allows immediate close after rm.protoRoom.EmptyTimeout = 0 + rm.lock.Unlock() rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave) - time.Sleep(defaultDelay) + time.Sleep(time.Duration(RoomDepartureGrace)*time.Second + defaultDelay) rm.CloseIfEmpty() require.Len(t, rm.GetParticipants(), 0) @@ -375,7 +395,9 @@ func TestRoomClosure(t *testing.T) { rm.OnClose(func() { isClosed = true }) + rm.lock.Lock() rm.protoRoom.EmptyTimeout = 1 + rm.lock.Unlock() time.Sleep(1010 * time.Millisecond) rm.CloseIfEmpty() @@ -643,7 +665,7 @@ func TestHiddenParticipants(t *testing.T) { require.Len(t, res.OtherParticipants, 2) require.Len(t, rm.GetParticipants(), 4) require.NotEmpty(t, res.IceServers) - require.Equal(t, "testregion", res.ServerRegion) + require.Equal(t, "testregion", res.ServerInfo.Region) }) t.Run("hidden participant subscribes to tracks", func(t *testing.T) { @@ -663,15 +685,35 @@ func TestHiddenParticipants(t *testing.T) { } func TestRoomUpdate(t *testing.T) { + t.Run("updates are sent when participant joined", func(t *testing.T) { + rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) + defer rm.Close() + + p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) + require.Equal(t, 0, p1.SendRoomUpdateCallCount()) + + p2 := newMockParticipant("p2", types.CurrentProtocol, false, false) + require.NoError(t, rm.Join(p2, nil, nil, iceServersForRoom)) + + // p1 should have received an update + time.Sleep(2 * defaultDelay) + require.LessOrEqual(t, 1, p1.SendRoomUpdateCallCount()) + require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(p1.SendRoomUpdateCallCount()-1).NumParticipants) + }) + t.Run("participants should receive metadata update", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) defer rm.Close() rm.SetMetadata("test metadata...") + // callbacks are updated from goroutine + time.Sleep(2 * defaultDelay) + for _, op := range rm.GetParticipants() { fp := op.(*typesfakes.FakeLocalParticipant) - require.Equal(t, 1, fp.SendRoomUpdateCallCount()) + // room updates are now sent for both participant joining and room metadata + require.GreaterOrEqual(t, fp.SendRoomUpdateCallCount(), 1) } }) } @@ -699,7 +741,7 @@ func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room { NodeId: "testnode", Region: "testregion", }, - telemetry.NewTelemetryService(webhook.NewNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}), + telemetry.NewTelemetryService(webhook.NewDefaultNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}), nil, ) for i := 0; i < opts.num+opts.numHidden; i++ { diff --git a/pkg/rtc/rtc_unix.go b/pkg/rtc/rtc_unix.go deleted file mode 100644 index 3b357a21f..000000000 --- a/pkg/rtc/rtc_unix.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build !windows -// +build !windows - -package rtc - -import ( - "net" - "syscall" - - "github.com/livekit/protocol/logger" -) - -func checkUDPReadBuffer() { - val, err := getUDPReadBuffer() - if err == nil { - if val < minUDPBufferSize { - logger.Warnw("UDP receive buffer is too small for a production set-up", nil, - "current", val, - "suggested", minUDPBufferSize) - } else { - logger.Debugw("UDP receive buffer size", "current", val) - } - } -} - -func getUDPReadBuffer() (int, error) { - conn, err := net.ListenUDP("udp4", nil) - if err != nil { - return 0, err - } - defer func() { _ = conn.Close() }() - _ = conn.SetReadBuffer(defaultUDPBufferSize) - fd, err := conn.File() - if err != nil { - return 0, nil - } - defer func() { _ = fd.Close() }() - - return syscall.GetsockoptInt(int(fd.Fd()), syscall.SOL_SOCKET, syscall.SO_RCVBUF) -} diff --git a/pkg/rtc/rtc_windows.go b/pkg/rtc/rtc_windows.go deleted file mode 100644 index beeab22cf..000000000 --- a/pkg/rtc/rtc_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build windows -// +build windows - -package rtc - -func checkUDPReadBuffer() { -} diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index dc49ade12..3c90209b6 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -10,7 +24,7 @@ import ( func HandleParticipantSignal(room types.Room, participant types.LocalParticipant, req *livekit.SignalRequest, pLogger logger.Logger) error { participant.UpdateLastSeenSignal() - switch msg := req.Message.(type) { + switch msg := req.GetMessage().(type) { case *livekit.SignalRequest_Offer: participant.HandleOffer(FromProtoSessionDescription(msg.Offer)) case *livekit.SignalRequest_Answer: @@ -73,6 +87,11 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant if msg.PingReq.Rtt > 0 { participant.UpdateSignalingRTT(uint32(msg.PingReq.Rtt)) } + + case *livekit.SignalRequest_UpdateMetadata: + if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { + room.UpdateParticipantMetadata(participant, msg.UpdateMetadata.Name, msg.UpdateMetadata.Metadata) + } } return nil } diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 3a08c731f..5a83e0a1f 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -40,7 +54,7 @@ type SubscribedTrack struct { needsNegotiation atomic.Bool bindLock sync.Mutex - onBindCallbacks []func() + onBindCallbacks []func(error) onClose atomic.Value // func(bool) bound atomic.Bool @@ -61,7 +75,7 @@ func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack { return s } -func (t *SubscribedTrack) AddOnBind(f func()) { +func (t *SubscribedTrack) AddOnBind(f func(error)) { t.bindLock.Lock() bound := t.bound.Load() if !bound { @@ -71,19 +85,21 @@ func (t *SubscribedTrack) AddOnBind(f func()) { if bound { // fire immediately, do not need to persist since bind is a one time event - go f() + go f(nil) } } // for DownTrack callback to notify us that it's bound -func (t *SubscribedTrack) Bound() { +func (t *SubscribedTrack) Bound(err error) { t.bindLock.Lock() - t.bound.Store(true) + if err == nil { + t.bound.Store(true) + } callbacks := t.onBindCallbacks t.onBindCallbacks = nil t.bindLock.Unlock() - if t.MediaTrack().Kind() == livekit.TrackType_VIDEO { + if err == nil && t.MediaTrack().Kind() == livekit.TrackType_VIDEO { // When AdaptiveStream is enabled, default the subscriber to LOW quality stream // we would want LOW instead of OFF for a couple of reasons // 1. when a subscriber unsubscribes from a track, we would forget their previously defined settings @@ -107,7 +123,7 @@ func (t *SubscribedTrack) Bound() { } for _, cb := range callbacks { - go cb() + go cb(err) } } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 9e880aeb3..1ed7d3f8c 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -40,6 +40,11 @@ var ( // amount of time to try otherwise before flagging subscription as failed subscriptionTimeout = iceFailedTimeout trackRemoveGracePeriod = time.Second + maxUnsubscribeWait = time.Second +) + +const ( + trackIDForReconcileSubscriptions = livekit.TrackID("subscriptions_reconcile") ) type SubscriptionManagerParams struct { @@ -48,34 +53,37 @@ type SubscriptionManagerParams struct { TrackResolver types.MediaTrackResolver OnTrackSubscribed func(subTrack types.SubscribedTrack) OnTrackUnsubscribed func(subTrack types.SubscribedTrack) - OnSubscriptionError func(trackID livekit.TrackID) + OnSubscriptionError func(trackID livekit.TrackID, fatal bool, err error) Telemetry telemetry.TelemetryService + + SubscriptionLimitVideo, SubscriptionLimitAudio int32 } // SubscriptionManager manages a participant's subscriptions type SubscriptionManager struct { - params SubscriptionManagerParams - lock sync.RWMutex - subscriptions map[livekit.TrackID]*trackSubscription - subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} - // keeps track of tracks that are already queued for reconcile to avoid duplicating reconcile requests - pendingReconcile map[livekit.TrackID]struct{} - reconcileCh chan livekit.TrackID - closeCh chan struct{} - doneCh chan struct{} + params SubscriptionManagerParams + lock sync.RWMutex + subscriptions map[livekit.TrackID]*trackSubscription + pendingUnsubscribes atomic.Int32 + + subscribedVideoCount, subscribedAudioCount atomic.Int32 + + subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} + reconcileCh chan livekit.TrackID + closeCh chan struct{} + doneCh chan struct{} onSubscribeStatusChanged func(publisherID livekit.ParticipantID, subscribed bool) } func NewSubscriptionManager(params SubscriptionManagerParams) *SubscriptionManager { m := &SubscriptionManager{ - params: params, - subscriptions: make(map[livekit.TrackID]*trackSubscription), - subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), - pendingReconcile: make(map[livekit.TrackID]struct{}), - reconcileCh: make(chan livekit.TrackID, 50), - closeCh: make(chan struct{}), - doneCh: make(chan struct{}), + params: params, + subscriptions: make(map[livekit.TrackID]*trackSubscription), + subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), + reconcileCh: make(chan livekit.TrackID, 50), + closeCh: make(chan struct{}), + doneCh: make(chan struct{}), } go m.reconcileWorker() @@ -96,6 +104,7 @@ func (m *SubscriptionManager) Close(willBeResumed bool) { subTracks := m.GetSubscribedTracks() downTracksToClose := make([]*sfu.DownTrack, 0, len(subTracks)) for _, st := range subTracks { + m.setDesired(st.ID(), false) dt := st.DownTrack() // nil check exists primarily for tests if dt != nil { @@ -103,8 +112,15 @@ func (m *SubscriptionManager) Close(willBeResumed bool) { } } - for _, dt := range downTracksToClose { - dt.CloseWithFlush(!willBeResumed) + if willBeResumed { + for _, dt := range downTracksToClose { + dt.CloseWithFlush(false) + } + } else { + // flush blocks, so execute in parallel + for _, dt := range downTracksToClose { + go dt.CloseWithFlush(true) + } } } @@ -118,17 +134,19 @@ func (m *SubscriptionManager) isClosed() bool { } func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { - m.lock.Lock() - sub, ok := m.subscriptions[trackID] - if !ok { + sub, desireChanged := m.setDesired(trackID, true) + if sub == nil { sLogger := m.params.Logger.WithValues( "trackID", trackID, ) sub = newTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + + m.lock.Lock() m.subscriptions[trackID] = sub + m.lock.Unlock() + + sub, desireChanged = m.setDesired(trackID, true) } - desireChanged := sub.setDesired(true) - m.lock.Unlock() if desireChanged { sub.logger.Infow("subscribing to track") } @@ -138,17 +156,13 @@ func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { } func (m *SubscriptionManager) UnsubscribeFromTrack(trackID livekit.TrackID) { - m.lock.Lock() - sub, ok := m.subscriptions[trackID] - m.lock.Unlock() - if !ok { + sub, desireChanged := m.setDesired(trackID, false) + if sub == nil || !desireChanged { return } - if sub.setDesired(false) { - sub.logger.Infow("unsubscribing from track") - m.queueReconcile(trackID) - } + sub.logger.Infow("unsubscribing from track") + m.queueReconcile(trackID) } func (m *SubscriptionManager) GetSubscribedTracks() []types.SubscribedTrack { @@ -240,6 +254,18 @@ func (m *SubscriptionManager) WaitUntilSubscribed(timeout time.Duration) error { return context.DeadlineExceeded } +func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) (*trackSubscription, bool) { + m.lock.RLock() + defer m.lock.RUnlock() + + sub, ok := m.subscriptions[trackID] + if !ok { + return nil, false + } + + return sub, sub.setDesired(desired) +} + func (m *SubscriptionManager) canReconcile() bool { p := m.params.Participant if m.isClosed() || p.IsClosed() || p.IsDisconnected() { @@ -252,7 +278,7 @@ func (m *SubscriptionManager) reconcileSubscriptions() { var needsToReconcile []*trackSubscription m.lock.RLock() for _, sub := range m.subscriptions { - if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsBind() { + if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsBind() || sub.needsCleanup() { needsToReconcile = append(needsToReconcile, sub) } } @@ -268,6 +294,15 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { return } if s.needsSubscribe() { + if m.pendingUnsubscribes.Load() != 0 && s.durationSinceStart() < maxUnsubscribeWait { + // enqueue this in a bit, after pending unsubscribes are complete + go func() { + time.Sleep(time.Duration(sfu.RTPBlankFramesCloseSeconds * float32(time.Second))) + m.queueReconcile(s.trackID) + }() + return + } + numAttempts := s.getNumAttempts() if numAttempts == 0 { m.params.Telemetry.TrackSubscribeRequested( @@ -282,26 +317,28 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { s.recordAttempt(false) switch err { - case ErrNoTrackPermission, ErrNoSubscribePermission, ErrNoReceiver, ErrNotOpen, ErrTrackNotAttached: + case ErrNoTrackPermission, ErrNoSubscribePermission, ErrNoReceiver, ErrNotOpen, ErrTrackNotAttached, ErrSubscriptionLimitExceeded: // these are errors that are outside of our control, so we'll keep trying // - ErrNoTrackPermission: publisher did not grant subscriber permission, may change any moment // - ErrNoSubscribePermission: participant was not granted canSubscribe, may change any moment // - ErrNoReceiver: Track is in the process of closing (another local track published to the same instance) // - ErrTrackNotAttached: Remote Track that is not attached, but may be attached later // - ErrNotOpen: Track is closing or already closed + // - ErrSubscriptionLimitExceeded: the participant have reached the limit of subscriptions, wait for the other subscription to be unsubscribed // We'll still log an event to reflect this in telemetry since it's been too long if s.durationSinceStart() > subscriptionTimeout { s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) } case ErrTrackNotFound: // source track was never published or closed - // if after timeout, we'd unsubscribe from it. + // if after timeout we'd unsubscribe from it. // this is the *only* case we'd change desired state if s.durationSinceStart() > notFoundTimeout { s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) s.logger.Infow("unsubscribing from track after notFoundTimeout", "error", err) s.setDesired(false) m.queueReconcile(s.trackID) + m.params.OnSubscriptionError(s.trackID, false, err) } default: // all other errors @@ -310,7 +347,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { "attempt", numAttempts, ) s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, false) - m.params.OnSubscriptionError(s.trackID) + m.params.OnSubscriptionError(s.trackID, true, err) } else { s.logger.Debugw("failed to subscribe, retrying", "error", err, @@ -332,6 +369,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // successfully unsubscribed, remove from map m.lock.Lock() if !s.isDesired() { + s.logger.Debugw("unsubscribe removing subscription") delete(m.subscriptions, s.trackID) } m.lock.Unlock() @@ -346,20 +384,22 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { if s.durationSinceStart() > subscriptionTimeout { s.logger.Errorw("track not bound after timeout", nil) s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), ErrTrackNotBound, false) - m.params.OnSubscriptionError(s.trackID) + m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) } } + + if s.needsCleanup() { + m.lock.Lock() + if !s.isDesired() { + s.logger.Debugw("cleanup removing subscription") + delete(m.subscriptions, s.trackID) + } + m.lock.Unlock() + } } // trigger an immediate reconciliation, when trackID is empty, will reconcile all subscriptions func (m *SubscriptionManager) queueReconcile(trackID livekit.TrackID) { - m.lock.Lock() - if _, ok := m.pendingReconcile[trackID]; ok { - // already reconciled - m.lock.Unlock() - return - } - m.lock.Unlock() select { case m.reconcileCh <- trackID: default: @@ -381,7 +421,6 @@ func (m *SubscriptionManager) reconcileWorker() { case trackID := <-m.reconcileCh: m.lock.Lock() s := m.subscriptions[trackID] - delete(m.pendingReconcile, trackID) m.lock.Unlock() if s != nil { m.reconcileSubscription(s) @@ -392,6 +431,21 @@ func (m *SubscriptionManager) reconcileWorker() { } } +func (m *SubscriptionManager) hasCapacityForSubscription(kind livekit.TrackType) bool { + switch kind { + case livekit.TrackType_VIDEO: + if m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Load() >= m.params.SubscriptionLimitVideo { + return false + } + + case livekit.TrackType_AUDIO: + if m.params.SubscriptionLimitAudio > 0 && m.subscribedAudioCount.Load() >= m.params.SubscriptionLimitAudio { + return false + } + } + return true +} + func (m *SubscriptionManager) subscribe(s *trackSubscription) error { s.logger.Debugw("executing subscribe") @@ -399,7 +453,12 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrNoSubscribePermission } - res := m.params.TrackResolver(m.params.Participant.Identity(), s.trackID) + if kind, ok := s.getKind(); ok && !m.hasCapacityForSubscription(kind) { + return ErrSubscriptionLimitExceeded + } + + trackID := s.trackID + res := m.params.TrackResolver(m.params.Participant.Identity(), trackID) s.logger.Debugw("resolved track", "result", res) if res.TrackChangedNotifier != nil && s.setChangedNotifier(res.TrackChangedNotifier) { @@ -407,18 +466,18 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { // we set the observer before checking for existence of track, so that we may get notified // when the track becomes available res.TrackChangedNotifier.AddObserver(string(m.params.Participant.ID()), func() { - m.queueReconcile(s.trackID) + m.queueReconcile(trackID) }) } if res.TrackRemovedNotifier != nil && s.setRemovedNotifier(res.TrackRemovedNotifier) { res.TrackRemovedNotifier.AddObserver(string(m.params.Participant.ID()), func() { // re-resolve the track in case the same track had been re-published - res := m.params.TrackResolver(m.params.Participant.Identity(), s.trackID) + res := m.params.TrackResolver(m.params.Participant.Identity(), trackID) if res.Track != nil { // do not unsubscribe, track is still available return } - s.handleSourceTrackRemoved() + m.handleSourceTrackRemoved(trackID) }) } @@ -426,18 +485,23 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { if track == nil { return ErrTrackNotFound } + s.trySetKind(track.Kind()) + if !m.hasCapacityForSubscription(track.Kind()) { + return ErrSubscriptionLimitExceeded + } + + s.setPublisher(res.PublisherIdentity, res.PublisherID) // since hasPermission defaults to true, we will want to send a message to the client the first time // that we discover permissions were denied permChanged := s.setHasPermission(res.HasPermission) if permChanged { - m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), s.trackID, res.HasPermission) + m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), trackID, res.HasPermission) } if !res.HasPermission { return ErrNoTrackPermission } - s.setPublisher(res.PublisherIdentity, res.PublisherID) subTrack, err := track.AddSubscriber(m.params.Participant) if err != nil && err != errAlreadySubscribed { // ignore already subscribed error @@ -447,12 +511,26 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { subTrack.OnClose(func(willBeResumed bool) { m.handleSubscribedTrackClose(s, willBeResumed) }) - subTrack.AddOnBind(func() { + subTrack.AddOnBind(func(err error) { + if err != nil { + s.logger.Infow("failed to bind track", "err", err) + s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) + m.UnsubscribeFromTrack(trackID) + m.params.OnSubscriptionError(trackID, false, err) + return + } s.setBound() s.maybeRecordSuccess(m.params.Telemetry, m.params.Participant.ID()) }) s.setSubscribedTrack(subTrack) + switch track.Kind() { + case livekit.TrackType_VIDEO: + m.subscribedVideoCount.Inc() + case livekit.TrackType_AUDIO: + m.subscribedAudioCount.Inc() + } + if subTrack.NeedsNegotiation() { m.params.Participant.Negotiate(false) } @@ -460,6 +538,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { go m.params.OnTrackSubscribed(subTrack) } + m.params.Logger.Debugw("subscribed to track", "trackID", trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) + // add mark the participant as someone we've subscribed to firstSubscribe := false publisherID := s.getPublisherID() @@ -471,7 +551,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { m.subscribedTo[publisherID] = pTracks firstSubscribe = true } - pTracks[s.trackID] = struct{}{} + pTracks[trackID] = struct{}{} m.lock.Unlock() if changedCB != nil && firstSubscribe { @@ -491,11 +571,25 @@ func (m *SubscriptionManager) unsubscribe(s *trackSubscription) error { track := subTrack.MediaTrack() pID := m.params.Participant.ID() - track.RemoveSubscriber(pID, false) + m.pendingUnsubscribes.Inc() + go func() { + defer m.pendingUnsubscribes.Dec() + track.RemoveSubscriber(pID, false) + }() return nil } +func (m *SubscriptionManager) handleSourceTrackRemoved(trackID livekit.TrackID) { + m.lock.Lock() + sub := m.subscriptions[trackID] + m.lock.Unlock() + + if sub != nil { + sub.handleSourceTrackRemoved() + } +} + // DownTrack closing is how the publisher signifies that the subscription is no longer fulfilled // this could be due to a few reasons: // - subscriber-initiated unsubscribe @@ -512,6 +606,14 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w } s.setSubscribedTrack(nil) + var relieveFromLimits bool + switch subTrack.MediaTrack().Kind() { + case livekit.TrackType_VIDEO: + relieveFromLimits = m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Dec() == m.params.SubscriptionLimitVideo-1 + case livekit.TrackType_AUDIO: + relieveFromLimits = m.params.SubscriptionLimitAudio > 0 && m.subscribedAudioCount.Dec() == m.params.SubscriptionLimitAudio-1 + } + // remove from subscribedTo publisherID := s.getPublisherID() lastSubscription := false @@ -581,7 +683,11 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w m.params.Participant.Negotiate(false) } - m.queueReconcile(s.trackID) + if relieveFromLimits { + m.queueReconcile(trackIDForReconcileSubscriptions) + } else { + m.queueReconcile(s.trackID) + } } // -------------------------------------------------------------------------------------- @@ -603,6 +709,7 @@ type trackSubscription struct { eventSent atomic.Bool numAttempts atomic.Int32 bound bool + kind atomic.Pointer[livekit.TrackType] // the later of when subscription was requested OR when the first failure was encountered OR when permission is granted // this timestamp determines when failures are reported @@ -705,6 +812,18 @@ func (s *trackSubscription) setSubscribedTrack(track types.SubscribedTrack) { } } +func (s *trackSubscription) trySetKind(kind livekit.TrackType) { + s.kind.CompareAndSwap(nil, &kind) +} + +func (s *trackSubscription) getKind() (livekit.TrackType, bool) { + kind := s.kind.Load() + if kind == nil { + return livekit.TrackType_AUDIO, false + } + return *kind, true +} + func (s *trackSubscription) getSubscribedTrack() types.SubscribedTrack { s.lock.RLock() defer s.lock.RUnlock() @@ -739,7 +858,6 @@ func (s *trackSubscription) setRemovedNotifier(notifier types.ChangeNotifier) bo func (s *trackSubscription) setRemovedNotifierLocked(notifier types.ChangeNotifier) bool { if s.removedNotifier == notifier { - return false } @@ -865,3 +983,9 @@ func (s *trackSubscription) needsBind() bool { defer s.lock.RUnlock() return s.desired && s.subscribedTrack != nil && !s.bound } + +func (s *trackSubscription) needsCleanup() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return !s.desired && s.subscribedTrack == nil +} diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index aa4441c1e..9ad6f20ea 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -39,7 +39,7 @@ func init() { } const ( - subSettleTimeout = 300 * time.Millisecond + subSettleTimeout = 600 * time.Millisecond subCheckInterval = 10 * time.Millisecond ) @@ -54,7 +54,7 @@ func TestSubscribe(t *testing.T) { sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) { subCount.Add(1) } - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } numParticipantSubscribed := atomic.Int32{} @@ -76,7 +76,10 @@ func TestSubscribe(t *testing.T) { require.NotNil(t, s.getSubscribedTrack()) require.Len(t, sm.GetSubscribedTracks(), 1) - require.Len(t, sm.GetSubscribedParticipants(), 1) + + require.Eventually(t, func() bool { + return len(sm.GetSubscribedParticipants()) == 1 + }, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item") require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent @@ -85,7 +88,6 @@ func TestSubscribe(t *testing.T) { // ensure bound setTestSubscribedTrackBound(t, s.getSubscribedTrack()) - require.Eventually(t, func() bool { return !s.needsBind() }, subSettleTimeout, subCheckInterval, "track was not bound") @@ -96,16 +98,22 @@ func TestSubscribe(t *testing.T) { time.Sleep(notFoundTimeout) require.False(t, failed.Load()) + resolver.SetPause(true) // ensure its resilience after being closed setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false) - require.True(t, s.needsSubscribe()) + require.Eventually(t, func() bool { + return s.needsSubscribe() + }, subSettleTimeout, subCheckInterval, "needs subscribe did not persist across track close") + resolver.SetPause(false) require.Eventually(t, func() bool { return s.isDesired() && !s.needsSubscribe() }, subSettleTimeout, subCheckInterval, "track was not resubscribed") // was subscribed twice, unsubscribed once (due to close) - require.Equal(t, int32(2), numParticipantSubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantSubscribed.Load() == 2 + }, subSettleTimeout, subCheckInterval, "participant subscribe status was not updated twice") require.Equal(t, int32(1), numParticipantUnsubscribed.Load()) }) @@ -115,7 +123,7 @@ func TestSubscribe(t *testing.T) { resolver := newTestResolver(false, true, "pub", "pubID") sm.params.TrackResolver = resolver.Resolve failed := atomic.Bool{} - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } @@ -156,7 +164,7 @@ func TestSubscribe(t *testing.T) { resolver := newTestResolver(true, true, "pub", "pubID") sm.params.TrackResolver = resolver.Resolve failed := atomic.Bool{} - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } @@ -209,9 +217,9 @@ func TestUnsubscribe(t *testing.T) { st.OnClose(func(willBeResumed bool) { sm.handleSubscribedTrackClose(s, willBeResumed) }) - res.Track.(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + res.Track.(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st, willBeResumed) - } + }) sm.lock.Lock() sm.subscriptions["track"] = s @@ -225,14 +233,23 @@ func TestUnsubscribe(t *testing.T) { require.False(t, s.isDesired()) require.Eventually(t, func() bool { - return !s.needsUnsubscribe() + if s.needsUnsubscribe() { + return false + } + if sm.pendingUnsubscribes.Load() != 0 { + return false + } + sm.lock.RLock() + subLen := len(sm.subscriptions) + sm.lock.RUnlock() + if subLen != 0 { + return false + } + return true }, subSettleTimeout, subCheckInterval, "Track was not unsubscribed") // no traces should be left require.Len(t, sm.GetSubscribedTracks(), 0) - sm.lock.RLock() - require.Len(t, sm.subscriptions, 0) - sm.lock.RUnlock() require.False(t, res.TrackChangedNotifier.HasObservers()) tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) @@ -269,14 +286,16 @@ func TestSubscribeStatusChanged(t *testing.T) { st2.OnClose(func(willBeResumed bool) { sm.handleSubscribedTrackClose(s2, willBeResumed) }) - st1.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + st1.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st1, willBeResumed) - } - st2.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + }) + st2.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st2, willBeResumed) - } + }) - require.Equal(t, int32(1), numParticipantSubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantSubscribed.Load() == 1 + }, subSettleTimeout, subCheckInterval, "should be subscribed to publisher") require.Equal(t, int32(0), numParticipantUnsubscribed.Load()) require.True(t, sm.IsSubscribedTo("pubID")) @@ -292,7 +311,9 @@ func TestSubscribeStatusChanged(t *testing.T) { require.Eventually(t, func() bool { return !s1.needsUnsubscribe() }, subSettleTimeout, subCheckInterval, "track1 should be unsubscribed") - require.Equal(t, int32(1), numParticipantUnsubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantUnsubscribed.Load() == 1 + }, subSettleTimeout, subCheckInterval, "should be subscribed to publisher") require.False(t, sm.IsSubscribedTo("pubID")) } @@ -319,14 +340,123 @@ func TestUpdateSettingsBeforeSubscription(t *testing.T) { }, subSettleTimeout, subCheckInterval, "Track should be subscribed") st := s.getSubscribedTrack().(*typesfakes.FakeSubscribedTrack) - require.Equal(t, 1, st.UpdateSubscriberSettingsCallCount()) + require.Eventually(t, func() bool { + return st.UpdateSubscriberSettingsCallCount() == 1 + }, subSettleTimeout, subCheckInterval, "UpdateSubscriberSettings should be called once") + applied := st.UpdateSubscriberSettingsArgsForCall(0) require.Equal(t, settings.Disabled, applied.Disabled) require.Equal(t, settings.Width, applied.Width) require.Equal(t, settings.Height, applied.Height) } +func TestSubscriptionLimits(t *testing.T) { + sm := newTestSubscriptionManagerWithParams(t, testSubscriptionParams{ + SubscriptionLimitAudio: 1, + SubscriptionLimitVideo: 1, + }) + defer sm.Close(false) + resolver := newTestResolver(true, true, "pub", "pubID") + sm.params.TrackResolver = resolver.Resolve + subCount := atomic.Int32{} + failed := atomic.Bool{} + sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) { + subCount.Add(1) + } + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { + failed.Store(true) + } + numParticipantSubscribed := atomic.Int32{} + numParticipantUnsubscribed := atomic.Int32{} + sm.OnSubscribeStatusChanged(func(pubID livekit.ParticipantID, subscribed bool) { + if subscribed { + numParticipantSubscribed.Add(1) + } else { + numParticipantUnsubscribed.Add(1) + } + }) + + sm.SubscribeToTrack("track") + s := sm.subscriptions["track"] + require.True(t, s.isDesired()) + require.Eventually(t, func() bool { + return subCount.Load() == 1 + }, subSettleTimeout, subCheckInterval, "track was not subscribed") + + require.NotNil(t, s.getSubscribedTrack()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + require.Eventually(t, func() bool { + return len(sm.GetSubscribedParticipants()) == 1 + }, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item") + require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) + + // ensure telemetry events are sent + tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) + require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount()) + + // ensure bound + setTestSubscribedTrackBound(t, s.getSubscribedTrack()) + require.Eventually(t, func() bool { + return !s.needsBind() + }, subSettleTimeout, subCheckInterval, "track was not bound") + + // telemetry event should have been sent + require.Equal(t, 1, tm.TrackSubscribedCallCount()) + + // reach subscription limit, subscribe pending + sm.SubscribeToTrack("track2") + s2 := sm.subscriptions["track2"] + time.Sleep(subscriptionTimeout * 2) + require.True(t, s2.needsSubscribe()) + require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) + require.Equal(t, 1, tm.TrackSubscribeFailedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + // unsubscribe track1, then track2 should be subscribed + sm.UnsubscribeFromTrack("track") + require.False(t, s.isDesired()) + require.True(t, s.needsUnsubscribe()) + // wait for unsubscribe to take effect + time.Sleep(reconcileInterval) + setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false) + require.Nil(t, s.getSubscribedTrack()) + + time.Sleep(reconcileInterval) + require.True(t, s2.isDesired()) + require.False(t, s2.needsSubscribe()) + require.EqualValues(t, 2, subCount.Load()) + require.NotNil(t, s2.getSubscribedTrack()) + require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + // ensure bound + setTestSubscribedTrackBound(t, s2.getSubscribedTrack()) + require.Eventually(t, func() bool { + return !s2.needsBind() + }, subSettleTimeout, subCheckInterval, "track was not bound") + + // subscribe to track1 again, which should pending + sm.SubscribeToTrack("track") + s = sm.subscriptions["track"] + require.True(t, s.isDesired()) + time.Sleep(subscriptionTimeout * 2) + require.True(t, s.needsSubscribe()) + require.Equal(t, 3, tm.TrackSubscribeRequestedCallCount()) + require.Equal(t, 2, tm.TrackSubscribeFailedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) +} + +type testSubscriptionParams struct { + SubscriptionLimitAudio int32 + SubscriptionLimitVideo int32 +} + func newTestSubscriptionManager(t *testing.T) *SubscriptionManager { + return newTestSubscriptionManagerWithParams(t, testSubscriptionParams{}) +} + +func newTestSubscriptionManagerWithParams(t *testing.T, params testSubscriptionParams) *SubscriptionManager { p := &typesfakes.FakeLocalParticipant{} p.CanSubscribeReturns(true) p.IDReturns("subID") @@ -336,11 +466,13 @@ func newTestSubscriptionManager(t *testing.T) *SubscriptionManager { Logger: logger.GetLogger(), OnTrackSubscribed: func(subTrack types.SubscribedTrack) {}, OnTrackUnsubscribed: func(subTrack types.SubscribedTrack) {}, - OnSubscriptionError: func(trackID livekit.TrackID) {}, + OnSubscriptionError: func(trackID livekit.TrackID, fatal bool, err error) {}, TrackResolver: func(identity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { return types.MediaResolverResult{} }, - Telemetry: &telemetryfakes.FakeTelemetryService{}, + Telemetry: &telemetryfakes.FakeTelemetryService{}, + SubscriptionLimitAudio: params.SubscriptionLimitAudio, + SubscriptionLimitVideo: params.SubscriptionLimitVideo, }) } @@ -350,6 +482,8 @@ type testResolver struct { hasTrack bool pubIdentity livekit.ParticipantIdentity pubID livekit.ParticipantID + + paused bool } func newTestResolver(hasPermission bool, hasTrack bool, pubIdentity livekit.ParticipantIdentity, pubID livekit.ParticipantID) *testResolver { @@ -361,6 +495,12 @@ func newTestResolver(hasPermission bool, hasTrack bool, pubIdentity livekit.Part } } +func (t *testResolver) SetPause(paused bool) { + t.lock.Lock() + defer t.lock.Unlock() + t.paused = paused +} + func (t *testResolver) Resolve(identity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { t.lock.Lock() defer t.lock.Unlock() @@ -371,7 +511,7 @@ func (t *testResolver) Resolve(identity livekit.ParticipantIdentity, trackID liv PublisherID: t.pubID, PublisherIdentity: t.pubIdentity, } - if t.hasTrack { + if t.hasTrack && !t.paused { mt := &typesfakes.FakeMediaTrack{} st := &typesfakes.FakeSubscribedTrack{} st.IDReturns(trackID) @@ -389,7 +529,7 @@ func setTestSubscribedTrackBound(t *testing.T, st types.SubscribedTrack) { require.True(t, ok) for i := 0; i < fst.AddOnBindCallCount(); i++ { - fst.AddOnBindArgsForCall(i)() + fst.AddOnBindArgsForCall(i)(nil) } } diff --git a/pkg/rtc/supervisor/participant_supervisor.go b/pkg/rtc/supervisor/participant_supervisor.go index 99126739c..1dc6b9d32 100644 --- a/pkg/rtc/supervisor/participant_supervisor.go +++ b/pkg/rtc/supervisor/participant_supervisor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package supervisor import ( diff --git a/pkg/rtc/supervisor/publication_monitor.go b/pkg/rtc/supervisor/publication_monitor.go index b51e8efd1..c5c61c557 100644 --- a/pkg/rtc/supervisor/publication_monitor.go +++ b/pkg/rtc/supervisor/publication_monitor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package supervisor import ( @@ -34,7 +48,7 @@ type PublicationMonitor struct { params PublicationMonitorParams lock sync.RWMutex - desiredPublishes deque.Deque + desiredPublishes deque.Deque[*publish] isConnected bool @@ -128,7 +142,7 @@ func (p *PublicationMonitor) Check() error { p.lock.RLock() var pub *publish if p.desiredPublishes.Len() > 0 { - pub = p.desiredPublishes.Front().(*publish) + pub = p.desiredPublishes.Front() } isMuted := p.isMuted @@ -160,7 +174,7 @@ func (p *PublicationMonitor) update() { for { var pub *publish if p.desiredPublishes.Len() > 0 { - pub = p.desiredPublishes.PopFront().(*publish) + pub = p.desiredPublishes.PopFront() } if pub == nil { diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index cbc70ba2c..0eaabd1ed 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -8,6 +22,7 @@ import ( "time" "github.com/bep/debounce" + "github.com/pion/dtls/v2/pkg/crypto/elliptic" "github.com/pion/ice/v2" "github.com/pion/interceptor" "github.com/pion/interceptor/pkg/cc" @@ -21,12 +36,14 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -49,6 +66,8 @@ const ( minConnectTimeoutAfterICE = 10 * time.Second maxConnectTimeoutAfterICE = 20 * time.Second // max duration for waiting pc to connect after ICE is connected + maxICECandidates = 20 + shortConnectionThreshold = 90 * time.Second ) @@ -182,7 +201,10 @@ type PCTransport struct { onNegotiationFailed func() // stream allocator for subscriber PC - streamAllocator *sfu.StreamAllocator + streamAllocator *streamallocator.StreamAllocator + + // only for subscriber PC + pacer pacer.Pacer previousAnswer *webrtc.SessionDescription // track id -> description map in previous offer sdp @@ -208,10 +230,10 @@ type PCTransport struct { pendingRestartIceOffer *webrtc.SessionDescription // for cleaner logging - allowedLocalCandidates []string - allowedRemoteCandidates []string - filteredLocalCandidates []string - filteredRemoteCandidates []string + allowedLocalCandidates *utils.DedupedSlice[string] + allowedRemoteCandidates *utils.DedupedSlice[string] + filteredLocalCandidates *utils.DedupedSlice[string] + filteredRemoteCandidates *utils.DedupedSlice[string] } type TransportParams struct { @@ -231,9 +253,7 @@ type TransportParams struct { } func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimator cc.BandwidthEstimator)) (*webrtc.PeerConnection, *webrtc.MediaEngine, error) { - directionConfig := params.DirectionConfig - - me, err := createMediaEngine(params.EnabledCodecs, directionConfig) + me, err := createMediaEngine(params.EnabledCodecs, params.DirectionConfig) if err != nil { return nil, nil, err } @@ -241,6 +261,10 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat se := params.Config.SettingEngine se.DisableMediaEngineCopy(true) + // Change elliptic curve to improve connectivity + // https://github.com/pion/dtls/pull/474 + se.SetDTLSEllipticCurves(elliptic.X25519, elliptic.P384, elliptic.P256) + // // Disable SRTP replay protection (https://datatracker.ietf.org/doc/html/rfc3711#page-15). // Needed due to lack of RTX stream support in Pion. @@ -293,28 +317,14 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } } - lf := serverlogger.NewLoggerFactory(params.Logger) + lf := pionlogger.NewLoggerFactory(params.Logger) if lf != nil { se.LoggerFactory = lf } ir := &interceptor.Registry{} if params.IsSendSide { - isSendSideBWE := false - for _, ext := range directionConfig.RTPHeaderExtension.Video { - if ext == sdp.TransportCCURI { - isSendSideBWE = true - break - } - } - for _, ext := range directionConfig.RTPHeaderExtension.Audio { - if ext == sdp.TransportCCURI { - isSendSideBWE = true - break - } - } - - if isSendSideBWE { + if params.CongestionControlConfig.UseSendSideBWE { gf, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) { return gcc.NewSendSideBWE( gcc.SendSideBWEInitialBitrate(1*1000*1000), @@ -364,13 +374,18 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { eventCh: make(chan event, 50), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, + allowedLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), + allowedRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), + filteredLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), + filteredRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), } if params.IsSendSide { - t.streamAllocator = sfu.NewStreamAllocator(sfu.StreamAllocatorParams{ + t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ Config: params.CongestionControlConfig, Logger: params.Logger, }) t.streamAllocator.Start() + t.pacer = pacer.NewPassThrough(params.Logger) } if err := t.createPeerConnection(); err != nil { @@ -409,6 +424,10 @@ func (t *PCTransport) createPeerConnection() error { return nil } +func (t *PCTransport) GetPacer() pacer.Pacer { + return t.pacer +} + func (t *PCTransport) SetSignalingRTT(rtt uint32) { t.signalingRTT.Store(rtt) } @@ -495,7 +514,7 @@ func (t *PCTransport) resetShortConn() { t.lock.Unlock() } -func (t *PCTransport) isShortConnection(at time.Time) (bool, time.Duration) { +func (t *PCTransport) IsShortConnection(at time.Time) (bool, time.Duration) { t.lock.RLock() defer t.lock.RUnlock() @@ -568,7 +587,7 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) { isShort := forceShortConn if !isShort { var duration time.Duration - isShort, duration = t.isShortConnection(time.Now()) + isShort, duration = t.IsShortConnection(time.Now()) if isShort { pair, err := t.getSelectedPair() if err != nil { @@ -599,6 +618,11 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState case webrtc.ICEConnectionStateChecking: t.setICEStartedAt(time.Now()) + + case webrtc.ICEConnectionStateDisconnected: + fallthrough + case webrtc.ICEConnectionStateFailed: + t.params.Logger.Infow("ice connection state change unexpected", "state", state.String()) } } @@ -614,6 +638,7 @@ func (t *PCTransport) onPeerConnectionStateChange(state webrtc.PeerConnectionSta } t.maybeNotifyFullyEstablished() + t.logICECandidates() } case webrtc.PeerConnectionStateFailed: t.params.Logger.Infow("peer connection failed") @@ -893,6 +918,9 @@ func (t *PCTransport) Close() { if t.streamAllocator != nil { t.streamAllocator.Stop() } + if t.pacer != nil { + t.pacer.Stop() + } _ = t.pc.Close() @@ -1085,7 +1113,7 @@ func (t *PCTransport) ResetShortConnOnICERestart() { t.resetShortConnOnICERestart.Store(true) } -func (t *PCTransport) OnStreamStateChange(f func(update *sfu.StreamStateUpdate) error) { +func (t *PCTransport) OnStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { if t.streamAllocator == nil { return } @@ -1098,7 +1126,7 @@ func (t *PCTransport) AddTrackToStreamAllocator(subTrack types.SubscribedTrack) return } - t.streamAllocator.AddTrack(subTrack.DownTrack(), sfu.AddTrackParams{ + t.streamAllocator.AddTrack(subTrack.DownTrack(), streamallocator.AddTrackParams{ Source: subTrack.MediaTrack().Source(), IsSimulcast: subTrack.MediaTrack().IsSimulcast(), PublisherID: subTrack.MediaTrack().PublisherID(), @@ -1113,6 +1141,22 @@ func (t *PCTransport) RemoveTrackFromStreamAllocator(subTrack types.SubscribedTr t.streamAllocator.RemoveTrack(subTrack.DownTrack()) } +func (t *PCTransport) SetAllowPauseOfStreamAllocator(allowPause bool) { + if t.streamAllocator == nil { + return + } + + t.streamAllocator.SetAllowPause(allowPause) +} + +func (t *PCTransport) SetChannelCapacityOfStreamAllocator(channelCapacity int64) { + if t.streamAllocator == nil { + return + } + + t.streamAllocator.SetChannelCapacity(channelCapacity) +} + func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { unknown := types.ICEConnectionTypeUnknown if t.pc == nil { @@ -1130,7 +1174,7 @@ func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { // Pion would have created a prflx candidate with the same address as the relay candidate. // to report an accurate connection type, we'll compare to see if existing relay candidates match t.lock.RLock() - allowedRemoteCandidates := t.allowedRemoteCandidates + allowedRemoteCandidates := t.allowedRemoteCandidates.Get() t.lock.RUnlock() for _, ci := range allowedRemoteCandidates { @@ -1263,7 +1307,7 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc } tr.SetMid(mid) - // save mid -> senders for migration resue + // save mid -> senders for migration reuse sender := tr.Sender() senders[mid] = sender @@ -1294,7 +1338,7 @@ func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) { } return } else if offer != nil { - // in migration case, can't reuse tranceiver before negotiated except track subscribed at previous node + // in migration case, can't reuse transceiver before negotiated except track subscribed at previous node t.canReuseTransceiver = false if err := t.parseTrackMid(*offer, senders); err != nil { t.params.Logger.Errorw("parse previous offer failed", err, "offer", offer.SDP) @@ -1318,12 +1362,12 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map } if split := strings.Split(msid, " "); len(split) == 2 { - trackid := split[1] + trackID := split[1] mid := lksdp.GetMidValue(m) if mid == "" { return ErrMidNotFound } - t.previousTrackDescription[trackid] = &trackDescription{ + t.previousTrackDescription[trackID] = &trackDescription{ mid: mid, sender: senders[mid], } @@ -1349,6 +1393,11 @@ func (t *PCTransport) postEvent(event event) { func (t *PCTransport) processEvents() { for event := range t.eventCh { + if t.isClosed.Load() { + // just drain the channel without processing events + continue + } + err := t.handleEvent(&event) if err != nil { t.params.Logger.Errorw("error handling event", err, "event", event.String()) @@ -1445,12 +1494,12 @@ func (t *PCTransport) clearLocalDescriptionSent() { t.cacheLocalCandidates = true t.cachedLocalCandidates = nil - t.allowedLocalCandidates = nil + t.allowedLocalCandidates.Clear() t.lock.Lock() - t.allowedRemoteCandidates = nil + t.allowedRemoteCandidates.Clear() t.lock.Unlock() - t.filteredLocalCandidates = nil - t.filteredRemoteCandidates = nil + t.filteredLocalCandidates.Clear() + t.filteredRemoteCandidates.Clear() } func (t *PCTransport) handleLocalICECandidate(e *event) error { @@ -1460,7 +1509,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { if t.preferTCP.Load() && c != nil && c.Protocol != webrtc.ICEProtocolTCP { cstr := c.String() t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) - t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) + t.filteredLocalCandidates.Add(cstr) filtered = true } @@ -1469,7 +1518,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { } if c != nil { - t.allowedLocalCandidates = append(t.allowedLocalCandidates, c.String()) + t.allowedLocalCandidates.Add(c.String()) } if t.cacheLocalCandidates { t.cachedLocalCandidates = append(t.cachedLocalCandidates, c) @@ -1489,7 +1538,7 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { filtered := false if t.preferTCP.Load() && !strings.Contains(c.Candidate, "tcp") { t.params.Logger.Debugw("filtering out remote candidate", "candidate", c.Candidate) - t.filteredRemoteCandidates = append(t.filteredRemoteCandidates, c.Candidate) + t.filteredRemoteCandidates.Add(c.Candidate) filtered = true } @@ -1498,7 +1547,7 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { } t.lock.Lock() - t.allowedRemoteCandidates = append(t.allowedRemoteCandidates, c.Candidate) + t.allowedRemoteCandidates.Add(c.Candidate) t.lock.Unlock() if t.pc.RemoteDescription() == nil { @@ -1516,10 +1565,10 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { func (t *PCTransport) handleLogICECandidates(e *event) error { t.params.Logger.Infow( "ice candidates", - "lc", t.allowedLocalCandidates, - "rc", t.allowedRemoteCandidates, - "lc (filtered)", t.filteredLocalCandidates, - "rc (filtered)", t.filteredRemoteCandidates, + "lc", t.allowedLocalCandidates.Get(), + "rc", t.allowedRemoteCandidates.Get(), + "lc (filtered)", t.filteredLocalCandidates.Get(), + "rc (filtered)", t.filteredRemoteCandidates.Get(), ) return nil @@ -1589,6 +1638,13 @@ func (t *PCTransport) setupSignalStateCheckTimer() { failed := t.negotiationState != NegotiationStateNone if t.negotiateCounter.Load() == negotiateVersion && failed { + t.params.Logger.Infow( + "negotiation timed out", + "localCurrent", t.pc.CurrentLocalDescription(), + "localPending", t.pc.PendingLocalDescription(), + "remoteCurrent", t.pc.CurrentRemoteDescription(), + "remotePending", t.pc.PendingRemoteDescription(), + ) if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { onNegotiationFailed() } @@ -1844,7 +1900,13 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) error { if err := t.setRemoteDescription(*sd); err != nil { - return err + // Pion will call RTPSender.Send method for each new added Downtrack, and return error if the DownTrack.Bind + // returns error. In case of Downtrack.Bind returns ErrUnsupportedCodec, the signal state will be stable as negotiation is aleady compelted + // before startRTPSenders, and the peerconnection state can be recovered by next negotiation which will be triggered + // by the SubscriptionManager unsubscribe the failure DownTrack. So don't treat this error as negotiation failure. + if !errors.Is(err, webrtc.ErrUnsupportedCodec) { + return err + } } t.clearSignalStateCheckTimer() @@ -1937,7 +1999,16 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool c.SDPFmtpLine += ";sprop-stereo=1" } if nack { - c.RTCPFeedback = append(c.RTCPFeedback, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBNACK}) + var nackFound bool + for _, fb := range c.RTCPFeedback { + if fb.Type == webrtc.TypeRTCPFBNACK { + nackFound = true + break + } + } + if !nackFound { + c.RTCPFeedback = append(c.RTCPFeedback, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBNACK}) + } } } configCodecs = append(configCodecs, c) diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index ef8d66d21..eb59df779 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -246,7 +260,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { // exchange ICE handleICEExchange(t, transportA, transportB) - // first anwser missed + // first answer missed var firstAnswerReceived atomic.Bool transportB.OnAnswer(func(sd webrtc.SessionDescription) error { if firstAnswerReceived.Load() { @@ -506,3 +520,40 @@ func connectTransports(t *testing.T, offerer, answerer *PCTransport, isICERestar return answerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected }, 10*time.Second, time.Millisecond*10, "answerer did not become connected") } + +func TestConfigureAudioTransceiver(t *testing.T) { + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for _, testcase := range []struct { + nack bool + stereo bool + }{ + {false, false}, + {true, false}, + {false, true}, + {true, true}, + } { + t.Run(fmt.Sprintf("nack=%v,stereo=%v", testcase.nack, testcase.stereo), func(t *testing.T) { + tr, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RtpTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) + require.NoError(t, err) + + configureAudioTransceiver(tr, testcase.stereo, testcase.nack) + codecs := tr.Sender().GetParameters().Codecs + for _, codec := range codecs { + if strings.Contains(codec.MimeType, webrtc.MimeTypeOpus) { + require.Equal(t, testcase.stereo, strings.Contains(codec.SDPFmtpLine, "sprop-stereo=1")) + var nackEnabled bool + for _, fb := range codec.RTCPFeedback { + if fb.Type == webrtc.TypeRTCPFBNACK { + nackEnabled = true + break + } + } + require.Equal(t, testcase.nack, nackEnabled) + } + } + }) + } +} diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index d4d347ba3..3f698ebe1 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -17,6 +31,8 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -33,22 +49,23 @@ const ( ) type TransportManagerParams struct { - Identity livekit.ParticipantIdentity - SID livekit.ParticipantID - SubscriberAsPrimary bool - Config *WebRTCConfig - ProtocolVersion types.ProtocolVersion - Telemetry telemetry.TelemetryService - CongestionControlConfig config.CongestionControlConfig - EnabledCodecs []*livekit.Codec - SimTracks map[uint32]SimulcastTrackInfo - ClientConf *livekit.ClientConfiguration - ClientInfo ClientInfo - Migration bool - AllowTCPFallback bool - TCPFallbackRTTThreshold int - TURNSEnabled bool - Logger logger.Logger + Identity livekit.ParticipantIdentity + SID livekit.ParticipantID + SubscriberAsPrimary bool + Config *WebRTCConfig + ProtocolVersion types.ProtocolVersion + Telemetry telemetry.TelemetryService + CongestionControlConfig config.CongestionControlConfig + EnabledCodecs []*livekit.Codec + SimTracks map[uint32]SimulcastTrackInfo + ClientConf *livekit.ClientConfiguration + ClientInfo ClientInfo + Migration bool + AllowTCPFallback bool + TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool + TURNSEnabled bool + Logger logger.Logger } type TransportManager struct { @@ -93,18 +110,32 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro } t.mediaLossProxy.OnMediaLossUpdate(t.onMediaLossUpdate) - enabledCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) - for _, c := range params.EnabledCodecs { - var disabled bool - for _, disableCodec := range params.ClientConf.GetDisabledCodecs().GetCodecs() { + subscribeCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) + publishCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) + shouldDisable := func(c *livekit.Codec, disabledCodecs []*livekit.Codec) bool { + for _, disableCodec := range disabledCodecs { // disable codec's fmtp is empty means disable this codec entirely if strings.EqualFold(c.Mime, disableCodec.Mime) && (disableCodec.FmtpLine == "" || disableCodec.FmtpLine == c.FmtpLine) { - disabled = true - break + return true } } - if !disabled { - enabledCodecs = append(enabledCodecs, c) + return false + } + for _, c := range params.EnabledCodecs { + var publishDisabled bool + var subscribeDisabled bool + if shouldDisable(c, params.ClientConf.GetDisabledCodecs().GetCodecs()) { + publishDisabled = true + subscribeDisabled = true + } + if shouldDisable(c, params.ClientConf.GetDisabledCodecs().GetPublish()) { + publishDisabled = true + } + if !publishDisabled { + publishCodecs = append(publishCodecs, c) + } + if !subscribeDisabled { + subscribeCodecs = append(subscribeCodecs, c) } } @@ -116,7 +147,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro DirectionConfig: params.Config.Publisher, CongestionControlConfig: params.CongestionControlConfig, Telemetry: params.Telemetry, - EnabledCodecs: enabledCodecs, + EnabledCodecs: publishCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER), SimTracks: params.SimTracks, ClientInfo: params.ClientInfo, @@ -148,7 +179,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro DirectionConfig: params.Config.Subscriber, CongestionControlConfig: params.CongestionControlConfig, Telemetry: params.Telemetry, - EnabledCodecs: enabledCodecs, + EnabledCodecs: subscribeCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER), ClientInfo: params.ClientInfo, IsOfferer: true, @@ -188,10 +219,6 @@ func (t *TransportManager) Close() { t.subscriber.Close() } -func (t *TransportManager) HaveAllTransportEverConnected() bool { - return t.publisher.HasEverConnected() && t.subscriber.HasEverConnected() -} - func (t *TransportManager) SubscriberClose() { t.subscriber.Close() } @@ -215,6 +242,10 @@ func (t *TransportManager) OnPublisherTrack(f func(track *webrtc.TrackRemote, rt t.publisher.OnTrack(f) } +func (t *TransportManager) HasPublisherEverConnected() bool { + return t.publisher.HasEverConnected() +} + func (t *TransportManager) IsPublisherEstablished() bool { return t.publisher.IsEstablished() } @@ -243,7 +274,7 @@ func (t *TransportManager) OnSubscriberInitialConnected(f func()) { t.onSubscriberInitialConnected = f } -func (t *TransportManager) OnSubscriberStreamStateChange(f func(update *sfu.StreamStateUpdate) error) { +func (t *TransportManager) OnSubscriberStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { t.subscriber.OnStreamStateChange(f) } @@ -267,6 +298,10 @@ func (t *TransportManager) WriteSubscriberRTCP(pkts []rtcp.Packet) error { return t.subscriber.WriteRTCP(pkts) } +func (t *TransportManager) GetSubscriberPacer() pacer.Pacer { + return t.subscriber.GetPacer() +} + func (t *TransportManager) OnPrimaryTransportInitialConnected(f func()) { t.onPrimaryTransportInitialConnected = f } @@ -363,7 +398,7 @@ func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescripti answer := lastAnswer.(webrtc.SessionDescription) parsedAnswer, err1 := answer.Unmarshal() if err1 != nil { - // should not happend + // should not happen t.params.Logger.Errorw("failed to parse last answer", err) return } @@ -456,15 +491,41 @@ func (t *TransportManager) NegotiateSubscriber(force bool) { t.subscriber.Negotiate(force) } -func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, resetShortConnection bool) { - if iceConfig != nil { - t.SetICEConfig(iceConfig) +func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) { + var ( + isShort bool + duration time.Duration + resetShortConnection bool + ) + switch reason { + case livekit.ReconnectReason_RR_PUBLISHER_FAILED: + resetShortConnection = true + isShort, duration = t.publisher.IsShortConnection(time.Now()) + + case livekit.ReconnectReason_RR_SUBSCRIBER_FAILED: + resetShortConnection = true + isShort, duration = t.subscriber.IsShortConnection(time.Now()) + } + + if isShort { + t.lock.Lock() + t.resetTransportConfigureLocked(false) + t.lock.Unlock() + t.params.Logger.Infow("short connection by client ice restart", "duration", duration, "reason", reason) + t.handleConnectionFailed(isShort) } if resetShortConnection { t.publisher.ResetShortConnOnICERestart() t.subscriber.ResetShortConnOnICERestart() } +} + +func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) { + if iceConfig != nil { + t.SetICEConfig(iceConfig) + } + t.subscriber.ICERestart() } @@ -478,14 +539,18 @@ func (t *TransportManager) SetICEConfig(iceConfig *livekit.ICEConfig) { t.configureICE(iceConfig, true) } +func (t *TransportManager) resetTransportConfigureLocked(reconfigured bool) { + t.failureCount = 0 + t.isTransportReconfigured = reconfigured + t.udpLossUnstableCount = 0 + t.lastFailure = time.Time{} +} + func (t *TransportManager) configureICE(iceConfig *livekit.ICEConfig, reset bool) { t.lock.Lock() isEqual := proto.Equal(t.iceConfig, iceConfig) if reset || !isEqual { - t.failureCount = 0 - t.isTransportReconfigured = !reset - t.udpLossUnstableCount = 0 - t.lastFailure = time.Time{} + t.resetTransportConfigureLocked(!reset) } if isEqual { @@ -552,7 +617,7 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { } // - // Checking only `PreferenceSubcriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will + // Checking only `PreferenceSubscriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will // flow through here. // // As both transports are switched to the same type on any failure, checking just subscriber should be fine. @@ -679,7 +744,7 @@ func (t *TransportManager) OnReceiverReport(dt *sfu.DownTrack, report *rtcp.Rece } func (t *TransportManager) onMediaLossUpdate(loss uint8) { - if t.params.TCPFallbackRTTThreshold == 0 { + if t.params.TCPFallbackRTTThreshold == 0 || !t.params.AllowUDPUnstableFallback { return } t.lock.Lock() @@ -751,3 +816,11 @@ func (t *TransportManager) SetSignalSourceValid(valid bool) { t.signalSourceValid.Store(valid) t.params.Logger.Debugw("signal source valid", "valid", valid) } + +func (t *TransportManager) SetSubscriberAllowPause(allowPause bool) { + t.subscriber.SetAllowPauseOfStreamAllocator(allowPause) +} + +func (t *TransportManager) SetSubscriberChannelCapacity(channelCapacity int64) { + t.subscriber.SetChannelCapacityOfStreamAllocator(channelCapacity) +} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 6bfc95edf..6431dc661 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package types import ( @@ -10,10 +24,12 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/pacer" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -85,6 +101,7 @@ const ( ParticipantCloseReasonMigrationRequested ParticipantCloseReasonOvercommitted ParticipantCloseReasonPublicationError + ParticipantCloseReasonSubscriptionError ) func (p ParticipantCloseReason) String() string { @@ -129,6 +146,8 @@ func (p ParticipantCloseReason) String() string { return "OVERCOMMITTED" case ParticipantCloseReasonPublicationError: return "PUBLICATION_ERROR" + case ParticipantCloseReasonSubscriptionError: + return "SUBSCRIPTION_ERROR" default: return fmt.Sprintf("%d", int(p)) } @@ -159,7 +178,7 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_SERVER_SHUTDOWN case ParticipantCloseReasonOvercommitted: return livekit.DisconnectReason_SERVER_SHUTDOWN - case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError: + case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError: return livekit.DisconnectReason_STATE_MISMATCH default: // the other types will map to unknown reason @@ -169,17 +188,57 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { // --------------------------------------------- +type SignallingCloseReason int + +const ( + SignallingCloseReasonUnknown SignallingCloseReason = iota + SignallingCloseReasonMigration + SignallingCloseReasonResume + SignallingCloseReasonTransportFailure + SignallingCloseReasonFullReconnectPublicationError + SignallingCloseReasonFullReconnectSubscriptionError + SignallingCloseReasonFullReconnectNegotiateFailed + SignallingCloseReasonParticipantClose +) + +func (s SignallingCloseReason) String() string { + switch s { + case SignallingCloseReasonUnknown: + return "UNKNOWN" + case SignallingCloseReasonMigration: + return "MIGRATION" + case SignallingCloseReasonResume: + return "RESUME" + case SignallingCloseReasonTransportFailure: + return "TRANSPORT_FAILURE" + case SignallingCloseReasonFullReconnectPublicationError: + return "FULL_RECONNECT_PUBLICATION_ERROR" + case SignallingCloseReasonFullReconnectSubscriptionError: + return "FULL_RECONNECT_SUBSCRIPTION_ERROR" + case SignallingCloseReasonFullReconnectNegotiateFailed: + return "FULL_RECONNECT_NEGOTIATE_FAILED" + case SignallingCloseReasonParticipantClose: + return "PARTICIPANT_CLOSE" + default: + return fmt.Sprintf("%d", int(s)) + } +} + +// --------------------------------------------- + //counterfeiter:generate . Participant type Participant interface { ID() livekit.ParticipantID Identity() livekit.ParticipantIdentity State() livekit.ParticipantInfo_State + CanSkipBroadcast() bool ToProto() *livekit.ParticipantInfo SetName(name string) SetMetadata(metadata string) + IsPublisher() bool GetPublishedTrack(sid livekit.TrackID) MediaTrack GetPublishedTracks() []MediaTrack RemovePublishedTrack(track MediaTrack, willBeResumed bool, shouldClose bool) @@ -193,14 +252,14 @@ type Participant interface { IsRecorder() bool Start() - Close(sendLeave bool, reason ParticipantCloseReason) error + Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error - SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) // updates from remotes UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, - timedVersion *livekit.TimedVersion, + timedVersion utils.TimedVersion, resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) LocalParticipant, resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error @@ -229,7 +288,10 @@ type AddTrackParams struct { type LocalParticipant interface { Participant + ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) + // getters + GetTrailer() []byte GetLogger() logger.Logger GetAdaptiveStream() bool ProtocolVersion() ProtocolVersion @@ -239,19 +301,21 @@ type LocalParticipant interface { IsDisconnected() bool IsIdle() bool SubscriberAsPrimary() bool + GetClientInfo() *livekit.ClientInfo GetClientConfiguration() *livekit.ClientConfiguration GetICEConnectionType() ICEConnectionType GetBufferFactory() *buffer.Factory SetResponseSink(sink routing.MessageSink) - CloseSignalConnection() + CloseSignalConnection(reason SignallingCloseReason) UpdateLastSeenSignal() SetSignalSourceValid(valid bool) + HandleSignalSourceClose() // permissions ClaimGrants() *auth.ClaimGrants SetPermission(permission *livekit.ParticipantPermission) bool - CanPublish() bool + CanPublishSource(source livekit.TrackSource) bool CanSubscribe() bool CanPublishData() bool @@ -263,7 +327,7 @@ type LocalParticipant interface { HandleAnswer(sdp webrtc.SessionDescription) Negotiate(force bool) - ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) + ICERestart(iceConfig *livekit.ICEConfig) AddTrackToSubscriber(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) AddTransceiverFromTrackToSubscriber(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) RemoveTrackFromSubscriber(sender *webrtc.RTPSender) error @@ -281,7 +345,6 @@ type LocalParticipant interface { // returns list of participant identities that the current participant is subscribed to GetSubscribedParticipants() []livekit.ParticipantID IsSubscribedTo(sid livekit.ParticipantID) bool - IsPublisher() bool GetAudioLevel() (smoothedLevel float64, active bool) GetConnectionQuality() *livekit.ConnectionQualityInfo @@ -295,7 +358,7 @@ type LocalParticipant interface { SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) SendRefreshToken(token string) error - SendReconnectResponse(reconnectResponse *livekit.ReconnectResponse) error + HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error IssueFullReconnect(reason ParticipantCloseReason) // callbacks @@ -311,7 +374,7 @@ type LocalParticipant interface { OnParticipantUpdate(callback func(LocalParticipant)) OnDataPacket(callback func(LocalParticipant, *livekit.DataPacket)) OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) - OnClose(callback func(LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) + OnClose(callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) OnReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) @@ -333,6 +396,12 @@ type LocalParticipant interface { UpdateSubscribedQuality(nodeID livekit.NodeID, trackID livekit.TrackID, maxQualities []SubscribedCodecQuality) error UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error + + // down stream bandwidth management + SetSubscriberAllowPause(allowPause bool) + SetSubscriberChannelCapacity(channelCapacity int64) + + GetPacer() pacer.Pacer } // Room is a container of participants, and can provide room-level actions @@ -349,6 +418,7 @@ type Room interface { UpdateVideoLayers(participant Participant, updateVideoLayers *livekit.UpdateVideoLayers) error ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) MediaResolverResult GetLocalParticipants() []LocalParticipant + UpdateParticipantMetadata(participant LocalParticipant, name string, metadata string) } // MediaTrack represents a media track @@ -394,6 +464,8 @@ type MediaTrack interface { Receivers() []sfu.TrackReceiver ClearAllReceivers(willBeResumed bool) + + IsEncrypted() bool } //counterfeiter:generate . LocalMediaTrack @@ -416,7 +488,7 @@ type LocalMediaTrack interface { //counterfeiter:generate . SubscribedTrack type SubscribedTrack interface { - AddOnBind(f func()) + AddOnBind(f func(error)) IsBound() bool Close(willBeResumed bool) OnClose(f func(willBeResumed bool)) diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 4c0257734..93449ae79 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package types type ProtocolVersion int diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index f6ee5b324..0b9a4517c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -128,6 +128,16 @@ type FakeLocalMediaTrack struct { iDReturnsOnCall map[int]struct { result1 livekit.TrackID } + IsEncryptedStub func() bool + isEncryptedMutex sync.RWMutex + isEncryptedArgsForCall []struct { + } + isEncryptedReturns struct { + result1 bool + } + isEncryptedReturnsOnCall map[int]struct { + result1 bool + } IsMutedStub func() bool isMutedMutex sync.RWMutex isMutedArgsForCall []struct { @@ -928,6 +938,59 @@ func (fake *FakeLocalMediaTrack) IDReturnsOnCall(i int, result1 livekit.TrackID) }{result1} } +func (fake *FakeLocalMediaTrack) IsEncrypted() bool { + fake.isEncryptedMutex.Lock() + ret, specificReturn := fake.isEncryptedReturnsOnCall[len(fake.isEncryptedArgsForCall)] + fake.isEncryptedArgsForCall = append(fake.isEncryptedArgsForCall, struct { + }{}) + stub := fake.IsEncryptedStub + fakeReturns := fake.isEncryptedReturns + fake.recordInvocation("IsEncrypted", []interface{}{}) + fake.isEncryptedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) IsEncryptedCallCount() int { + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() + return len(fake.isEncryptedArgsForCall) +} + +func (fake *FakeLocalMediaTrack) IsEncryptedCalls(stub func() bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = stub +} + +func (fake *FakeLocalMediaTrack) IsEncryptedReturns(result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + fake.isEncryptedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalMediaTrack) IsEncryptedReturnsOnCall(i int, result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + if fake.isEncryptedReturnsOnCall == nil { + fake.isEncryptedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isEncryptedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalMediaTrack) IsMuted() bool { fake.isMutedMutex.Lock() ret, specificReturn := fake.isMutedReturnsOnCall[len(fake.isMutedArgsForCall)] @@ -1947,6 +2010,8 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.hasSdpCidMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() fake.isMutedMutex.RLock() defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 67ce745dd..6e3863a45 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -9,9 +9,11 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" "github.com/pion/rtcp" webrtc "github.com/pion/webrtc/v3" ) @@ -67,16 +69,6 @@ type FakeLocalParticipant struct { arg2 *webrtc.RTPTransceiver arg3 sfu.DownTrackState } - CanPublishStub func() bool - canPublishMutex sync.RWMutex - canPublishArgsForCall []struct { - } - canPublishReturns struct { - result1 bool - } - canPublishReturnsOnCall map[int]struct { - result1 bool - } CanPublishDataStub func() bool canPublishDataMutex sync.RWMutex canPublishDataArgsForCall []struct { @@ -87,6 +79,27 @@ type FakeLocalParticipant struct { canPublishDataReturnsOnCall map[int]struct { result1 bool } + CanPublishSourceStub func(livekit.TrackSource) bool + canPublishSourceMutex sync.RWMutex + canPublishSourceArgsForCall []struct { + arg1 livekit.TrackSource + } + canPublishSourceReturns struct { + result1 bool + } + canPublishSourceReturnsOnCall map[int]struct { + result1 bool + } + CanSkipBroadcastStub func() bool + canSkipBroadcastMutex sync.RWMutex + canSkipBroadcastArgsForCall []struct { + } + canSkipBroadcastReturns struct { + result1 bool + } + canSkipBroadcastReturnsOnCall map[int]struct { + result1 bool + } CanSubscribeStub func() bool canSubscribeMutex sync.RWMutex canSubscribeArgsForCall []struct { @@ -107,11 +120,12 @@ type FakeLocalParticipant struct { claimGrantsReturnsOnCall map[int]struct { result1 *auth.ClaimGrants } - CloseStub func(bool, types.ParticipantCloseReason) error + CloseStub func(bool, types.ParticipantCloseReason, bool) error closeMutex sync.RWMutex closeArgsForCall []struct { arg1 bool arg2 types.ParticipantCloseReason + arg3 bool } closeReturns struct { result1 error @@ -119,9 +133,10 @@ type FakeLocalParticipant struct { closeReturnsOnCall map[int]struct { result1 error } - CloseSignalConnectionStub func() + CloseSignalConnectionStub func(types.SignallingCloseReason) closeSignalConnectionMutex sync.RWMutex closeSignalConnectionArgsForCall []struct { + arg1 types.SignallingCloseReason } ConnectedAtStub func() time.Time connectedAtMutex sync.RWMutex @@ -198,6 +213,16 @@ type FakeLocalParticipant struct { getClientConfigurationReturnsOnCall map[int]struct { result1 *livekit.ClientConfiguration } + GetClientInfoStub func() *livekit.ClientInfo + getClientInfoMutex sync.RWMutex + getClientInfoArgsForCall []struct { + } + getClientInfoReturns struct { + result1 *livekit.ClientInfo + } + getClientInfoReturnsOnCall map[int]struct { + result1 *livekit.ClientInfo + } GetConnectionQualityStub func() *livekit.ConnectionQualityInfo getConnectionQualityMutex sync.RWMutex getConnectionQualityArgsForCall []struct { @@ -228,6 +253,16 @@ type FakeLocalParticipant struct { getLoggerReturnsOnCall map[int]struct { result1 logger.Logger } + GetPacerStub func() pacer.Pacer + getPacerMutex sync.RWMutex + getPacerArgsForCall []struct { + } + getPacerReturns struct { + result1 pacer.Pacer + } + getPacerReturnsOnCall map[int]struct { + result1 pacer.Pacer + } GetPublishedTrackStub func(livekit.TrackID) types.MediaTrack getPublishedTrackMutex sync.RWMutex getPublishedTrackArgsForCall []struct { @@ -269,6 +304,16 @@ type FakeLocalParticipant struct { getSubscribedTracksReturnsOnCall map[int]struct { result1 []types.SubscribedTrack } + GetTrailerStub func() []byte + getTrailerMutex sync.RWMutex + getTrailerArgsForCall []struct { + } + getTrailerReturns struct { + result1 []byte + } + getTrailerReturnsOnCall map[int]struct { + result1 []byte + } HandleAnswerStub func(webrtc.SessionDescription) handleAnswerMutex sync.RWMutex handleAnswerArgsForCall []struct { @@ -279,6 +324,22 @@ type FakeLocalParticipant struct { handleOfferArgsForCall []struct { arg1 webrtc.SessionDescription } + HandleReconnectAndSendResponseStub func(livekit.ReconnectReason, *livekit.ReconnectResponse) error + handleReconnectAndSendResponseMutex sync.RWMutex + handleReconnectAndSendResponseArgsForCall []struct { + arg1 livekit.ReconnectReason + arg2 *livekit.ReconnectResponse + } + handleReconnectAndSendResponseReturns struct { + result1 error + } + handleReconnectAndSendResponseReturnsOnCall map[int]struct { + result1 error + } + HandleSignalSourceCloseStub func() + handleSignalSourceCloseMutex sync.RWMutex + handleSignalSourceCloseArgsForCall []struct { + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -301,11 +362,10 @@ type FakeLocalParticipant struct { hiddenReturnsOnCall map[int]struct { result1 bool } - ICERestartStub func(*livekit.ICEConfig, livekit.ReconnectReason) + ICERestartStub func(*livekit.ICEConfig) iCERestartMutex sync.RWMutex iCERestartArgsForCall []struct { arg1 *livekit.ICEConfig - arg2 livekit.ReconnectReason } IDStub func() livekit.ParticipantID iDMutex sync.RWMutex @@ -435,10 +495,10 @@ type FakeLocalParticipant struct { onClaimsChangedArgsForCall []struct { arg1 func(types.LocalParticipant) } - OnCloseStub func(func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) + OnCloseStub func(func(types.LocalParticipant)) onCloseMutex sync.RWMutex onCloseArgsForCall []struct { - arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + arg1 func(types.LocalParticipant) } OnDataPacketStub func(func(types.LocalParticipant, *livekit.DataPacket)) onDataPacketMutex sync.RWMutex @@ -564,17 +624,6 @@ type FakeLocalParticipant struct { sendParticipantUpdateReturnsOnCall map[int]struct { result1 error } - SendReconnectResponseStub func(*livekit.ReconnectResponse) error - sendReconnectResponseMutex sync.RWMutex - sendReconnectResponseArgsForCall []struct { - arg1 *livekit.ReconnectResponse - } - sendReconnectResponseReturns struct { - result1 error - } - sendReconnectResponseReturnsOnCall map[int]struct { - result1 error - } SendRefreshTokenStub func(string) error sendRefreshTokenMutex sync.RWMutex sendRefreshTokenArgsForCall []struct { @@ -658,6 +707,16 @@ type FakeLocalParticipant struct { setSignalSourceValidArgsForCall []struct { arg1 bool } + SetSubscriberAllowPauseStub func(bool) + setSubscriberAllowPauseMutex sync.RWMutex + setSubscriberAllowPauseArgsForCall []struct { + arg1 bool + } + SetSubscriberChannelCapacityStub func(int64) + setSubscriberChannelCapacityMutex sync.RWMutex + setSubscriberChannelCapacityArgsForCall []struct { + arg1 int64 + } SetTrackMutedStub func(livekit.TrackID, bool, bool) setTrackMutedMutex sync.RWMutex setTrackMutedArgsForCall []struct { @@ -694,17 +753,17 @@ type FakeLocalParticipant struct { subscriberAsPrimaryReturnsOnCall map[int]struct { result1 bool } - SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, utils.TimedVersion) subscriptionPermissionMutex sync.RWMutex subscriptionPermissionArgsForCall []struct { } subscriptionPermissionReturns struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } subscriptionPermissionReturnsOnCall map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } SubscriptionPermissionUpdateStub func(livekit.ParticipantID, livekit.TrackID, bool) subscriptionPermissionUpdateMutex sync.RWMutex @@ -723,6 +782,18 @@ type FakeLocalParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } + ToProtoWithVersionStub func() (*livekit.ParticipantInfo, utils.TimedVersion) + toProtoWithVersionMutex sync.RWMutex + toProtoWithVersionArgsForCall []struct { + } + toProtoWithVersionReturns struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } + toProtoWithVersionReturnsOnCall map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } UncacheDownTrackStub func(*webrtc.RTPTransceiver) uncacheDownTrackMutex sync.RWMutex uncacheDownTrackArgsForCall []struct { @@ -779,11 +850,11 @@ type FakeLocalParticipant struct { arg1 livekit.TrackID arg2 *livekit.UpdateTrackSettings } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant } @@ -1060,59 +1131,6 @@ func (fake *FakeLocalParticipant) CacheDownTrackArgsForCall(i int) (livekit.Trac return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeLocalParticipant) CanPublish() bool { - fake.canPublishMutex.Lock() - ret, specificReturn := fake.canPublishReturnsOnCall[len(fake.canPublishArgsForCall)] - fake.canPublishArgsForCall = append(fake.canPublishArgsForCall, struct { - }{}) - stub := fake.CanPublishStub - fakeReturns := fake.canPublishReturns - fake.recordInvocation("CanPublish", []interface{}{}) - fake.canPublishMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) CanPublishCallCount() int { - fake.canPublishMutex.RLock() - defer fake.canPublishMutex.RUnlock() - return len(fake.canPublishArgsForCall) -} - -func (fake *FakeLocalParticipant) CanPublishCalls(stub func() bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = stub -} - -func (fake *FakeLocalParticipant) CanPublishReturns(result1 bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = nil - fake.canPublishReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) CanPublishReturnsOnCall(i int, result1 bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = nil - if fake.canPublishReturnsOnCall == nil { - fake.canPublishReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.canPublishReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) CanPublishData() bool { fake.canPublishDataMutex.Lock() ret, specificReturn := fake.canPublishDataReturnsOnCall[len(fake.canPublishDataArgsForCall)] @@ -1166,6 +1184,120 @@ func (fake *FakeLocalParticipant) CanPublishDataReturnsOnCall(i int, result1 boo }{result1} } +func (fake *FakeLocalParticipant) CanPublishSource(arg1 livekit.TrackSource) bool { + fake.canPublishSourceMutex.Lock() + ret, specificReturn := fake.canPublishSourceReturnsOnCall[len(fake.canPublishSourceArgsForCall)] + fake.canPublishSourceArgsForCall = append(fake.canPublishSourceArgsForCall, struct { + arg1 livekit.TrackSource + }{arg1}) + stub := fake.CanPublishSourceStub + fakeReturns := fake.canPublishSourceReturns + fake.recordInvocation("CanPublishSource", []interface{}{arg1}) + fake.canPublishSourceMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) CanPublishSourceCallCount() int { + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() + return len(fake.canPublishSourceArgsForCall) +} + +func (fake *FakeLocalParticipant) CanPublishSourceCalls(stub func(livekit.TrackSource) bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = stub +} + +func (fake *FakeLocalParticipant) CanPublishSourceArgsForCall(i int) livekit.TrackSource { + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() + argsForCall := fake.canPublishSourceArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) CanPublishSourceReturns(result1 bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = nil + fake.canPublishSourceReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) CanPublishSourceReturnsOnCall(i int, result1 bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = nil + if fake.canPublishSourceReturnsOnCall == nil { + fake.canPublishSourceReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canPublishSourceReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) CanSkipBroadcast() bool { + fake.canSkipBroadcastMutex.Lock() + ret, specificReturn := fake.canSkipBroadcastReturnsOnCall[len(fake.canSkipBroadcastArgsForCall)] + fake.canSkipBroadcastArgsForCall = append(fake.canSkipBroadcastArgsForCall, struct { + }{}) + stub := fake.CanSkipBroadcastStub + fakeReturns := fake.canSkipBroadcastReturns + fake.recordInvocation("CanSkipBroadcast", []interface{}{}) + fake.canSkipBroadcastMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastCallCount() int { + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() + return len(fake.canSkipBroadcastArgsForCall) +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastCalls(stub func() bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = stub +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastReturns(result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + fake.canSkipBroadcastReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastReturnsOnCall(i int, result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + if fake.canSkipBroadcastReturnsOnCall == nil { + fake.canSkipBroadcastReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canSkipBroadcastReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) CanSubscribe() bool { fake.canSubscribeMutex.Lock() ret, specificReturn := fake.canSubscribeReturnsOnCall[len(fake.canSubscribeArgsForCall)] @@ -1272,19 +1404,20 @@ func (fake *FakeLocalParticipant) ClaimGrantsReturnsOnCall(i int, result1 *auth. }{result1} } -func (fake *FakeLocalParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason) error { +func (fake *FakeLocalParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason, arg3 bool) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { arg1 bool arg2 types.ParticipantCloseReason - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.CloseStub fakeReturns := fake.closeReturns - fake.recordInvocation("Close", []interface{}{arg1, arg2}) + fake.recordInvocation("Close", []interface{}{arg1, arg2, arg3}) fake.closeMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -1298,17 +1431,17 @@ func (fake *FakeLocalParticipant) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeLocalParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason) error) { +func (fake *FakeLocalParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason, bool) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeLocalParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason) { +func (fake *FakeLocalParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason, bool) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalParticipant) CloseReturns(result1 error) { @@ -1334,15 +1467,16 @@ func (fake *FakeLocalParticipant) CloseReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeLocalParticipant) CloseSignalConnection() { +func (fake *FakeLocalParticipant) CloseSignalConnection(arg1 types.SignallingCloseReason) { fake.closeSignalConnectionMutex.Lock() fake.closeSignalConnectionArgsForCall = append(fake.closeSignalConnectionArgsForCall, struct { - }{}) + arg1 types.SignallingCloseReason + }{arg1}) stub := fake.CloseSignalConnectionStub - fake.recordInvocation("CloseSignalConnection", []interface{}{}) + fake.recordInvocation("CloseSignalConnection", []interface{}{arg1}) fake.closeSignalConnectionMutex.Unlock() if stub != nil { - fake.CloseSignalConnectionStub() + fake.CloseSignalConnectionStub(arg1) } } @@ -1352,12 +1486,19 @@ func (fake *FakeLocalParticipant) CloseSignalConnectionCallCount() int { return len(fake.closeSignalConnectionArgsForCall) } -func (fake *FakeLocalParticipant) CloseSignalConnectionCalls(stub func()) { +func (fake *FakeLocalParticipant) CloseSignalConnectionCalls(stub func(types.SignallingCloseReason)) { fake.closeSignalConnectionMutex.Lock() defer fake.closeSignalConnectionMutex.Unlock() fake.CloseSignalConnectionStub = stub } +func (fake *FakeLocalParticipant) CloseSignalConnectionArgsForCall(i int) types.SignallingCloseReason { + fake.closeSignalConnectionMutex.RLock() + defer fake.closeSignalConnectionMutex.RUnlock() + argsForCall := fake.closeSignalConnectionArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) ConnectedAt() time.Time { fake.connectedAtMutex.Lock() ret, specificReturn := fake.connectedAtReturnsOnCall[len(fake.connectedAtArgsForCall)] @@ -1743,6 +1884,59 @@ func (fake *FakeLocalParticipant) GetClientConfigurationReturnsOnCall(i int, res }{result1} } +func (fake *FakeLocalParticipant) GetClientInfo() *livekit.ClientInfo { + fake.getClientInfoMutex.Lock() + ret, specificReturn := fake.getClientInfoReturnsOnCall[len(fake.getClientInfoArgsForCall)] + fake.getClientInfoArgsForCall = append(fake.getClientInfoArgsForCall, struct { + }{}) + stub := fake.GetClientInfoStub + fakeReturns := fake.getClientInfoReturns + fake.recordInvocation("GetClientInfo", []interface{}{}) + fake.getClientInfoMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetClientInfoCallCount() int { + fake.getClientInfoMutex.RLock() + defer fake.getClientInfoMutex.RUnlock() + return len(fake.getClientInfoArgsForCall) +} + +func (fake *FakeLocalParticipant) GetClientInfoCalls(stub func() *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = stub +} + +func (fake *FakeLocalParticipant) GetClientInfoReturns(result1 *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = nil + fake.getClientInfoReturns = struct { + result1 *livekit.ClientInfo + }{result1} +} + +func (fake *FakeLocalParticipant) GetClientInfoReturnsOnCall(i int, result1 *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = nil + if fake.getClientInfoReturnsOnCall == nil { + fake.getClientInfoReturnsOnCall = make(map[int]struct { + result1 *livekit.ClientInfo + }) + } + fake.getClientInfoReturnsOnCall[i] = struct { + result1 *livekit.ClientInfo + }{result1} +} + func (fake *FakeLocalParticipant) GetConnectionQuality() *livekit.ConnectionQualityInfo { fake.getConnectionQualityMutex.Lock() ret, specificReturn := fake.getConnectionQualityReturnsOnCall[len(fake.getConnectionQualityArgsForCall)] @@ -1902,6 +2096,59 @@ func (fake *FakeLocalParticipant) GetLoggerReturnsOnCall(i int, result1 logger.L }{result1} } +func (fake *FakeLocalParticipant) GetPacer() pacer.Pacer { + fake.getPacerMutex.Lock() + ret, specificReturn := fake.getPacerReturnsOnCall[len(fake.getPacerArgsForCall)] + fake.getPacerArgsForCall = append(fake.getPacerArgsForCall, struct { + }{}) + stub := fake.GetPacerStub + fakeReturns := fake.getPacerReturns + fake.recordInvocation("GetPacer", []interface{}{}) + fake.getPacerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetPacerCallCount() int { + fake.getPacerMutex.RLock() + defer fake.getPacerMutex.RUnlock() + return len(fake.getPacerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetPacerCalls(stub func() pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = stub +} + +func (fake *FakeLocalParticipant) GetPacerReturns(result1 pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = nil + fake.getPacerReturns = struct { + result1 pacer.Pacer + }{result1} +} + +func (fake *FakeLocalParticipant) GetPacerReturnsOnCall(i int, result1 pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = nil + if fake.getPacerReturnsOnCall == nil { + fake.getPacerReturnsOnCall = make(map[int]struct { + result1 pacer.Pacer + }) + } + fake.getPacerReturnsOnCall[i] = struct { + result1 pacer.Pacer + }{result1} +} + func (fake *FakeLocalParticipant) GetPublishedTrack(arg1 livekit.TrackID) types.MediaTrack { fake.getPublishedTrackMutex.Lock() ret, specificReturn := fake.getPublishedTrackReturnsOnCall[len(fake.getPublishedTrackArgsForCall)] @@ -2122,6 +2369,59 @@ func (fake *FakeLocalParticipant) GetSubscribedTracksReturnsOnCall(i int, result }{result1} } +func (fake *FakeLocalParticipant) GetTrailer() []byte { + fake.getTrailerMutex.Lock() + ret, specificReturn := fake.getTrailerReturnsOnCall[len(fake.getTrailerArgsForCall)] + fake.getTrailerArgsForCall = append(fake.getTrailerArgsForCall, struct { + }{}) + stub := fake.GetTrailerStub + fakeReturns := fake.getTrailerReturns + fake.recordInvocation("GetTrailer", []interface{}{}) + fake.getTrailerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetTrailerCallCount() int { + fake.getTrailerMutex.RLock() + defer fake.getTrailerMutex.RUnlock() + return len(fake.getTrailerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetTrailerCalls(stub func() []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = stub +} + +func (fake *FakeLocalParticipant) GetTrailerReturns(result1 []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = nil + fake.getTrailerReturns = struct { + result1 []byte + }{result1} +} + +func (fake *FakeLocalParticipant) GetTrailerReturnsOnCall(i int, result1 []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = nil + if fake.getTrailerReturnsOnCall == nil { + fake.getTrailerReturnsOnCall = make(map[int]struct { + result1 []byte + }) + } + fake.getTrailerReturnsOnCall[i] = struct { + result1 []byte + }{result1} +} + func (fake *FakeLocalParticipant) HandleAnswer(arg1 webrtc.SessionDescription) { fake.handleAnswerMutex.Lock() fake.handleAnswerArgsForCall = append(fake.handleAnswerArgsForCall, struct { @@ -2186,6 +2486,92 @@ func (fake *FakeLocalParticipant) HandleOfferArgsForCall(i int) webrtc.SessionDe return argsForCall.arg1 } +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponse(arg1 livekit.ReconnectReason, arg2 *livekit.ReconnectResponse) error { + fake.handleReconnectAndSendResponseMutex.Lock() + ret, specificReturn := fake.handleReconnectAndSendResponseReturnsOnCall[len(fake.handleReconnectAndSendResponseArgsForCall)] + fake.handleReconnectAndSendResponseArgsForCall = append(fake.handleReconnectAndSendResponseArgsForCall, struct { + arg1 livekit.ReconnectReason + arg2 *livekit.ReconnectResponse + }{arg1, arg2}) + stub := fake.HandleReconnectAndSendResponseStub + fakeReturns := fake.handleReconnectAndSendResponseReturns + fake.recordInvocation("HandleReconnectAndSendResponse", []interface{}{arg1, arg2}) + fake.handleReconnectAndSendResponseMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseCallCount() int { + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() + return len(fake.handleReconnectAndSendResponseArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseCalls(stub func(livekit.ReconnectReason, *livekit.ReconnectResponse) error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = stub +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseArgsForCall(i int) (livekit.ReconnectReason, *livekit.ReconnectResponse) { + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() + argsForCall := fake.handleReconnectAndSendResponseArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturns(result1 error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = nil + fake.handleReconnectAndSendResponseReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturnsOnCall(i int, result1 error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = nil + if fake.handleReconnectAndSendResponseReturnsOnCall == nil { + fake.handleReconnectAndSendResponseReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleReconnectAndSendResponseReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleSignalSourceClose() { + fake.handleSignalSourceCloseMutex.Lock() + fake.handleSignalSourceCloseArgsForCall = append(fake.handleSignalSourceCloseArgsForCall, struct { + }{}) + stub := fake.HandleSignalSourceCloseStub + fake.recordInvocation("HandleSignalSourceClose", []interface{}{}) + fake.handleSignalSourceCloseMutex.Unlock() + if stub != nil { + fake.HandleSignalSourceCloseStub() + } +} + +func (fake *FakeLocalParticipant) HandleSignalSourceCloseCallCount() int { + fake.handleSignalSourceCloseMutex.RLock() + defer fake.handleSignalSourceCloseMutex.RUnlock() + return len(fake.handleSignalSourceCloseArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleSignalSourceCloseCalls(stub func()) { + fake.handleSignalSourceCloseMutex.Lock() + defer fake.handleSignalSourceCloseMutex.Unlock() + fake.HandleSignalSourceCloseStub = stub +} + func (fake *FakeLocalParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] @@ -2301,17 +2687,16 @@ func (fake *FakeLocalParticipant) HiddenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeLocalParticipant) ICERestart(arg1 *livekit.ICEConfig, arg2 livekit.ReconnectReason) { +func (fake *FakeLocalParticipant) ICERestart(arg1 *livekit.ICEConfig) { fake.iCERestartMutex.Lock() fake.iCERestartArgsForCall = append(fake.iCERestartArgsForCall, struct { arg1 *livekit.ICEConfig - arg2 livekit.ReconnectReason - }{arg1, arg2}) + }{arg1}) stub := fake.ICERestartStub - fake.recordInvocation("ICERestart", []interface{}{arg1, arg2}) + fake.recordInvocation("ICERestart", []interface{}{arg1}) fake.iCERestartMutex.Unlock() if stub != nil { - fake.ICERestartStub(arg1, arg2) + fake.ICERestartStub(arg1) } } @@ -2321,17 +2706,17 @@ func (fake *FakeLocalParticipant) ICERestartCallCount() int { return len(fake.iCERestartArgsForCall) } -func (fake *FakeLocalParticipant) ICERestartCalls(stub func(*livekit.ICEConfig, livekit.ReconnectReason)) { +func (fake *FakeLocalParticipant) ICERestartCalls(stub func(*livekit.ICEConfig)) { fake.iCERestartMutex.Lock() defer fake.iCERestartMutex.Unlock() fake.ICERestartStub = stub } -func (fake *FakeLocalParticipant) ICERestartArgsForCall(i int) (*livekit.ICEConfig, livekit.ReconnectReason) { +func (fake *FakeLocalParticipant) ICERestartArgsForCall(i int) *livekit.ICEConfig { fake.iCERestartMutex.RLock() defer fake.iCERestartMutex.RUnlock() argsForCall := fake.iCERestartArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1 } func (fake *FakeLocalParticipant) ID() livekit.ParticipantID { @@ -3030,10 +3415,10 @@ func (fake *FakeLocalParticipant) OnClaimsChangedArgsForCall(i int) func(types.L return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnClose(arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) { +func (fake *FakeLocalParticipant) OnClose(arg1 func(types.LocalParticipant)) { fake.onCloseMutex.Lock() fake.onCloseArgsForCall = append(fake.onCloseArgsForCall, struct { - arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + arg1 func(types.LocalParticipant) }{arg1}) stub := fake.OnCloseStub fake.recordInvocation("OnClose", []interface{}{arg1}) @@ -3049,13 +3434,13 @@ func (fake *FakeLocalParticipant) OnCloseCallCount() int { return len(fake.onCloseArgsForCall) } -func (fake *FakeLocalParticipant) OnCloseCalls(stub func(func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID))) { +func (fake *FakeLocalParticipant) OnCloseCalls(stub func(func(types.LocalParticipant))) { fake.onCloseMutex.Lock() defer fake.onCloseMutex.Unlock() fake.OnCloseStub = stub } -func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) { +func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalParticipant) { fake.onCloseMutex.RLock() defer fake.onCloseMutex.RUnlock() argsForCall := fake.onCloseArgsForCall[i] @@ -3786,67 +4171,6 @@ func (fake *FakeLocalParticipant) SendParticipantUpdateReturnsOnCall(i int, resu }{result1} } -func (fake *FakeLocalParticipant) SendReconnectResponse(arg1 *livekit.ReconnectResponse) error { - fake.sendReconnectResponseMutex.Lock() - ret, specificReturn := fake.sendReconnectResponseReturnsOnCall[len(fake.sendReconnectResponseArgsForCall)] - fake.sendReconnectResponseArgsForCall = append(fake.sendReconnectResponseArgsForCall, struct { - arg1 *livekit.ReconnectResponse - }{arg1}) - stub := fake.SendReconnectResponseStub - fakeReturns := fake.sendReconnectResponseReturns - fake.recordInvocation("SendReconnectResponse", []interface{}{arg1}) - fake.sendReconnectResponseMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) SendReconnectResponseCallCount() int { - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() - return len(fake.sendReconnectResponseArgsForCall) -} - -func (fake *FakeLocalParticipant) SendReconnectResponseCalls(stub func(*livekit.ReconnectResponse) error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = stub -} - -func (fake *FakeLocalParticipant) SendReconnectResponseArgsForCall(i int) *livekit.ReconnectResponse { - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() - argsForCall := fake.sendReconnectResponseArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) SendReconnectResponseReturns(result1 error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = nil - fake.sendReconnectResponseReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeLocalParticipant) SendReconnectResponseReturnsOnCall(i int, result1 error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = nil - if fake.sendReconnectResponseReturnsOnCall == nil { - fake.sendReconnectResponseReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.sendReconnectResponseReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeLocalParticipant) SendRefreshToken(arg1 string) error { fake.sendRefreshTokenMutex.Lock() ret, specificReturn := fake.sendRefreshTokenReturnsOnCall[len(fake.sendRefreshTokenArgsForCall)] @@ -4334,6 +4658,70 @@ func (fake *FakeLocalParticipant) SetSignalSourceValidArgsForCall(i int) bool { return argsForCall.arg1 } +func (fake *FakeLocalParticipant) SetSubscriberAllowPause(arg1 bool) { + fake.setSubscriberAllowPauseMutex.Lock() + fake.setSubscriberAllowPauseArgsForCall = append(fake.setSubscriberAllowPauseArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.SetSubscriberAllowPauseStub + fake.recordInvocation("SetSubscriberAllowPause", []interface{}{arg1}) + fake.setSubscriberAllowPauseMutex.Unlock() + if stub != nil { + fake.SetSubscriberAllowPauseStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseCallCount() int { + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + return len(fake.setSubscriberAllowPauseArgsForCall) +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseCalls(stub func(bool)) { + fake.setSubscriberAllowPauseMutex.Lock() + defer fake.setSubscriberAllowPauseMutex.Unlock() + fake.SetSubscriberAllowPauseStub = stub +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseArgsForCall(i int) bool { + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + argsForCall := fake.setSubscriberAllowPauseArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacity(arg1 int64) { + fake.setSubscriberChannelCapacityMutex.Lock() + fake.setSubscriberChannelCapacityArgsForCall = append(fake.setSubscriberChannelCapacityArgsForCall, struct { + arg1 int64 + }{arg1}) + stub := fake.SetSubscriberChannelCapacityStub + fake.recordInvocation("SetSubscriberChannelCapacity", []interface{}{arg1}) + fake.setSubscriberChannelCapacityMutex.Unlock() + if stub != nil { + fake.SetSubscriberChannelCapacityStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityCallCount() int { + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() + return len(fake.setSubscriberChannelCapacityArgsForCall) +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityCalls(stub func(int64)) { + fake.setSubscriberChannelCapacityMutex.Lock() + defer fake.setSubscriberChannelCapacityMutex.Unlock() + fake.SetSubscriberChannelCapacityStub = stub +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityArgsForCall(i int) int64 { + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() + argsForCall := fake.setSubscriberChannelCapacityArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) SetTrackMuted(arg1 livekit.TrackID, arg2 bool, arg3 bool) { fake.setTrackMutedMutex.Lock() fake.setTrackMutedArgsForCall = append(fake.setTrackMutedArgsForCall, struct { @@ -4530,7 +4918,7 @@ func (fake *FakeLocalParticipant) SubscriberAsPrimaryReturnsOnCall(i int, result }{result1} } -func (fake *FakeLocalParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() ret, specificReturn := fake.subscriptionPermissionReturnsOnCall[len(fake.subscriptionPermissionArgsForCall)] fake.subscriptionPermissionArgsForCall = append(fake.subscriptionPermissionArgsForCall, struct { @@ -4554,35 +4942,35 @@ func (fake *FakeLocalParticipant) SubscriptionPermissionCallCount() int { return len(fake.subscriptionPermissionArgsForCall) } -func (fake *FakeLocalParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion)) { +func (fake *FakeLocalParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, utils.TimedVersion)) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = stub } -func (fake *FakeLocalParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil fake.subscriptionPermissionReturns = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } -func (fake *FakeLocalParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil if fake.subscriptionPermissionReturnsOnCall == nil { fake.subscriptionPermissionReturnsOnCall = make(map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }) } fake.subscriptionPermissionReturnsOnCall[i] = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } @@ -4673,6 +5061,62 @@ func (fake *FakeLocalParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.P }{result1} } +func (fake *FakeLocalParticipant) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + ret, specificReturn := fake.toProtoWithVersionReturnsOnCall[len(fake.toProtoWithVersionArgsForCall)] + fake.toProtoWithVersionArgsForCall = append(fake.toProtoWithVersionArgsForCall, struct { + }{}) + stub := fake.ToProtoWithVersionStub + fakeReturns := fake.toProtoWithVersionReturns + fake.recordInvocation("ToProtoWithVersion", []interface{}{}) + fake.toProtoWithVersionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionCallCount() int { + fake.toProtoWithVersionMutex.RLock() + defer fake.toProtoWithVersionMutex.RUnlock() + return len(fake.toProtoWithVersionArgsForCall) +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionCalls(stub func() (*livekit.ParticipantInfo, utils.TimedVersion)) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = stub +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionReturns(result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + fake.toProtoWithVersionReturns = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionReturnsOnCall(i int, result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + if fake.toProtoWithVersionReturnsOnCall == nil { + fake.toProtoWithVersionReturnsOnCall = make(map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }) + } + fake.toProtoWithVersionReturnsOnCall[i] = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + func (fake *FakeLocalParticipant) UncacheDownTrack(arg1 *webrtc.RTPTransceiver) { fake.uncacheDownTrackMutex.Lock() fake.uncacheDownTrackArgsForCall = append(fake.uncacheDownTrackArgsForCall, struct { @@ -4989,12 +5433,12 @@ func (fake *FakeLocalParticipant) UpdateSubscribedTrackSettingsArgsForCall(i int return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 *livekit.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant }{arg1, arg2, arg3, arg4}) @@ -5017,13 +5461,13 @@ func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] @@ -5221,10 +5665,12 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.addTransceiverFromTrackToSubscriberMutex.RUnlock() fake.cacheDownTrackMutex.RLock() defer fake.cacheDownTrackMutex.RUnlock() - fake.canPublishMutex.RLock() - defer fake.canPublishMutex.RUnlock() fake.canPublishDataMutex.RLock() defer fake.canPublishDataMutex.RUnlock() + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() fake.canSubscribeMutex.RLock() defer fake.canSubscribeMutex.RUnlock() fake.claimGrantsMutex.RLock() @@ -5247,12 +5693,16 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getCachedDownTrackMutex.RUnlock() fake.getClientConfigurationMutex.RLock() defer fake.getClientConfigurationMutex.RUnlock() + fake.getClientInfoMutex.RLock() + defer fake.getClientInfoMutex.RUnlock() fake.getConnectionQualityMutex.RLock() defer fake.getConnectionQualityMutex.RUnlock() fake.getICEConnectionTypeMutex.RLock() defer fake.getICEConnectionTypeMutex.RUnlock() fake.getLoggerMutex.RLock() defer fake.getLoggerMutex.RUnlock() + fake.getPacerMutex.RLock() + defer fake.getPacerMutex.RUnlock() fake.getPublishedTrackMutex.RLock() defer fake.getPublishedTrackMutex.RUnlock() fake.getPublishedTracksMutex.RLock() @@ -5261,10 +5711,16 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getSubscribedParticipantsMutex.RUnlock() fake.getSubscribedTracksMutex.RLock() defer fake.getSubscribedTracksMutex.RUnlock() + fake.getTrailerMutex.RLock() + defer fake.getTrailerMutex.RUnlock() fake.handleAnswerMutex.RLock() defer fake.handleAnswerMutex.RUnlock() fake.handleOfferMutex.RLock() defer fake.handleOfferMutex.RUnlock() + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() + fake.handleSignalSourceCloseMutex.RLock() + defer fake.handleSignalSourceCloseMutex.RUnlock() fake.hasPermissionMutex.RLock() defer fake.hasPermissionMutex.RUnlock() fake.hiddenMutex.RLock() @@ -5335,8 +5791,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.sendJoinResponseMutex.RUnlock() fake.sendParticipantUpdateMutex.RLock() defer fake.sendParticipantUpdateMutex.RUnlock() - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() fake.sendRefreshTokenMutex.RLock() defer fake.sendRefreshTokenMutex.RUnlock() fake.sendRoomUpdateMutex.RLock() @@ -5359,6 +5813,10 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.setResponseSinkMutex.RUnlock() fake.setSignalSourceValidMutex.RLock() defer fake.setSignalSourceValidMutex.RUnlock() + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() fake.setTrackMutedMutex.RLock() defer fake.setTrackMutedMutex.RUnlock() fake.startMutex.RLock() @@ -5375,6 +5833,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.subscriptionPermissionUpdateMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.toProtoWithVersionMutex.RLock() + defer fake.toProtoWithVersionMutex.RUnlock() fake.uncacheDownTrackMutex.RLock() defer fake.uncacheDownTrackMutex.RUnlock() fake.unsubscribeFromTrackMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index de0e870f4..bb0c07e31 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -93,6 +93,16 @@ type FakeMediaTrack struct { iDReturnsOnCall map[int]struct { result1 livekit.TrackID } + IsEncryptedStub func() bool + isEncryptedMutex sync.RWMutex + isEncryptedArgsForCall []struct { + } + isEncryptedReturns struct { + result1 bool + } + isEncryptedReturnsOnCall map[int]struct { + result1 bool + } IsMutedStub func() bool isMutedMutex sync.RWMutex isMutedArgsForCall []struct { @@ -689,6 +699,59 @@ func (fake *FakeMediaTrack) IDReturnsOnCall(i int, result1 livekit.TrackID) { }{result1} } +func (fake *FakeMediaTrack) IsEncrypted() bool { + fake.isEncryptedMutex.Lock() + ret, specificReturn := fake.isEncryptedReturnsOnCall[len(fake.isEncryptedArgsForCall)] + fake.isEncryptedArgsForCall = append(fake.isEncryptedArgsForCall, struct { + }{}) + stub := fake.IsEncryptedStub + fakeReturns := fake.isEncryptedReturns + fake.recordInvocation("IsEncrypted", []interface{}{}) + fake.isEncryptedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMediaTrack) IsEncryptedCallCount() int { + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() + return len(fake.isEncryptedArgsForCall) +} + +func (fake *FakeMediaTrack) IsEncryptedCalls(stub func() bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = stub +} + +func (fake *FakeMediaTrack) IsEncryptedReturns(result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + fake.isEncryptedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeMediaTrack) IsEncryptedReturnsOnCall(i int, result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + if fake.isEncryptedReturnsOnCall == nil { + fake.isEncryptedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isEncryptedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeMediaTrack) IsMuted() bool { fake.isMutedMutex.Lock() ret, specificReturn := fake.isMutedReturnsOnCall[len(fake.isMutedArgsForCall)] @@ -1522,6 +1585,8 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() fake.isMutedMutex.RLock() defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index efacc7a5b..a660644cb 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -6,14 +6,26 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" ) type FakeParticipant struct { - CloseStub func(bool, types.ParticipantCloseReason) error + CanSkipBroadcastStub func() bool + canSkipBroadcastMutex sync.RWMutex + canSkipBroadcastArgsForCall []struct { + } + canSkipBroadcastReturns struct { + result1 bool + } + canSkipBroadcastReturnsOnCall map[int]struct { + result1 bool + } + CloseStub func(bool, types.ParticipantCloseReason, bool) error closeMutex sync.RWMutex closeArgsForCall []struct { arg1 bool arg2 types.ParticipantCloseReason + arg3 bool } closeReturns struct { result1 error @@ -94,6 +106,16 @@ type FakeParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } + IsPublisherStub func() bool + isPublisherMutex sync.RWMutex + isPublisherArgsForCall []struct { + } + isPublisherReturns struct { + result1 bool + } + isPublisherReturnsOnCall map[int]struct { + result1 bool + } IsRecorderStub func() bool isRecorderMutex sync.RWMutex isRecorderArgsForCall []struct { @@ -135,17 +157,17 @@ type FakeParticipant struct { stateReturnsOnCall map[int]struct { result1 livekit.ParticipantInfo_State } - SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, utils.TimedVersion) subscriptionPermissionMutex sync.RWMutex subscriptionPermissionArgsForCall []struct { } subscriptionPermissionReturns struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } subscriptionPermissionReturnsOnCall map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } ToProtoStub func() *livekit.ParticipantInfo toProtoMutex sync.RWMutex @@ -157,11 +179,11 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant } @@ -186,19 +208,73 @@ type FakeParticipant struct { invocationsMutex sync.RWMutex } -func (fake *FakeParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason) error { +func (fake *FakeParticipant) CanSkipBroadcast() bool { + fake.canSkipBroadcastMutex.Lock() + ret, specificReturn := fake.canSkipBroadcastReturnsOnCall[len(fake.canSkipBroadcastArgsForCall)] + fake.canSkipBroadcastArgsForCall = append(fake.canSkipBroadcastArgsForCall, struct { + }{}) + stub := fake.CanSkipBroadcastStub + fakeReturns := fake.canSkipBroadcastReturns + fake.recordInvocation("CanSkipBroadcast", []interface{}{}) + fake.canSkipBroadcastMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) CanSkipBroadcastCallCount() int { + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() + return len(fake.canSkipBroadcastArgsForCall) +} + +func (fake *FakeParticipant) CanSkipBroadcastCalls(stub func() bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = stub +} + +func (fake *FakeParticipant) CanSkipBroadcastReturns(result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + fake.canSkipBroadcastReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) CanSkipBroadcastReturnsOnCall(i int, result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + if fake.canSkipBroadcastReturnsOnCall == nil { + fake.canSkipBroadcastReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canSkipBroadcastReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason, arg3 bool) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { arg1 bool arg2 types.ParticipantCloseReason - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.CloseStub fakeReturns := fake.closeReturns - fake.recordInvocation("Close", []interface{}{arg1, arg2}) + fake.recordInvocation("Close", []interface{}{arg1, arg2, arg3}) fake.closeMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -212,17 +288,17 @@ func (fake *FakeParticipant) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason) error) { +func (fake *FakeParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason, bool) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason) { +func (fake *FakeParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason, bool) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeParticipant) CloseReturns(result1 error) { @@ -636,6 +712,59 @@ func (fake *FakeParticipant) IdentityReturnsOnCall(i int, result1 livekit.Partic }{result1} } +func (fake *FakeParticipant) IsPublisher() bool { + fake.isPublisherMutex.Lock() + ret, specificReturn := fake.isPublisherReturnsOnCall[len(fake.isPublisherArgsForCall)] + fake.isPublisherArgsForCall = append(fake.isPublisherArgsForCall, struct { + }{}) + stub := fake.IsPublisherStub + fakeReturns := fake.isPublisherReturns + fake.recordInvocation("IsPublisher", []interface{}{}) + fake.isPublisherMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) IsPublisherCallCount() int { + fake.isPublisherMutex.RLock() + defer fake.isPublisherMutex.RUnlock() + return len(fake.isPublisherArgsForCall) +} + +func (fake *FakeParticipant) IsPublisherCalls(stub func() bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = stub +} + +func (fake *FakeParticipant) IsPublisherReturns(result1 bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = nil + fake.isPublisherReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) IsPublisherReturnsOnCall(i int, result1 bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = nil + if fake.isPublisherReturnsOnCall == nil { + fake.isPublisherReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isPublisherReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) IsRecorder() bool { fake.isRecorderMutex.Lock() ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] @@ -864,7 +993,7 @@ func (fake *FakeParticipant) StateReturnsOnCall(i int, result1 livekit.Participa }{result1} } -func (fake *FakeParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() ret, specificReturn := fake.subscriptionPermissionReturnsOnCall[len(fake.subscriptionPermissionArgsForCall)] fake.subscriptionPermissionArgsForCall = append(fake.subscriptionPermissionArgsForCall, struct { @@ -888,35 +1017,35 @@ func (fake *FakeParticipant) SubscriptionPermissionCallCount() int { return len(fake.subscriptionPermissionArgsForCall) } -func (fake *FakeParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion)) { +func (fake *FakeParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, utils.TimedVersion)) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = stub } -func (fake *FakeParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil fake.subscriptionPermissionReturns = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } -func (fake *FakeParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil if fake.subscriptionPermissionReturnsOnCall == nil { fake.subscriptionPermissionReturnsOnCall = make(map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }) } fake.subscriptionPermissionReturnsOnCall[i] = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } @@ -973,12 +1102,12 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } -func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 *livekit.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant }{arg1, arg2, arg3, arg4}) @@ -1001,13 +1130,13 @@ func (fake *FakeParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] @@ -1101,6 +1230,8 @@ func (fake *FakeParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error func (fake *FakeParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.debugInfoMutex.RLock() @@ -1117,6 +1248,8 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() + fake.isPublisherMutex.RLock() + defer fake.isPublisherMutex.RUnlock() fake.isRecorderMutex.RLock() defer fake.isRecorderMutex.RUnlock() fake.removePublishedTrackMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_room.go b/pkg/rtc/types/typesfakes/fake_room.go index 84449cf56..67d68e8ca 100644 --- a/pkg/rtc/types/typesfakes/fake_room.go +++ b/pkg/rtc/types/typesfakes/fake_room.go @@ -82,6 +82,13 @@ type FakeRoom struct { syncStateReturnsOnCall map[int]struct { result1 error } + UpdateParticipantMetadataStub func(types.LocalParticipant, string, string) + updateParticipantMetadataMutex sync.RWMutex + updateParticipantMetadataArgsForCall []struct { + arg1 types.LocalParticipant + arg2 string + arg3 string + } UpdateSubscriptionPermissionStub func(types.LocalParticipant, *livekit.SubscriptionPermission) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { @@ -497,6 +504,40 @@ func (fake *FakeRoom) SyncStateReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeRoom) UpdateParticipantMetadata(arg1 types.LocalParticipant, arg2 string, arg3 string) { + fake.updateParticipantMetadataMutex.Lock() + fake.updateParticipantMetadataArgsForCall = append(fake.updateParticipantMetadataArgsForCall, struct { + arg1 types.LocalParticipant + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.UpdateParticipantMetadataStub + fake.recordInvocation("UpdateParticipantMetadata", []interface{}{arg1, arg2, arg3}) + fake.updateParticipantMetadataMutex.Unlock() + if stub != nil { + fake.UpdateParticipantMetadataStub(arg1, arg2, arg3) + } +} + +func (fake *FakeRoom) UpdateParticipantMetadataCallCount() int { + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() + return len(fake.updateParticipantMetadataArgsForCall) +} + +func (fake *FakeRoom) UpdateParticipantMetadataCalls(stub func(types.LocalParticipant, string, string)) { + fake.updateParticipantMetadataMutex.Lock() + defer fake.updateParticipantMetadataMutex.Unlock() + fake.UpdateParticipantMetadataStub = stub +} + +func (fake *FakeRoom) UpdateParticipantMetadataArgsForCall(i int) (types.LocalParticipant, string, string) { + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() + argsForCall := fake.updateParticipantMetadataArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + func (fake *FakeRoom) UpdateSubscriptionPermission(arg1 types.LocalParticipant, arg2 *livekit.SubscriptionPermission) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] @@ -683,6 +724,8 @@ func (fake *FakeRoom) Invocations() map[string][][]interface{} { defer fake.simulateScenarioMutex.RUnlock() fake.syncStateMutex.RLock() defer fake.syncStateMutex.RUnlock() + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateSubscriptionsMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_subscribed_track.go b/pkg/rtc/types/typesfakes/fake_subscribed_track.go index b14ceb2c4..833b8e4b3 100644 --- a/pkg/rtc/types/typesfakes/fake_subscribed_track.go +++ b/pkg/rtc/types/typesfakes/fake_subscribed_track.go @@ -11,10 +11,10 @@ import ( ) type FakeSubscribedTrack struct { - AddOnBindStub func(func()) + AddOnBindStub func(func(error)) addOnBindMutex sync.RWMutex addOnBindArgsForCall []struct { - arg1 func() + arg1 func(error) } CloseStub func(bool) closeMutex sync.RWMutex @@ -174,10 +174,10 @@ type FakeSubscribedTrack struct { invocationsMutex sync.RWMutex } -func (fake *FakeSubscribedTrack) AddOnBind(arg1 func()) { +func (fake *FakeSubscribedTrack) AddOnBind(arg1 func(error)) { fake.addOnBindMutex.Lock() fake.addOnBindArgsForCall = append(fake.addOnBindArgsForCall, struct { - arg1 func() + arg1 func(error) }{arg1}) stub := fake.AddOnBindStub fake.recordInvocation("AddOnBind", []interface{}{arg1}) @@ -193,13 +193,13 @@ func (fake *FakeSubscribedTrack) AddOnBindCallCount() int { return len(fake.addOnBindArgsForCall) } -func (fake *FakeSubscribedTrack) AddOnBindCalls(stub func(func())) { +func (fake *FakeSubscribedTrack) AddOnBindCalls(stub func(func(error))) { fake.addOnBindMutex.Lock() defer fake.addOnBindMutex.Unlock() fake.AddOnBindStub = stub } -func (fake *FakeSubscribedTrack) AddOnBindArgsForCall(i int) func() { +func (fake *FakeSubscribedTrack) AddOnBindArgsForCall(i int) func(error) { fake.addOnBindMutex.RLock() defer fake.addOnBindMutex.RUnlock() argsForCall := fake.addOnBindArgsForCall[i] diff --git a/pkg/rtc/unhandlesimulcast.go b/pkg/rtc/unhandlesimulcast.go index 0fd611443..568c7dc1b 100644 --- a/pkg/rtc/unhandlesimulcast.go +++ b/pkg/rtc/unhandlesimulcast.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 93309a83a..961134c83 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -30,7 +44,7 @@ type UpTrackManager struct { // publishedTracks that participant is publishing publishedTracks map[livekit.TrackID]types.MediaTrack subscriptionPermission *livekit.SubscriptionPermission - subscriptionPermissionVersion *utils.TimedVersion + subscriptionPermissionVersion utils.TimedVersion // subscriber permission for published tracks subscriberPermissions map[livekit.ParticipantIdentity]*livekit.TrackPermission // subscriberIdentity => *livekit.TrackPermission @@ -127,47 +141,36 @@ func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack { func (u *UpTrackManager) UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, - timedVersion *livekit.TimedVersion, + timedVersion utils.TimedVersion, resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, resolverBySid func(participantID livekit.ParticipantID) types.LocalParticipant, ) error { u.lock.Lock() - if timedVersion != nil { + if !timedVersion.IsZero() { // it's possible for permission updates to come from another node. In that case // they would be the authority for this participant's permissions // we do not want to initialize subscriptionPermissionVersion too early since if another machine is the // owner for the data, we'd prefer to use their TimedVersion - if u.subscriptionPermissionVersion != nil { - tv := utils.NewTimedVersionFromProto(timedVersion) - // ignore older version - if !tv.After(u.subscriptionPermissionVersion) { - perms := "" - if u.subscriptionPermission != nil { - perms = u.subscriptionPermission.String() - } - u.params.Logger.Infow( - "skipping older subscription permission version", - "existingValue", perms, - "existingVersion", u.subscriptionPermissionVersion.ToProto().String(), - "requestingValue", subscriptionPermission.String(), - "requestingVersion", timedVersion.String(), - ) - u.lock.Unlock() - return nil + // ignore older version + if !timedVersion.After(&u.subscriptionPermissionVersion) { + perms := "" + if u.subscriptionPermission != nil { + perms = u.subscriptionPermission.String() } - u.subscriptionPermissionVersion.Update(tv) - } else { - u.subscriptionPermissionVersion = utils.NewTimedVersionFromProto(timedVersion) + u.params.Logger.Debugw( + "skipping older subscription permission version", + "existingValue", perms, + "existingVersion", u.subscriptionPermissionVersion.ToProto().String(), + "requestingValue", subscriptionPermission.String(), + "requestingVersion", timedVersion.String(), + ) + u.lock.Unlock() + return nil } + u.subscriptionPermissionVersion.Update(&timedVersion) } else { // for requests coming from the current node, use local versions - tv := u.params.VersionGenerator.New() - // use current time as the new/updated version - if u.subscriptionPermissionVersion == nil { - u.subscriptionPermissionVersion = tv - } else { - u.subscriptionPermissionVersion.Update(tv) - } + u.subscriptionPermissionVersion.Update(u.params.VersionGenerator.New()) } // store as is for use when migrating @@ -205,15 +208,15 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( return nil } -func (u *UpTrackManager) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (u *UpTrackManager) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { u.lock.RLock() defer u.lock.RUnlock() - if u.subscriptionPermissionVersion == nil { - return nil, nil + if u.subscriptionPermissionVersion.IsZero() { + return nil, u.subscriptionPermissionVersion.Load() } - return u.subscriptionPermission, u.subscriptionPermissionVersion.ToProto() + return u.subscriptionPermission, u.subscriptionPermissionVersion.Load() } func (u *UpTrackManager) HasPermission(trackID livekit.TrackID, subIdentity livekit.ParticipantIdentity) bool { diff --git a/pkg/rtc/uptrackmanager_test.go b/pkg/rtc/uptrackmanager_test.go index 88e4c2ee7..46a8c96cf 100644 --- a/pkg/rtc/uptrackmanager_test.go +++ b/pkg/rtc/uptrackmanager_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -21,6 +35,7 @@ var defaultUptrackManagerParams = UpTrackManagerParams{ func TestUpdateSubscriptionPermission(t *testing.T) { t.Run("updates subscription permission", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -34,14 +49,14 @@ func TestUpdateSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.Nil(t, um.subscriberPermissions) // nobody is allowed to subscribe subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.NotNil(t, um.subscriberPermissions) require.Equal(t, 0, len(um.subscriberPermissions)) @@ -77,7 +92,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, sidResolver) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -102,7 +117,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms3, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.Equal(t, 3, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -111,6 +126,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { t.Run("updates subscription permission using both", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -154,7 +170,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - err := um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, sidResolver) + err := um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) @@ -173,17 +189,37 @@ func TestUpdateSubscriptionPermission(t *testing.T) { return nil } - err = um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, badSidResolver) + err = um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, badSidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) }) + + t.Run("update versions", func(t *testing.T) { + um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() + + v0, v1, v2 := vg.Next(), vg.Next(), vg.Next() + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v1, nil, nil) + require.Equal(t, v1.Load(), um.subscriptionPermissionVersion.Load(), "first update should be applied") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v2, nil, nil) + require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "ordered updates should be applied") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil, nil) + require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "out of order updates should be ignored") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion{}, nil, nil) + require.True(t, um.subscriptionPermissionVersion.After(&v2), "zero version in updates should use next local version") + }) } func TestSubscriptionPermission(t *testing.T) { t.Run("checks subscription permission", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -197,7 +233,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -205,7 +241,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.False(t, um.hasPermissionLocked("audio", "p1")) require.False(t, um.hasPermissionLocked("audio", "p2")) @@ -222,7 +258,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -257,7 +293,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("screen", "p1")) diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index 36d7cfc7b..b9b532d8b 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -10,8 +24,6 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/rtc/types" ) const ( @@ -45,14 +57,6 @@ func UnpackDataTrackLabel(packed string) (participantID livekit.ParticipantID, t return } -func ToProtoParticipants(participants []types.LocalParticipant) []*livekit.ParticipantInfo { - infos := make([]*livekit.ParticipantInfo, 0, len(participants)) - for _, op := range participants { - infos = append(infos, op.ToProto()) - } - return infos -} - func ToProtoSessionDescription(sd webrtc.SessionDescription) *livekit.SessionDescription { return &livekit.SessionDescription{ Type: sd.Type.String(), @@ -140,7 +144,7 @@ func LoggerWithParticipant(l logger.Logger, identity livekit.ParticipantIdentity } values = append(values, "remote", isRemote) // enable sampling per participant - return l.WithItemSampler().WithValues(values...) + return l.WithValues(values...) } func LoggerWithRoom(l logger.Logger, name livekit.RoomName, roomID livekit.RoomID) logger.Logger { diff --git a/pkg/rtc/utils_test.go b/pkg/rtc/utils_test.go index 9e13e8fe2..813f910d8 100644 --- a/pkg/rtc/utils_test.go +++ b/pkg/rtc/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 2f9a60490..69c4e391c 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( @@ -12,7 +26,6 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -103,6 +116,14 @@ func (r *WrappedReceiver) Codecs() []webrtc.RTPCodecParameters { return codecs } +func (r *WrappedReceiver) DeleteDownTrack(participantID livekit.ParticipantID) { + if r.TrackReceiver != nil { + r.TrackReceiver.DeleteDownTrack(participantID) + } +} + +// -------------------------------------------- + type DummyReceiver struct { receiver atomic.Value trackID livekit.TrackID @@ -296,11 +317,11 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { +func (d *DummyReceiver) GetCalculatedClockRate(layer int32) uint32 { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetRTCPSenderReportDataExt(layer) + return r.GetCalculatedClockRate(layer) } - return nil + return 0 } func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { diff --git a/pkg/service/auth.go b/pkg/service/auth.go index c83cfbb1a..af940e83a 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/auth_test.go b/pkg/service/auth_test.go index f61d9fbc0..8a70d6a18 100644 --- a/pkg/service/auth_test.go +++ b/pkg/service/auth_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/egress.go b/pkg/service/egress.go index abafa7162..1ae4b5a69 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -10,7 +24,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -18,42 +31,37 @@ import ( ) type EgressService struct { - psrpcClient rpc.EgressClient - clientDeprecated egress.RPCClient - store ServiceStore - es EgressStore - roomService livekit.RoomService - telemetry telemetry.TelemetryService - launcher rtc.EgressLauncher + client rpc.EgressClient + store ServiceStore + es EgressStore + roomService livekit.RoomService + telemetry telemetry.TelemetryService + launcher rtc.EgressLauncher } type egressLauncher struct { - psrpcClient rpc.EgressClient - clientDeprecated egress.RPCClient - es EgressStore - telemetry telemetry.TelemetryService + client rpc.EgressClient + es EgressStore + telemetry telemetry.TelemetryService } func NewEgressLauncher( - psrpcClient rpc.EgressClient, - clientDeprecated egress.RPCClient, + client rpc.EgressClient, es EgressStore, ts telemetry.TelemetryService) rtc.EgressLauncher { - if psrpcClient == nil && clientDeprecated == nil { + if client == nil { return nil } return &egressLauncher{ - psrpcClient: psrpcClient, - clientDeprecated: clientDeprecated, - es: es, - telemetry: ts, + client: client, + es: es, + telemetry: ts, } } func NewEgressService( - psrpcClient rpc.EgressClient, - clientDeprecated egress.RPCClient, + client rpc.EgressClient, store ServiceStore, es EgressStore, rs livekit.RoomService, @@ -61,13 +69,12 @@ func NewEgressService( launcher rtc.EgressLauncher, ) *EgressService { return &EgressService{ - psrpcClient: psrpcClient, - clientDeprecated: clientDeprecated, - store: store, - es: es, - roomService: rs, - telemetry: ts, - launcher: launcher, + client: client, + store: store, + es: es, + roomService: rs, + telemetry: ts, + launcher: launcher, } } @@ -175,21 +182,12 @@ func (s *egressLauncher) StartEgress(ctx context.Context, req *rpc.StartEgressRe return s.StartEgressWithClusterId(ctx, "", req) } func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId string, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { - var info *livekit.EgressInfo - var err error - // Ensure we have an Egress ID if req.EgressId == "" { req.EgressId = utils.NewGuid(utils.EgressPrefix) } - if s.psrpcClient != nil { - info, err = s.psrpcClient.StartEgress(ctx, clusterId, req) - } else { - logger.Infow("using deprecated egress client") - // SendRequest will transform rpc.StartEgressRequest into deprecated livekit.StartEgressRequest - info, err = s.clientDeprecated.SendRequest(ctx, req) - } + info, err := s.client.StartEgress(ctx, clusterId, req) if err != nil { return nil, err } @@ -213,7 +211,7 @@ func (s *EgressService) UpdateLayout(ctx context.Context, req *livekit.UpdateLay if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } @@ -249,36 +247,27 @@ func (s *EgressService) UpdateStream(ctx context.Context, req *livekit.UpdateStr return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } - race := rpc.NewRace[livekit.EgressInfo](ctx) - if s.clientDeprecated != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.clientDeprecated.SendRequest(ctx, &livekit.EgressRequest{ - EgressId: req.EgressId, - Request: &livekit.EgressRequest_UpdateStream{ - UpdateStream: req, - }, - }) - }) - } - if s.psrpcClient != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.psrpcClient.UpdateStream(ctx, req.EgressId, req) - }) - } - _, info, err := race.Wait() + info, err := s.client.UpdateStream(ctx, req.EgressId, req) if err != nil { - return nil, err - } - - go func() { - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not write egress info", err) + var loadErr error + info, loadErr = s.es.LoadEgress(ctx, req.EgressId) + if loadErr != nil { + return nil, loadErr } - }() + + switch info.Status { + case livekit.EgressStatus_EGRESS_STARTING, + livekit.EgressStatus_EGRESS_ACTIVE: + return nil, err + default: + return nil, twirp.NewError(twirp.FailedPrecondition, + fmt.Sprintf("egress with status %s cannot be updated", info.Status.String())) + } + } return info, nil } @@ -290,7 +279,7 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } @@ -300,10 +289,13 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR if err != nil { return nil, err } - items = []*livekit.EgressInfo{info} + + if !req.Active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + items = []*livekit.EgressInfo{info} + } } else { var err error - items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName)) + items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName), req.Active) if err != nil { return nil, err } @@ -318,39 +310,26 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } - info, err := s.es.LoadEgress(ctx, req.EgressId) + info, err := s.client.StopEgress(ctx, req.EgressId, req) if err != nil { - return nil, err - } else { - if info.Status != livekit.EgressStatus_EGRESS_STARTING && - info.Status != livekit.EgressStatus_EGRESS_ACTIVE { - return nil, twirp.NewError(twirp.FailedPrecondition, fmt.Sprintf("egress with status %s cannot be stopped", info.Status.String())) + var loadErr error + info, loadErr = s.es.LoadEgress(ctx, req.EgressId) + if loadErr != nil { + return nil, loadErr } - } - race := rpc.NewRace[livekit.EgressInfo](ctx) - if s.clientDeprecated != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.clientDeprecated.SendRequest(ctx, &livekit.EgressRequest{ - EgressId: req.EgressId, - Request: &livekit.EgressRequest_Stop{ - Stop: req, - }, - }) - }) - } - if s.psrpcClient != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.psrpcClient.StopEgress(ctx, req.EgressId, req) - }) - } - _, info, err = race.Wait() - if err != nil { - return nil, err + switch info.Status { + case livekit.EgressStatus_EGRESS_STARTING, + livekit.EgressStatus_EGRESS_ACTIVE: + return nil, err + default: + return nil, twirp.NewError(twirp.FailedPrecondition, + fmt.Sprintf("egress with status %s cannot be stopped", info.Status.String())) + } } go func() { diff --git a/pkg/service/errors.go b/pkg/service/errors.go index b856579da..a1473c3a2 100644 --- a/pkg/service/errors.go +++ b/pkg/service/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 6d66474a9..45c0bb789 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -5,6 +19,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -55,7 +70,17 @@ func (s *IngressService) CreateIngress(ctx context.Context, req *livekit.CreateI AppendLogFields(ctx, fields...) }() - ig, err := s.CreateIngressWithUrlPrefix(ctx, s.conf.RTMPBaseURL, req) + var urlPrefix string + switch req.InputType { + case livekit.IngressInput_RTMP_INPUT: + urlPrefix = s.conf.RTMPBaseURL + case livekit.IngressInput_WHIP_INPUT: + urlPrefix = s.conf.WHIPBaseURL + default: + return nil, ingress.ErrInvalidIngressType + } + + ig, err := s.CreateIngressWithUrlPrefix(ctx, urlPrefix, req) if err != nil { return nil, err } @@ -83,6 +108,7 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref InputType: req.InputType, Audio: req.Audio, Video: req.Video, + BypassTranscoding: req.BypassTranscoding, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, ParticipantName: req.ParticipantName, @@ -90,14 +116,49 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref State: &livekit.IngressState{}, } + if err := ingress.ValidateForSerialization(info); err != nil { + return nil, err + } + if err = s.store.StoreIngress(ctx, info); err != nil { logger.Errorw("could not write ingress info", err) return nil, err } + s.telemetry.IngressCreated(ctx, info) return info, nil } +func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.IngressInfo) error { + if req.Name != "" { + info.Name = req.Name + } + if req.RoomName != "" { + info.RoomName = req.RoomName + } + if req.ParticipantIdentity != "" { + info.ParticipantIdentity = req.ParticipantIdentity + } + if req.ParticipantName != "" { + info.ParticipantName = req.ParticipantName + } + if req.BypassTranscoding != nil { + info.BypassTranscoding = *req.BypassTranscoding + } + if req.Audio != nil { + info.Audio = req.Audio + } + if req.Video != nil { + info.Video = req.Video + } + + if err := ingress.ValidateForSerialization(info); err != nil { + return err + } + + return nil +} + func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateIngressRequest) (*livekit.IngressInfo, error) { fields := []interface{}{ "ingress", req.IngressId, @@ -132,28 +193,19 @@ func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateI fallthrough case livekit.IngressState_ENDPOINT_INACTIVE: - if req.Name != "" { - info.Name = req.Name - } - if req.RoomName != "" { - info.RoomName = req.RoomName - } - if req.ParticipantIdentity != "" { - info.ParticipantIdentity = req.ParticipantIdentity - } - if req.ParticipantName != "" { - info.ParticipantName = req.ParticipantName - } - if req.Audio != nil { - info.Audio = req.Audio - } - if req.Video != nil { - info.Video = req.Video + err = updateInfoUsingRequest(req, info) + if err != nil { + return nil, err } case livekit.IngressState_ENDPOINT_BUFFERING, livekit.IngressState_ENDPOINT_PUBLISHING: - // Do not update store the returned state as the ingress service will do it + err := updateInfoUsingRequest(req, info) + if err != nil { + return nil, err + } + + // Do not store the returned state as the ingress service will do it if _, err = s.psrpcClient.UpdateIngress(ctx, req.IngressId, req); err != nil { logger.Warnw("could not update active ingress", err) } @@ -178,10 +230,19 @@ func (s *IngressService) ListIngress(ctx context.Context, req *livekit.ListIngre return nil, ErrIngressNotConnected } - infos, err := s.store.ListIngress(ctx, livekit.RoomName(req.RoomName)) - if err != nil { - logger.Errorw("could not list ingress info", err) - return nil, err + var infos []*livekit.IngressInfo + if req.IngressId != "" { + info, err := s.store.LoadIngress(ctx, req.IngressId) + if err != nil { + return nil, err + } + infos = []*livekit.IngressInfo{info} + } else { + infos, err = s.store.ListIngress(ctx, livekit.RoomName(req.RoomName)) + if err != nil { + logger.Errorw("could not list ingress info", err) + return nil, err + } } return &livekit.ListIngressResponse{Items: infos}, nil @@ -217,5 +278,8 @@ func (s *IngressService) DeleteIngress(ctx context.Context, req *livekit.DeleteI } info.State.Status = livekit.IngressState_ENDPOINT_INACTIVE + + s.telemetry.IngressDeleted(ctx, info) + return info, nil } diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index 607838f4b..36da68fc5 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -42,7 +56,7 @@ type ServiceStore interface { type EgressStore interface { StoreEgress(ctx context.Context, info *livekit.EgressInfo) error LoadEgress(ctx context.Context, egressID string) (*livekit.EgressInfo, error) - ListEgress(ctx context.Context, roomName livekit.RoomName) ([]*livekit.EgressInfo, error) + ListEgress(ctx context.Context, roomName livekit.RoomName, active bool) ([]*livekit.EgressInfo, error) UpdateEgress(ctx context.Context, info *livekit.EgressInfo) error } diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index b9e49e4b6..b8fedd8ac 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -1,15 +1,26 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( "context" "errors" - "time" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -17,12 +28,11 @@ import ( ) type IOInfoService struct { - psrpcServer rpc.IOInfoServer - es EgressStore - is IngressStore - telemetry telemetry.TelemetryService - ecDeprecated egress.RPCClient - shutdown chan struct{} + ioServer rpc.IOInfoServer + es EgressStore + is IngressStore + telemetry telemetry.TelemetryService + shutdown chan struct{} } func NewIOInfoService( @@ -31,22 +41,20 @@ func NewIOInfoService( es EgressStore, is IngressStore, ts telemetry.TelemetryService, - ec egress.RPCClient, ) (*IOInfoService, error) { s := &IOInfoService{ - es: es, - is: is, - telemetry: ts, - ecDeprecated: ec, - shutdown: make(chan struct{}), + es: es, + is: is, + telemetry: ts, + shutdown: make(chan struct{}), } if bus != nil { - psrpcServer, err := rpc.NewIOInfoServer(string(nodeID), s, bus) + ioServer, err := rpc.NewIOInfoServer(string(nodeID), s, bus) if err != nil { return nil, err } - s.psrpcServer = psrpcServer + s.ioServer = ioServer } return s, nil @@ -60,30 +68,23 @@ func (s *IOInfoService) Start() error { logger.Errorw("failed to start redis egress worker", err) return err } - - go s.egressWorkerDeprecated() } return nil } func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) { + err := s.es.UpdateEgress(ctx, info) + switch info.Status { + case livekit.EgressStatus_EGRESS_ACTIVE: + s.telemetry.EgressUpdated(ctx, info) + case livekit.EgressStatus_EGRESS_COMPLETE, livekit.EgressStatus_EGRESS_FAILED, livekit.EgressStatus_EGRESS_ABORTED, livekit.EgressStatus_EGRESS_LIMIT_REACHED: - // make sure endedAt is set so it eventually gets deleted - if info.EndedAt == 0 { - info.EndedAt = time.Now().UnixNano() - } - - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not update egress", err) - return nil, err - } - // log results if info.Error != "" { logger.Errorw("egress failed", errors.New(info.Error), "egressID", info.EgressId) @@ -92,12 +93,10 @@ func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.Egre } s.telemetry.EgressEnded(ctx, info) - - default: - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not update egress", err) - return nil, err - } + } + if err != nil { + logger.Errorw("could not update egress", err) + return nil, err } return &emptypb.Empty{}, nil @@ -118,61 +117,62 @@ func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoReques } else if req.StreamKey != "" { info, err = s.is.LoadIngressFromStreamKey(context.Background(), req.StreamKey) } else { - err = errors.New("request needs to specity either IngressId or StreamKey") + err = errors.New("request needs to specify either IngressId or StreamKey") } return info, err } func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) { + info, err := s.is.LoadIngress(ctx, req.IngressId) + if err != nil { + return nil, err + } + if err := s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil { logger.Errorw("could not update ingress", err) return nil, err } + + if info.State.Status != req.State.Status { + info.State = req.State + + switch req.State.Status { + case livekit.IngressState_ENDPOINT_ERROR, + livekit.IngressState_ENDPOINT_INACTIVE: + s.telemetry.IngressEnded(ctx, info) + + if req.State.Error != "" { + logger.Infow("ingress failed", "error", req.State.Error, "ingressID", req.IngressId) + } else { + logger.Infow("ingress ended", "ingressID", req.IngressId) + } + + case livekit.IngressState_ENDPOINT_PUBLISHING: + s.telemetry.IngressStarted(ctx, info) + + logger.Infow("ingress started", "ingressID", req.IngressId) + + case livekit.IngressState_ENDPOINT_BUFFERING: + s.telemetry.IngressUpdated(ctx, info) + + logger.Infow("ingress buffering", "ingressID", req.IngressId) + } + } else { + // Status didn't change, send Updated event + info.State = req.State + + s.telemetry.IngressUpdated(ctx, info) + + logger.Infow("ingress updated", "ingressID", req.IngressId) + } + return &emptypb.Empty{}, nil } func (s *IOInfoService) Stop() { close(s.shutdown) - if s.psrpcServer != nil { - s.psrpcServer.Shutdown() + if s.ioServer != nil { + s.ioServer.Shutdown() } } - -// Deprecated -func (s *IOInfoService) egressWorkerDeprecated() error { - if s.ecDeprecated == nil { - return nil - } - - go func() { - sub, err := s.ecDeprecated.GetUpdateChannel(context.Background()) - if err != nil { - logger.Errorw("failed to subscribe to results channel", err) - } - - resChan := sub.Channel() - for { - select { - case msg := <-resChan: - b := sub.Payload(msg) - info := &livekit.EgressInfo{} - if err = proto.Unmarshal(b, info); err != nil { - logger.Errorw("failed to read results", err) - continue - } - _, err = s.UpdateEgressInfo(context.Background(), info) - if err != nil { - logger.Errorw("failed to update egress info", err) - } - - case <-s.shutdown: - _ = sub.Close() - s.es.(*RedisStore).Stop() - return - } - } - }() - - return nil -} diff --git a/pkg/service/localstore.go b/pkg/service/localstore.go index 53518022b..a651c24a0 100644 --- a/pkg/service/localstore.go +++ b/pkg/service/localstore.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index def52e203..e8fe4ecc9 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -27,16 +41,15 @@ const ( RoomInternalKey = "room_internal" // EgressKey is a hash of egressID => egress info - EgressKey = "egress" - EndedEgressKey = "ended_egress" - RoomEgressPrefix = "egress:room:" - DeprecatedRoomEgressPrefix = "room_egress:" + EgressKey = "egress" + EndedEgressKey = "ended_egress" + RoomEgressPrefix = "egress:room:" // IngressKey is a hash of ingressID => ingress info IngressKey = "ingress" - StreamKeyKey = "stream_key" - IngressStatePrefix = "ingress_state:" - RoomIngressPrefix = "room_ingress:" + StreamKeyKey = "{ingress}_stream_key" + IngressStatePrefix = "{ingress}_state:" + RoomIngressPrefix = "room_{ingress}:" // RoomParticipantsPrefix is hash of participant_name => ParticipantInfo RoomParticipantsPrefix = "room_participants:" @@ -247,9 +260,9 @@ func (s *RedisStore) LockRoom(_ context.Context, roomName livekit.RoomName, dura return "", ErrRoomLockFailed } -func (s *RedisStore) UnlockRoom(ctx context.Context, roomName livekit.RoomName, uid string) error { +func (s *RedisStore) UnlockRoom(_ context.Context, roomName livekit.RoomName, uid string) error { key := RoomLockPrefix + string(roomName) - res, err := s.unlockScript.Run(ctx, s.rc, []string{key}, uid).Result() + res, err := s.unlockScript.Run(s.ctx, s.rc, []string{key}, uid).Result() if err != nil { return err } @@ -321,10 +334,10 @@ func (s *RedisStore) StoreEgress(_ context.Context, info *livekit.EgressInfo) er return err } - tx := s.rc.TxPipeline() - tx.HSet(s.ctx, EgressKey, info.EgressId, data) - tx.SAdd(s.ctx, RoomEgressPrefix+info.RoomName, info.EgressId) - if _, err = tx.Exec(s.ctx); err != nil { + pp := s.rc.Pipeline() + pp.HSet(s.ctx, EgressKey, info.EgressId, data) + pp.SAdd(s.ctx, RoomEgressPrefix+info.RoomName, info.EgressId) + if _, err = pp.Exec(s.ctx); err != nil { return errors.Wrap(err, "could not store egress info") } @@ -350,7 +363,7 @@ func (s *RedisStore) LoadEgress(_ context.Context, egressID string) (*livekit.Eg } } -func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([]*livekit.EgressInfo, error) { +func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName, active bool) ([]*livekit.EgressInfo, error) { var infos []*livekit.EgressInfo if roomName == "" { @@ -368,7 +381,11 @@ func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([ if err != nil { return nil, err } - infos = append(infos, info) + + // if active, filter status starting, active, and ending + if !active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + infos = append(infos, info) + } } } else { egressIDs, err := s.rc.SMembers(s.ctx, RoomEgressPrefix+string(roomName)).Result() @@ -389,7 +406,11 @@ func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([ if err != nil { return nil, err } - infos = append(infos, info) + + // if active, filter status starting, active, and ending + if !active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + infos = append(infos, info) + } } } @@ -403,10 +424,10 @@ func (s *RedisStore) UpdateEgress(_ context.Context, info *livekit.EgressInfo) e } if info.EndedAt != 0 { - tx := s.rc.TxPipeline() - tx.HSet(s.ctx, EgressKey, info.EgressId, data) - tx.HSet(s.ctx, EndedEgressKey, info.EgressId, egressEndedValue(info.RoomName, info.EndedAt)) - _, err = tx.Exec(s.ctx) + pp := s.rc.Pipeline() + pp.HSet(s.ctx, EgressKey, info.EgressId, data) + pp.HSet(s.ctx, EndedEgressKey, info.EgressId, egressEndedValue(info.RoomName, info.EndedAt)) + _, err = pp.Exec(s.ctx) } else { err = s.rc.HSet(s.ctx, EgressKey, info.EgressId, data).Err() } @@ -450,11 +471,12 @@ func (s *RedisStore) CleanEndedEgress() error { } if endedAt < expiry { - tx := s.rc.TxPipeline() - tx.HDel(s.ctx, EndedEgressKey, egressID) - tx.SRem(s.ctx, RoomEgressPrefix+roomName, egressID) - tx.HDel(s.ctx, EgressKey, egressID) - if _, err := tx.Exec(s.ctx); err != nil { + pp := s.rc.Pipeline() + pp.SRem(s.ctx, RoomEgressPrefix+roomName, egressID) + pp.HDel(s.ctx, EgressKey, egressID) + // Delete the EndedEgressKey entry last so that future sweeper runs get another chance to delete dangling data is the deletion partially failed. + pp.HDel(s.ctx, EndedEgressKey, egressID) + if _, err := pp.Exec(s.ctx); err != nil { return err } } @@ -497,11 +519,10 @@ func (s *RedisStore) storeIngress(_ context.Context, info *livekit.IngressInfo) } // ignore state - infoCopy := livekit.IngressInfo{} - infoCopy = *info + infoCopy := proto.Clone(info).(*livekit.IngressInfo) infoCopy.State = nil - data, err := proto.Marshal(&infoCopy) + data, err := proto.Marshal(infoCopy) if err != nil { return err } @@ -582,11 +603,6 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat txf := func(tx *redis.Tx) error { var oldStartedAt int64 - info, err := s.loadIngress(tx, ingressId) - if err != nil { - return err - } - oldState, err := s.loadIngressState(tx, ingressId) switch err { case ErrIngressNotFound: @@ -604,7 +620,6 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat } p.Set(s.ctx, IngressStatePrefix+ingressId, data, 0) - p.HSet(s.ctx, StreamKeyKey, info.StreamKey, info.IngressId) return nil }) @@ -624,7 +639,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Retry if the key has been changed. for i := 0; i < maxRetries; i++ { - err := s.rc.Watch(s.ctx, txf, IngressKey, IngressStatePrefix+ingressId) + err := s.rc.Watch(s.ctx, txf, IngressStatePrefix+ingressId) switch err { case redis.TxFailedErr: // Optimistic lock lost. Retry. @@ -707,7 +722,7 @@ func (s *RedisStore) LoadIngressFromStreamKey(_ context.Context, streamKey strin } } -func (s *RedisStore) ListIngress(ctx context.Context, roomName livekit.RoomName) ([]*livekit.IngressInfo, error) { +func (s *RedisStore) ListIngress(_ context.Context, roomName livekit.RoomName) ([]*livekit.IngressInfo, error) { var infos []*livekit.IngressInfo if roomName == "" { diff --git a/pkg/service/redisstore_test.go b/pkg/service/redisstore_test.go index 17e8e5021..32e550378 100644 --- a/pkg/service/redisstore_test.go +++ b/pkg/service/redisstore_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( @@ -189,12 +203,12 @@ func TestEgressStore(t *testing.T) { require.NoError(t, rs.UpdateEgress(ctx, info)) // list - list, err := rs.ListEgress(ctx, "") + list, err := rs.ListEgress(ctx, "", false) require.NoError(t, err) require.Len(t, list, 2) // list by room - list, err = rs.ListEgress(ctx, livekit.RoomName(roomName)) + list, err = rs.ListEgress(ctx, livekit.RoomName(roomName), false) require.NoError(t, err) require.Len(t, list, 1) @@ -207,7 +221,7 @@ func TestEgressStore(t *testing.T) { require.NoError(t, rs.CleanEndedEgress()) // list - list, err = rs.ListEgress(ctx, livekit.RoomName(roomName)) + list, err = rs.ListEgress(ctx, livekit.RoomName(roomName), false) require.NoError(t, err) require.Len(t, list, 0) } diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 8c82f6754..9b38b9768 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/roomallocator_test.go b/pkg/service/roomallocator_test.go index ddf305d66..4397e22c4 100644 --- a/pkg/service/roomallocator_test.go +++ b/pkg/service/roomallocator_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 28254ea62..fe4008cb3 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -11,6 +25,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/version" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -67,7 +82,7 @@ func NewLocalRoomManager( egressLauncher rtc.EgressLauncher, versionGenerator utils.TimedVersionGenerator, ) (*RoomManager, error) { - rtcConf, err := rtc.NewWebRTCConfig(conf, currentNode.Ip) + rtcConf, err := rtc.NewWebRTCConfig(conf) if err != nil { return nil, err } @@ -193,7 +208,7 @@ func (r *RoomManager) Stop() { for _, room := range rooms { for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomManagerStop) + _ = p.Close(true, types.ParticipantCloseReasonRoomManagerStop, false) } room.Close() } @@ -228,11 +243,35 @@ func (r *RoomManager) StartSession( if pi.Identity == "" { return nil } + participant := room.GetParticipant(pi.Identity) if participant != nil { - // When reconnecting, it means WS has interrupted by underlying peer connection is still ok - // in this mode, we'll keep the participant SID, and just swap the sink for the underlying connection + // When reconnecting, it means WS has interrupted but underlying peer connection is still ok in this state, + // we'll keep the participant SID, and just swap the sink for the underlying connection if pi.Reconnect { + if participant.IsClosed() { + // Send leave request if participant is closed, i. e. handle the case of client trying to resume crossing wires with + // server closing the participant due to some irrecoverable condition. Such a condition would have triggered + // a full reconnect when that condition occurred. + // + // It is possible that the client did not get that send request. So, send it again. + logger.Infow("cannot restart a closed participant", + "room", roomName, + "nodeID", r.currentNode.Id, + "participant", pi.Identity, + "reason", pi.ReconnectReason, + ) + _ = responseSink.WriteMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: &livekit.LeaveRequest{ + CanReconnect: true, + Reason: livekit.DisconnectReason_STATE_MISMATCH, + }, + }, + }) + return errors.New("could not restart closed participant") + } + logger.Infow("resuming RTC session", "room", roomName, "nodeID", r.currentNode.Id, @@ -243,20 +282,27 @@ func (r *RoomManager) StartSession( if iceConfig == nil { iceConfig = &livekit.ICEConfig{} } - if err = room.ResumeParticipant(participant, requestSource, responseSink, - r.iceServersForRoom(protoRoom, iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS), - pi.ReconnectReason); err != nil { + if err = room.ResumeParticipant( + participant, + requestSource, + responseSink, + r.iceServersForRoom( + protoRoom, + iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS, + ), + pi.ReconnectReason, + ); err != nil { logger.Warnw("could not resume participant", err, "participant", pi.Identity) return err } r.telemetry.ParticipantResumed(ctx, room.ToProto(), participant.ToProto(), livekit.NodeID(r.currentNode.Id), pi.ReconnectReason) go r.rtcSessionWorker(room, participant, requestSource) return nil - } else { - participant.GetLogger().Infow("removing duplicate participant") - // we need to clean up the existing participant, so a new one can join - room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonDuplicateIdentity) } + + // we need to clean up the existing participant, so a new one can join + participant.GetLogger().Infow("removing duplicate participant") + room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonDuplicateIdentity) } else if pi.Reconnect { // send leave request if participant is trying to reconnect without keep subscribe state // but missing from the room @@ -278,6 +324,9 @@ func (r *RoomManager) StartSession( "sdk", pi.Client.Sdk, "sdkVersion", pi.Client.Version, "protocol", pi.Client.Protocol, + "reconnect", pi.Reconnect, + "reconnectReason", pi.ReconnectReason, + "adaptiveStream", pi.AdaptiveStream, ) clientConf := r.clientConfManager.GetConfiguration(pi.Client) @@ -302,6 +351,10 @@ func (r *RoomManager) StartSession( if r.config.RTC.ReconnectOnSubscriptionError != nil { reconnectOnSubscriptionError = *r.config.RTC.ReconnectOnSubscriptionError } + subscriberAllowPause := r.config.RTC.CongestionControl.AllowPause + if pi.SubscriberAllowPause != nil { + subscriberAllowPause = *pi.SubscriberAllowPause + } participant, err = rtc.NewParticipant(rtc.ParticipantParams{ Identity: pi.Identity, Name: pi.Name, @@ -312,6 +365,7 @@ func (r *RoomManager) StartSession( VideoConfig: r.config.Video, ProtocolVersion: pv, Telemetry: r.telemetry, + Trailer: room.Trailer(), PLIThrottleConfig: r.config.RTC.PLIThrottle, CongestionControlConfig: r.config.RTC.CongestionControl, EnabledCodecs: protoRoom.EnabledCodecs, @@ -333,6 +387,9 @@ func (r *RoomManager) StartSession( ReconnectOnSubscriptionError: reconnectOnSubscriptionError, VersionGenerator: r.versionGenerator, TrackResolver: room.ResolveMediaTrackForSubscriber, + SubscriberAllowPause: subscriberAllowPause, + SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, + SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, }) if err != nil { return err @@ -345,7 +402,7 @@ func (r *RoomManager) StartSession( } if err = room.Join(participant, requestSource, &opts, r.iceServersForRoom(protoRoom, iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS)); err != nil { pLogger.Errorw("could not join room", err) - _ = participant.Close(true, types.ParticipantCloseReasonJoinFailed) + _ = participant.Close(true, types.ParticipantCloseReasonJoinFailed, false) return err } if err = r.roomStore.StoreParticipant(ctx, roomName, participant.ToProto()); err != nil { @@ -366,7 +423,7 @@ func (r *RoomManager) StartSession( clientMeta := &livekit.AnalyticsClientMeta{Region: r.currentNode.Region, Node: r.currentNode.Id} r.telemetry.ParticipantJoined(ctx, protoRoom, participant.ToProto(), pi.Client, clientMeta, true) - participant.OnClose(func(p types.LocalParticipant, disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID) { + participant.OnClose(func(p types.LocalParticipant) { if err := r.roomStore.DeleteParticipant(ctx, roomName, p.Identity()); err != nil { pLogger.Errorw("could not delete participant", err) } @@ -375,8 +432,6 @@ func (r *RoomManager) StartSession( proto := room.ToProto() persistRoomForParticipantCount(proto) r.telemetry.ParticipantLeft(ctx, proto, p.ToProto(), true) - - room.RemoveDisallowedSubscriptions(p, disallowedSubscriptions) }) participant.OnClaimsChanged(func(participant types.LocalParticipant) { pLogger.Debugw("refreshing client token after claims change") @@ -441,7 +496,7 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room newRoom.Logger.Infow("room closed") }) - newRoom.OnMetadataUpdate(func(metadata string) { + newRoom.OnRoomUpdated(func() { if err := r.roomStore.StoreRoom(ctx, newRoom.ToProto(), newRoom.Internal()); err != nil { newRoom.Logger.Errorw("could not handle metadata update", err) } @@ -476,7 +531,7 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa false, ) defer func() { - pLogger.Debugw("RTC session finishing") + pLogger.Debugw("RTC session finishing", "connID", requestSource.ConnectionID()) requestSource.Close() }() @@ -502,14 +557,14 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa case <-tokenTicker.C: // refresh token with the first API Key/secret pair if err := r.refreshToken(participant); err != nil { - pLogger.Errorw("could not refresh token", err) + pLogger.Errorw("could not refresh token", err, "connID", requestSource.ConnectionID()) } case obj := <-requestSource.ReadChan(): // In single node mode, the request source is directly tied to the signal message channel // this means ICE restart isn't possible in single node mode if obj == nil { if room.GetParticipantRequestSource(participant.Identity()) == requestSource { - participant.SetSignalSourceValid(false) + participant.HandleSignalSourceClose() } return } @@ -582,19 +637,14 @@ func (r *RoomManager) handleRTCMessage(ctx context.Context, roomName livekit.Roo } pLogger.Debugw("updating participant", "metadata", rm.UpdateParticipant.Metadata, "permission", rm.UpdateParticipant.Permission) - if rm.UpdateParticipant.Name != "" { - participant.SetName(rm.UpdateParticipant.Name) - } - if rm.UpdateParticipant.Metadata != "" { - participant.SetMetadata(rm.UpdateParticipant.Metadata) - } + room.UpdateParticipantMetadata(participant, rm.UpdateParticipant.Name, rm.UpdateParticipant.Metadata) if rm.UpdateParticipant.Permission != nil { participant.SetPermission(rm.UpdateParticipant.Permission) } case *livekit.RTCNodeMessage_DeleteRoom: room.Logger.Infow("deleting room") for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonServiceRequestDeleteRoom) + _ = p.Close(true, types.ParticipantCloseReasonServiceRequestDeleteRoom, false) } room.Close() case *livekit.RTCNodeMessage_UpdateSubscriptions: @@ -678,7 +728,7 @@ func (r *RoomManager) iceServersForRoom(ri *livekit.Room, tlsOnly bool) []*livek } if !hasSTUN { - iceServers = append(iceServers, iceServerForStunServers(config.DefaultStunServers)) + iceServers = append(iceServers, iceServerForStunServers(rtcconfig.DefaultStunServers)) } return iceServers } diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index d4a4fe5a0..3e8997587 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -1,7 +1,22 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( "context" + "fmt" "strconv" "time" @@ -14,6 +29,7 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" ) @@ -125,6 +141,11 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq if err := EnsureCreatePermission(ctx); err != nil { return nil, twirpAuthError(err) } + + if _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false); err == ErrRoomNotFound { + return nil, twirp.NotFoundError("room not found") + } + err := s.router.WriteRoomRTC(ctx, livekit.RoomName(req.Room), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_DeleteRoom{ DeleteRoom: req, @@ -185,6 +206,11 @@ func (s *RoomService) GetParticipant(ctx context.Context, req *livekit.RoomParti func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (*livekit.RemoveParticipantResponse, error) { AppendLogFields(ctx, "room", req.Room, "participant", req.Identity) + + if _, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)); err == ErrParticipantNotFound { + return nil, twirp.NotFoundError("participant not found") + } + err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_RemoveParticipant{ RemoveParticipant: req, @@ -212,7 +238,7 @@ func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomPa } func (s *RoomService) MutePublishedTrack(ctx context.Context, req *livekit.MuteRoomTrackRequest) (*livekit.MuteRoomTrackResponse, error) { - AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "track", req.TrackSid, "muted", req.Muted) + AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "trackID", req.TrackSid, "muted", req.Muted) if err := EnsureAdminPermission(ctx, livekit.RoomName(req.Room)); err != nil { return nil, twirpAuthError(err) } @@ -273,20 +299,24 @@ func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.Update } var participant *livekit.ParticipantInfo + var detailedError error err = s.confirmExecution(func() error { participant, err = s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) if err != nil { return err } if req.Metadata != "" && participant.Metadata != req.Metadata { + detailedError = fmt.Errorf("metadata does not match") return ErrOperationFailed } if req.Permission != nil && !proto.Equal(req.Permission, participant.Permission) { + detailedError = fmt.Errorf("permissions do not match, expected: %v, actual: %v", req.Permission, participant.Permission) return ErrOperationFailed } return nil }) if err != nil { + logger.Warnw("could not confirm participant update", detailedError) return nil, err } @@ -298,7 +328,7 @@ func (s *RoomService) UpdateSubscriptions(ctx context.Context, req *livekit.Upda for _, pt := range req.ParticipantTracks { trackSIDs = append(trackSIDs, pt.TrackSids...) } - AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "track", trackSIDs) + AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "trackID", trackSIDs) err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_UpdateSubscriptions{ UpdateSubscriptions: req, diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index 744e18442..a7433a090 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( @@ -17,7 +31,7 @@ import ( ) func TestDeleteRoom(t *testing.T) { - t.Run("normal deletion", func(t *testing.T) { + t.Run("delete non-existent", func(t *testing.T) { svc := newTestRoomService(config.RoomConfig{}) grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{ @@ -29,7 +43,12 @@ func TestDeleteRoom(t *testing.T) { _, err := svc.DeleteRoom(ctx, &livekit.DeleteRoomRequest{ Room: "testroom", }) - require.NoError(t, err) + require.Error(t, err) + if terr, ok := err.(twirp.Error); ok { + require.Equal(t, twirp.NotFound, terr.Code()) + } else { + require.Fail(t, "should be twirp error") + } }) t.Run("missing permissions", func(t *testing.T) { diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 6a609057e..5259622bc 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -5,15 +19,17 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "os" "strconv" "strings" + "sync" "time" "github.com/gorilla/websocket" - "github.com/sebest/xff" "github.com/ua-parser/uap-go/uaparser" + "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" @@ -38,6 +54,9 @@ type RTCService struct { limits config.LimitConfig parser *uaparser.Parser telemetry telemetry.TelemetryService + + mu sync.Mutex + connections map[*websocket.Conn]struct{} } func NewRTCService( @@ -59,6 +78,7 @@ func NewRTCService( limits: conf.Limit, parser: uaparser.NewFromSaved(), telemetry: telemetry, + connections: map[*websocket.Conn]struct{}{}, } // allow connections from any origin, since script may be hosted anywhere @@ -104,6 +124,7 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic publishParam := r.FormValue("publish") adaptiveStreamParam := r.FormValue("adaptive_stream") participantID := r.FormValue("sid") + subscriberAllowPauseParam := r.FormValue("subscriber_allow_pause") if onlyName != "" { roomName = onlyName @@ -111,7 +132,7 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic // this is new connection for existing participant - with publish only permissions if publishParam != "" { - // Make sure grant has CanPublish set, + // Make sure grant has GetCanPublish set, if !claims.Video.GetCanPublish() { return "", routing.ParticipantInit{}, http.StatusUnauthorized, rtc.ErrPermissionDenied } @@ -160,6 +181,10 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic if adaptiveStreamParam != "" { pi.AdaptiveStream = boolValue(adaptiveStreamParam) } + if subscriberAllowPauseParam != "" { + subscriberAllowPause := boolValue(subscriberAllowPauseParam) + pi.SubscriberAllowPause = &subscriberAllowPause + } return roomName, pi, http.StatusOK, nil } @@ -186,10 +211,15 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // give it a few attempts to start session var cr connectionResult + var initialResponse *livekit.SignalResponse for i := 0; i < 3; i++ { + if err = r.Context().Err(); err != nil { + break + } + connectionTimeout := 3 * time.Second * time.Duration(i+1) ctx := utils.ContextWithAttempt(r.Context(), i) - cr, err = s.startConnection(ctx, roomName, pi, connectionTimeout) + cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) if err == nil { break } @@ -206,8 +236,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { prometheus.IncrementParticipantJoin(1) - if !pi.Reconnect && cr.InitialResponse.GetJoin() != nil { - pi.ID = livekit.ParticipantID(cr.InitialResponse.GetJoin().GetParticipant().GetSid()) + if !pi.Reconnect && initialResponse.GetJoin() != nil { + pi.ID = livekit.ParticipantID(initialResponse.GetJoin().GetParticipant().GetSid()) } var signalStats *telemetry.BytesTrackStats @@ -245,9 +275,19 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + s.mu.Lock() + s.connections[conn] = struct{}{} + s.mu.Unlock() + + defer func() { + s.mu.Lock() + delete(s.connections, conn) + s.mu.Unlock() + }() + // websocket established sigConn := NewWSSignalConnection(conn) - if count, err := sigConn.WriteResponse(cr.InitialResponse); err != nil { + if count, err := sigConn.WriteResponse(initialResponse); err != nil { pLogger.Warnw("could not write initial response", err) return } else { @@ -255,7 +295,12 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { signalStats.AddBytes(uint64(count), true) } } - pLogger.Infow("new client WS connected", "connID", cr.ConnectionID) + pLogger.Infow("new client WS connected", + "connID", cr.ConnectionID, + "reconnect", pi.Reconnect, + "reconnectReason", pi.ReconnectReason, + "adaptiveStream", pi.AdaptiveStream, + ) // handle responses go func() { @@ -275,6 +320,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { return case msg := <-cr.ResponseSource.ReadChan(): if msg == nil { + pLogger.Infow("nothing to read from response source", "connID", cr.ConnectionID) return } res, ok := msg.(*livekit.SignalResponse) @@ -292,8 +338,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Debugw("sending answer", "answer", m) } - if pi.ID == "" && cr.InitialResponse.GetJoin() != nil { - pi.ID = livekit.ParticipantID(cr.InitialResponse.GetJoin().GetParticipant().GetSid()) + if pi.ID == "" && res.GetJoin() != nil { + pi.ID = livekit.ParticipantID(res.GetJoin().GetParticipant().GetSid()) signalStats = telemetry.NewBytesTrackStats( telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeSignal, pi.ID), pi.ID, @@ -313,16 +359,15 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // handle incoming requests from websocket for { req, count, err := sigConn.ReadRequest() - // normal closure if err != nil { + // normal/expected closure if err == io.EOF || strings.HasSuffix(err.Error(), "use of closed network connection") || websocket.IsCloseError(err, websocket.CloseAbnormalClosure, websocket.CloseGoingAway, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { - pLogger.Debugw("exit ws read loop for closed connection", "connID", cr.ConnectionID) - return + pLogger.Infow("exit ws read loop for closed connection", "connID", cr.ConnectionID, "wsError", err) } else { - pLogger.Errorw("error reading from websocket", err) - return + pLogger.Errorw("error reading from websocket", err, "connID", cr.ConnectionID) } + return } if signalStats != nil { signalStats.AddBytes(uint64(count), false) @@ -364,8 +409,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if err := cr.RequestSink.WriteMessage(req); err != nil { - pLogger.Warnw("error writing to request sink", err, - "connID", cr.ConnectionID) + pLogger.Warnw("error writing to request sink", err, "connID", cr.ConnectionID) } } } @@ -390,6 +434,10 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { ci.Sdk = livekit.ClientInfo_GO case "unity": ci.Sdk = livekit.ClientInfo_UNITY + case "reactnative": + ci.Sdk = livekit.ClientInfo_REACT_NATIVE + case "rust": + ci.Sdk = livekit.ClientInfo_RUST } ci.Version = values.Get("version") @@ -400,13 +448,11 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { ci.DeviceModel = values.Get("device_model") ci.Network = values.Get("network") // get real address (forwarded http header) - check Cloudflare headers first, fall back to X-Forwarded-For - ci.Address = r.Header.Get("CF-Connecting-IP") - if len(ci.Address) == 0 { - ci.Address = xff.GetRemoteAddr(r) - } + ci.Address = GetClientIP(r) // attempt to parse types for SDKs that support browser as a platform if ci.Sdk == livekit.ClientInfo_JS || + ci.Sdk == livekit.ClientInfo_REACT_NATIVE || ci.Sdk == livekit.ClientInfo_FLUTTER || ci.Sdk == livekit.ClientInfo_UNITY { client := s.parser.Parse(r.UserAgent()) @@ -431,39 +477,55 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { return ci } -type connectionResult struct { - Room *livekit.Room - ConnectionID livekit.ConnectionID - RequestSink routing.MessageSink - ResponseSource routing.MessageSource - InitialResponse *livekit.SignalResponse +func (s *RTCService) DrainConnections(interval time.Duration) { + s.mu.Lock() + conns := maps.Clone(s.connections) + s.mu.Unlock() + + // jitter drain start + time.Sleep(time.Duration(rand.Int63n(int64(interval)))) + + t := time.NewTicker(interval) + defer t.Stop() + + for c := range conns { + c.Close() + <-t.C + } } -func (s *RTCService) startConnection(ctx context.Context, roomName livekit.RoomName, pi routing.ParticipantInit, timeout time.Duration) (connectionResult, error) { +type connectionResult struct { + Room *livekit.Room + ConnectionID livekit.ConnectionID + RequestSink routing.MessageSink + ResponseSource routing.MessageSource +} + +func (s *RTCService) startConnection(ctx context.Context, roomName livekit.RoomName, pi routing.ParticipantInit, timeout time.Duration) (connectionResult, *livekit.SignalResponse, error) { var cr connectionResult var err error cr.Room, err = s.roomAllocator.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: string(roomName)}) if err != nil { - return cr, err + return cr, nil, err } // this needs to be started first *before* using router functions on this node cr.ConnectionID, cr.RequestSink, cr.ResponseSource, err = s.router.StartParticipantSignal(ctx, roomName, pi) if err != nil { - return cr, err + return cr, nil, err } // wait for the first message before upgrading to websocket. If no one is // responding to our connection attempt, we should terminate the connection // instead of waiting forever on the WebSocket - cr.InitialResponse, err = readInitialResponse(cr.ResponseSource, timeout) + initialResponse, err := readInitialResponse(cr.ResponseSource, timeout) if err != nil { // close the connection to avoid leaking cr.RequestSink.Close() cr.ResponseSource.Close() - return cr, err + return cr, nil, err } - return cr, nil + return cr, initialResponse, nil } func readInitialResponse(source routing.MessageSource, timeout time.Duration) (*livekit.SignalResponse, error) { diff --git a/pkg/service/server.go b/pkg/service/server.go index bbd9f6775..69191ff81 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -8,7 +22,9 @@ import ( "net" "net/http" _ "net/http/pprof" + "runtime" "runtime/pprof" + "strconv" "time" "github.com/pion/turn/v2" @@ -176,14 +192,14 @@ func (s *LivekitServer) Start() error { listeners := make([]net.Listener, 0) promListeners := make([]net.Listener, 0) for _, addr := range addresses { - ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, s.config.Port)) + ln, err := net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(int(s.config.Port)))) if err != nil { return err } listeners = append(listeners, ln) if s.promServer != nil { - ln, err = net.Listen("tcp", fmt.Sprintf("%s:%d", addr, s.config.PrometheusPort)) + ln, err = net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(int(s.config.PrometheusPort)))) if err != nil { return err } @@ -217,11 +233,18 @@ func (s *LivekitServer) Start() error { values = append(values, "region", s.config.Region) } logger.Infow("starting LiveKit server", values...) + if runtime.GOOS == "windows" { + logger.Infow("Windows detected, capacity management is unavailable") + } for _, promLn := range promListeners { go s.promServer.Serve(promLn) } + if err := s.signalServer.Start(); err != nil { + return err + } + httpGroup := &errgroup.Group{} for _, ln := range listeners { l := ln diff --git a/pkg/service/servicefakes/fake_egress_store.go b/pkg/service/servicefakes/fake_egress_store.go index 06ee8d9cc..37f13670f 100644 --- a/pkg/service/servicefakes/fake_egress_store.go +++ b/pkg/service/servicefakes/fake_egress_store.go @@ -10,11 +10,12 @@ import ( ) type FakeEgressStore struct { - ListEgressStub func(context.Context, livekit.RoomName) ([]*livekit.EgressInfo, error) + ListEgressStub func(context.Context, livekit.RoomName, bool) ([]*livekit.EgressInfo, error) listEgressMutex sync.RWMutex listEgressArgsForCall []struct { arg1 context.Context arg2 livekit.RoomName + arg3 bool } listEgressReturns struct { result1 []*livekit.EgressInfo @@ -66,19 +67,20 @@ type FakeEgressStore struct { invocationsMutex sync.RWMutex } -func (fake *FakeEgressStore) ListEgress(arg1 context.Context, arg2 livekit.RoomName) ([]*livekit.EgressInfo, error) { +func (fake *FakeEgressStore) ListEgress(arg1 context.Context, arg2 livekit.RoomName, arg3 bool) ([]*livekit.EgressInfo, error) { fake.listEgressMutex.Lock() ret, specificReturn := fake.listEgressReturnsOnCall[len(fake.listEgressArgsForCall)] fake.listEgressArgsForCall = append(fake.listEgressArgsForCall, struct { arg1 context.Context arg2 livekit.RoomName - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.ListEgressStub fakeReturns := fake.listEgressReturns - fake.recordInvocation("ListEgress", []interface{}{arg1, arg2}) + fake.recordInvocation("ListEgress", []interface{}{arg1, arg2, arg3}) fake.listEgressMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1, ret.result2 @@ -92,17 +94,17 @@ func (fake *FakeEgressStore) ListEgressCallCount() int { return len(fake.listEgressArgsForCall) } -func (fake *FakeEgressStore) ListEgressCalls(stub func(context.Context, livekit.RoomName) ([]*livekit.EgressInfo, error)) { +func (fake *FakeEgressStore) ListEgressCalls(stub func(context.Context, livekit.RoomName, bool) ([]*livekit.EgressInfo, error)) { fake.listEgressMutex.Lock() defer fake.listEgressMutex.Unlock() fake.ListEgressStub = stub } -func (fake *FakeEgressStore) ListEgressArgsForCall(i int) (context.Context, livekit.RoomName) { +func (fake *FakeEgressStore) ListEgressArgsForCall(i int) (context.Context, livekit.RoomName, bool) { fake.listEgressMutex.RLock() defer fake.listEgressMutex.RUnlock() argsForCall := fake.listEgressArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeEgressStore) ListEgressReturns(result1 []*livekit.EgressInfo, result2 error) { diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 835788b34..862d79e44 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -13,7 +27,8 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" - "github.com/livekit/psrpc/middleware" + "github.com/livekit/psrpc/pkg/metadata" + "github.com/livekit/psrpc/pkg/middleware" ) type SessionHandler func( @@ -27,6 +42,7 @@ type SessionHandler func( type SignalServer struct { server rpc.TypedSignalServer + nodeID livekit.NodeID } func NewSignalServer( @@ -36,21 +52,17 @@ func NewSignalServer( config config.SignalRelayConfig, sessionHandler SessionHandler, ) (*SignalServer, error) { - ri := middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - }) - s, err := rpc.NewTypedSignalServer(nodeID, &signalService{region, sessionHandler}, bus, psrpc.WithServerStreamInterceptors(ri), psrpc.WithServerChannelSize(config.StreamBufferSize)) + s, err := rpc.NewTypedSignalServer( + nodeID, + &signalService{region, sessionHandler, config}, + bus, + middleware.WithServerMetrics(prometheus.PSRPCMetricsObserver{}), + psrpc.WithServerChannelSize(config.StreamBufferSize), + ) if err != nil { return nil, err } - logger.Debugw("starting relay signal server", "topic", nodeID) - if err := s.RegisterRelaySignalTopic(nodeID); err != nil { - return nil, err - } - - return &SignalServer{s}, nil + return &SignalServer{s, nodeID}, nil } func NewDefaultSignalServer( @@ -69,12 +81,41 @@ func NewDefaultSignalServer( responseSink routing.MessageSink, ) error { prometheus.IncrementParticipantRtcInit(1) + + if rr, ok := router.(*routing.RedisRouter); ok { + rtcNode, err := router.GetNodeForRoom(ctx, roomName) + if err != nil { + return err + } + + if rtcNode.Id != currentNode.Id { + err = routing.ErrIncorrectRTCNode + logger.Errorw("called participant on incorrect node", err, + "rtcNode", rtcNode, + ) + return err + } + + pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) + pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) + + // RTC session should start on this node + if err := rr.SetParticipantRTCNode(pKey, pKeyB62, currentNode.Id); err != nil { + return err + } + } + return roomManager.StartSession(ctx, roomName, pi, requestSource, responseSink) } return NewSignalServer(livekit.NodeID(currentNode.Id), currentNode.Region, bus, config, sessionHandler) } +func (s *SignalServer) Start() error { + logger.Debugw("starting relay signal server", "topic", s.nodeID) + return s.server.RegisterRelaySignalTopic(s.nodeID) +} + func (r *SignalServer) Stop() { r.server.Kill() } @@ -82,15 +123,10 @@ func (r *SignalServer) Stop() { type signalService struct { region string sessionHandler SessionHandler + config config.SignalRelayConfig } func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]) (err error) { - // copy the context to prevent a race between the session handler closing - // and the delivery of any parting messages from the client. take care to - // copy the incoming rpc headers to avoid dropping any session vars. - ctx, cancel := context.WithCancel(psrpc.NewContextWithIncomingHeader(context.Background(), psrpc.IncomingHeader(stream.Context()))) - defer cancel() - req, ok := <-stream.Channel() if !ok { return nil @@ -106,51 +142,68 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe return errors.Wrap(err, "failed to read participant from session") } - reqChan := routing.NewDefaultMessageChannel() - defer reqChan.Close() - - err = r.sessionHandler( - ctx, - livekit.RoomName(ss.RoomName), - *pi, - livekit.ConnectionID(ss.ConnectionId), - reqChan, - &relaySignalResponseSink{stream}, - ) - if err != nil { - logger.Errorw("could not handle new participant", err, - "room", ss.RoomName, - "participant", ss.Identity, - "connectionID", ss.ConnectionId, - ) - } - - for msg := range stream.Channel() { - if err = reqChan.WriteMessage(msg.Request); err != nil { - break - } - } - - logger.Debugw("participant signal stream closed", + l := logger.GetLogger().WithValues( "room", ss.RoomName, "participant", ss.Identity, - "connectionID", ss.ConnectionId, + "connID", ss.ConnectionId, ) + + sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ + Logger: l, + Stream: stream, + Config: r.config, + Writer: signalResponseMessageWriter{}, + ConnectionID: livekit.ConnectionID(ss.ConnectionId), + }) + reqChan := routing.NewDefaultMessageChannel(livekit.ConnectionID(ss.ConnectionId)) + + go func() { + err := routing.CopySignalStreamToMessageChannel[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]( + stream, + reqChan, + signalRequestMessageReader{}, + r.config, + ) + l.Infow("signal stream closed", "error", err) + + reqChan.Close() + }() + + // copy the context to prevent a race between the session handler closing + // and the delivery of any parting messages from the client. take care to + // copy the incoming rpc headers to avoid dropping any session vars. + ctx := metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context())) + + err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) + if err != nil { + l.Errorw("could not handle new participant", err) + return + } + + stream.Hijack() return } -type relaySignalResponseSink struct { - psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest] +type signalResponseMessageWriter struct{} + +func (e signalResponseMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalResponse { + r := &rpc.RelaySignalResponse{ + Seq: seq, + Responses: make([]*livekit.SignalResponse, 0, len(msgs)), + Close: close, + } + for _, m := range msgs { + r.Responses = append(r.Responses, m.(*livekit.SignalResponse)) + } + return r } -func (s *relaySignalResponseSink) Close() { - s.ServerStream.Close(nil) -} +type signalRequestMessageReader struct{} -func (s *relaySignalResponseSink) IsClosed() bool { - return s.Context().Err() != nil -} - -func (s *relaySignalResponseSink) WriteMessage(msg proto.Message) error { - return s.Send(&rpc.RelaySignalResponse{Response: msg.(*livekit.SignalResponse)}) +func (e signalRequestMessageReader) Read(rm *rpc.RelaySignalRequest) ([]proto.Message, error) { + msgs := make([]proto.Message, 0, len(rm.Requests)) + for _, m := range rm.Requests { + msgs = append(msgs, m) + } + return msgs, nil } diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go new file mode 100644 index 000000000..3e202636e --- /dev/null +++ b/pkg/service/signal_test.go @@ -0,0 +1,99 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" +) + +func init() { + prometheus.Init("node", livekit.NodeType_CONTROLLER, "test") +} + +func TestSignal(t *testing.T) { + bus := psrpc.NewLocalMessageBus() + cfg := config.SignalRelayConfig{ + Enabled: false, + RetryTimeout: 30 * time.Second, + MinRetryInterval: 500 * time.Millisecond, + MaxRetryInterval: 5 * time.Second, + StreamBufferSize: 1000, + } + + reqMessageIn := &livekit.SignalRequest{ + Message: &livekit.SignalRequest_Ping{Ping: 123}, + } + resMessageIn := &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Pong{Pong: 321}, + } + + var reqMessageOut proto.Message + var resErr error + done := make(chan struct{}) + + client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) + require.NoError(t, err) + + server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + go func() { + reqMessageOut = <-requestSource.ReadChan() + resErr = responseSink.WriteMessage(resMessageIn) + responseSink.Close() + close(done) + }() + return nil + }) + require.NoError(t, err) + + err = server.Start() + require.NoError(t, err) + + _, reqSink, resSource, err := client.StartParticipantSignal( + context.Background(), + livekit.RoomName("room1"), + routing.ParticipantInit{}, + livekit.NodeID("node1"), + ) + require.NoError(t, err) + + err = reqSink.WriteMessage(reqMessageIn) + require.NoError(t, err) + + <-done + require.True(t, proto.Equal(reqMessageIn, reqMessageOut), "req message should match %s %s", protojson.Format(reqMessageIn), protojson.Format(reqMessageOut)) + require.NoError(t, resErr) + + resMessageOut := <-resSource.ReadChan() + require.True(t, proto.Equal(resMessageIn, resMessageOut), "res message should match %s %s", protojson.Format(resMessageIn), protojson.Format(resMessageOut)) +} diff --git a/pkg/service/turn.go b/pkg/service/turn.go index 3cdd39072..08d971fb4 100644 --- a/pkg/service/turn.go +++ b/pkg/service/turn.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( @@ -11,9 +25,9 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" "github.com/livekit/livekit-server/pkg/config" - logging "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -39,7 +53,7 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone serverConfig := turn.ServerConfig{ Realm: LivekitRealm, AuthHandler: authHandler, - LoggerFactory: logging.NewLoggerFactory(logger.GetLogger()), + LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), } var relayAddrGen turn.RelayAddressGenerator = &turn.RelayAddressGeneratorPortRange{ RelayAddress: net.ParseIP(conf.RTC.NodeIP), diff --git a/pkg/service/utils.go b/pkg/service/utils.go index 07eca562d..32076455f 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -1,6 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( + "net" "net/http" "regexp" @@ -22,3 +37,18 @@ func IsValidDomain(domain string) bool { domainRegexp := regexp.MustCompile(`^(?i)[a-z0-9-]+(\.[a-z0-9-]+)+\.?$`) return domainRegexp.MatchString(domain) } + +func GetClientIP(r *http.Request) string { + // CF proxy typically is first thing the user reaches + if ip := r.Header.Get("CF-Connecting-IP"); ip != "" { + return ip + } + if ip := r.Header.Get("X-Forwarded-For"); ip != "" { + return ip + } + if ip := r.Header.Get("X-Real-IP"); ip != "" { + return ip + } + ip, _, _ := net.SplitHostPort(r.RemoteAddr) + return ip +} diff --git a/pkg/service/utils_test.go b/pkg/service/utils_test.go index d46cb0707..99c19ac35 100644 --- a/pkg/service/utils_test.go +++ b/pkg/service/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 67310da29..955f2ecef 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build wireinject // +build wireinject @@ -18,7 +32,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" redisLiveKit "github.com/livekit/protocol/redis" "github.com/livekit/protocol/rpc" @@ -45,8 +58,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live telemetry.NewTelemetryService, getMessageBus, NewIOInfoService, - getEgressClient, - egress.NewRedisRPCClient, + rpc.NewEgressClient, getEgressStore, NewEgressLauncher, NewEgressService, @@ -89,10 +101,11 @@ func getNodeID(currentNode routing.LocalNode) livekit.NodeID { func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { // prefer keyfile if set if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return nil, err - } else if st.Mode().Perm() != 0600 { - return nil, fmt.Errorf("key file must have permission set to 600") + } else if st.Mode().Perm()&otherFilter != 0000 { + return nil, fmt.Errorf("key file others permissions must be set to 0") } f, err := os.Open(conf.KeyFile) if err != nil { @@ -114,7 +127,7 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { return auth.NewFileBasedKeyProviderFromMap(conf.Keys), nil } -func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.Notifier, error) { +func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook if len(wc.URLs) == 0 { return nil, nil @@ -124,7 +137,7 @@ func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webh return nil, ErrWebHookMissingAPIKey } - return webhook.NewNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { @@ -148,14 +161,6 @@ func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { return psrpc.NewRedisMessageBus(rc) } -func getEgressClient(conf *config.Config, nodeID livekit.NodeID, bus psrpc.MessageBus) (rpc.EgressClient, error) { - if conf.Egress.UsePsRPC { - return rpc.NewEgressClient(nodeID, bus) - } - - return nil, nil -} - func getEgressStore(s ObjectStore) EgressStore { switch store := s.(type) { case *RedisStore: diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 080858485..b051fb954 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -13,7 +13,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" redis2 "github.com/livekit/protocol/redis" "github.com/livekit/protocol/rpc" @@ -53,28 +52,27 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - egressClient, err := getEgressClient(conf, nodeID, messageBus) + egressClient, err := rpc.NewEgressClient(nodeID, messageBus) if err != nil { return nil, err } - rpcClient := egress.NewRedisRPCClient(nodeID, universalClient) egressStore := getEgressStore(objectStore) keyProvider, err := createKeyProvider(conf) if err != nil { return nil, err } - notifier, err := createWebhookNotifier(conf, keyProvider) + queuedNotifier, err := createWebhookNotifier(conf, keyProvider) if err != nil { return nil, err } analyticsService := telemetry.NewAnalyticsService(conf, currentNode) - telemetryService := telemetry.NewTelemetryService(notifier, analyticsService) - rtcEgressLauncher := NewEgressLauncher(egressClient, rpcClient, egressStore, telemetryService) + telemetryService := telemetry.NewTelemetryService(queuedNotifier, analyticsService) + rtcEgressLauncher := NewEgressLauncher(egressClient, egressStore, telemetryService) roomService, err := NewRoomService(roomConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher) if err != nil { return nil, err } - egressService := NewEgressService(egressClient, rpcClient, objectStore, egressStore, roomService, telemetryService, rtcEgressLauncher) + egressService := NewEgressService(egressClient, objectStore, egressStore, roomService, telemetryService, rtcEgressLauncher) ingressConfig := getIngressConfig(conf) ingressClient, err := rpc.NewIngressClient(nodeID, messageBus) if err != nil { @@ -82,7 +80,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live } ingressStore := getIngressStore(objectStore) ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, roomService, telemetryService) - ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService, rpcClient) + ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService) if err != nil { return nil, err } @@ -134,10 +132,11 @@ func getNodeID(currentNode routing.LocalNode) livekit.NodeID { func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return nil, err - } else if st.Mode().Perm() != 0600 { - return nil, fmt.Errorf("key file must have permission set to 600") + } else if st.Mode().Perm()&otherFilter != 0000 { + return nil, fmt.Errorf("key file others permissions must be set to 0") } f, err := os.Open(conf.KeyFile) if err != nil { @@ -159,7 +158,7 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { return auth.NewFileBasedKeyProviderFromMap(conf.Keys), nil } -func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.Notifier, error) { +func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook if len(wc.URLs) == 0 { return nil, nil @@ -169,7 +168,7 @@ func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webh return nil, ErrWebHookMissingAPIKey } - return webhook.NewNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { @@ -193,14 +192,6 @@ func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { return psrpc.NewRedisMessageBus(rc) } -func getEgressClient(conf *config.Config, nodeID livekit.NodeID, bus psrpc.MessageBus) (rpc.EgressClient, error) { - if conf.Egress.UsePsRPC { - return rpc.NewEgressClient(nodeID, bus) - } - - return nil, nil -} - func getEgressStore(s ObjectStore) EgressStore { switch store := s.(type) { case *RedisStore: diff --git a/pkg/service/wsprotocol.go b/pkg/service/wsprotocol.go index 06e971719..50a4dc2fb 100644 --- a/pkg/service/wsprotocol.go +++ b/pkg/service/wsprotocol.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/sfu/audio/audiolevel.go b/pkg/sfu/audio/audiolevel.go index 15b549c9c..7e834fda7 100644 --- a/pkg/sfu/audio/audiolevel.go +++ b/pkg/sfu/audio/audiolevel.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package audio import ( diff --git a/pkg/sfu/audio/audiolevel_test.go b/pkg/sfu/audio/audiolevel_test.go index aadd4ca29..8b8f03eba 100644 --- a/pkg/sfu/audio/audiolevel_test.go +++ b/pkg/sfu/audio/audiolevel_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package audio import ( diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index e27dfe50a..63a7c6289 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -10,6 +24,7 @@ import ( "github.com/gammazero/deque" "github.com/pion/rtcp" "github.com/pion/rtp" + "github.com/pion/rtp/codecs" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "go.uber.org/atomic" @@ -26,22 +41,22 @@ import ( ) const ( - ReportDelta = 1e9 + ReportDelta = time.Second ) type pendingPacket struct { - arrivalTime int64 + arrivalTime time.Time packet []byte } type ExtPacket struct { VideoLayer - Arrival int64 + Arrival time.Time Packet *rtp.Packet Payload interface{} KeyFrame bool RawPacket []byte - DependencyDescriptor *dd.DependencyDescriptor + DependencyDescriptor *ExtDependencyDescriptor } // Buffer contains all packets @@ -52,12 +67,12 @@ type Buffer struct { videoPool *sync.Pool audioPool *sync.Pool codecType webrtc.RTPCodecType - extPackets deque.Deque + extPackets deque.Deque[*ExtPacket] pPackets []pendingPacket closeOnce sync.Once mediaSSRC uint32 clockRate uint32 - lastReport int64 + lastReport time.Time twccExt uint8 audioLevelExt uint8 bound bool @@ -92,10 +107,9 @@ type Buffer struct { // logger logger logger.Logger - // depencency descriptor - ddExt uint8 - ddParser *DependencyDescriptorParser - maxLayerChangedCB func(int32, int32) + // dependency descriptor + ddExt uint8 + ddParser *DependencyDescriptorParser paused bool frameRateCalculator [DefaultMaxLayerSpatial + 1]FrameRateCalculator @@ -162,7 +176,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() b.clockRate = codec.ClockRate - b.lastReport = time.Now().UnixNano() + b.lastReport = time.Now() b.mime = strings.ToLower(codec.MimeType) for _, ext := range params.HeaderExtensions { @@ -174,9 +188,6 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) } b.ddParser = NewDependencyDescriptorParser(b.ddExt, b.logger, func(spatial, temporal int32) { - if b.maxLayerChangedCB != nil { - b.maxLayerChangedCB(spatial, temporal) - } frc.SetMaxLayer(spatial, temporal) }) @@ -193,8 +204,17 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili case strings.HasPrefix(b.mime, "video/"): b.codecType = webrtc.RTPCodecTypeVideo b.bucket = bucket.NewBucket(b.videoPool.Get().(*[]byte)) - if b.frameRateCalculator[0] == nil && strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) { - b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) + if b.frameRateCalculator[0] == nil { + if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) { + b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) + } + + if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP9) { + frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger) + for i := range b.frameRateCalculator { + b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) + } + } } default: @@ -224,7 +244,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili return } b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBNACK) - b.nacker = nack.NewNACKQueue() + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) } } @@ -250,12 +270,12 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { copy(packet, pkt) b.pPackets = append(b.pPackets, pendingPacket{ packet: packet, - arrivalTime: time.Now().UnixNano(), + arrivalTime: time.Now(), }) return } - b.calc(pkt, time.Now().UnixNano()) + b.calc(pkt, time.Now()) return } @@ -290,7 +310,7 @@ func (b *Buffer) ReadExtended(buf []byte) (*ExtPacket, error) { } b.Lock() if b.extPackets.Len() > 0 { - ep := b.extPackets.PopFront().(*ExtPacket) + ep := b.extPackets.PopFront() ep = b.patchExtPacket(ep, buf) if ep == nil { b.Unlock() @@ -382,7 +402,7 @@ func (b *Buffer) SetRTT(rtt uint32) { } } -func (b *Buffer) calc(pkt []byte, arrivalTime int64) { +func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) { pktBuf, err := b.bucket.AddPacket(pkt) if err != nil { // @@ -430,21 +450,21 @@ func (b *Buffer) calc(pkt []byte, arrivalTime int64) { func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { n, err := b.getPacket(buf, ep.Packet.SequenceNumber) if err != nil { - b.logger.Warnw("could not get packet", err, "sn", ep.Packet.SequenceNumber) + b.logger.Warnw("could not get packet", err, "sn", ep.Packet.SequenceNumber, "headSN", b.bucket.HeadSequenceNumber()) return nil } ep.RawPacket = buf[:n] // patch RTP packet to point payload to new buffer - rtp := *ep.Packet + pkt := *ep.Packet payloadStart := ep.Packet.Header.MarshalSize() payloadEnd := payloadStart + len(ep.Packet.Payload) if payloadEnd > n { b.logger.Warnw("unexpected marshal size", nil, "max", n, "need", payloadEnd) return nil } - rtp.Payload = buf[payloadStart:payloadEnd] - ep.Packet = &rtp + pkt.Payload = buf[payloadStart:payloadEnd] + ep.Packet = &pkt return ep } @@ -485,7 +505,7 @@ func (b *Buffer) doFpsCalc(ep *ExtPacket) { } } -func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) { +func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) { flowState := b.rtpStats.Update(&p.Header, len(p.Payload), int(p.PaddingSize), arrivalTime) if b.nacker != nil { @@ -499,12 +519,12 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) { } } -func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime int64) { +func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time) { // submit to TWCC even if it is a padding only packet. Clients use padding only packets as probes // for bandwidth estimation if b.twcc != nil && b.twccExt != 0 { if ext := p.GetExtension(b.twccExt); ext != nil { - b.twcc.Push(binary.BigEndian.Uint16(ext[0:2]), arrivalTime, p.Marker) + b.twcc.Push(binary.BigEndian.Uint16(ext[0:2]), arrivalTime.UnixNano(), p.Marker) } } @@ -529,7 +549,7 @@ func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime int64) { } } -func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPacket { +func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time) *ExtPacket { ep := &ExtPacket{ Packet: rtpPacket, Arrival: arrivalTime, @@ -550,7 +570,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack if err == nil && ddVal != nil { ep.DependencyDescriptor = ddVal ep.VideoLayer = videoLayer - // TODO : notify active decode target change if changed. + // DD-TODO : notify active decode target change if changed. } } switch b.mime { @@ -566,13 +586,28 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack } else { // vp8 with DependencyDescriptor enabled, use the TID from the descriptor vp8Packet.TID = uint8(ep.Temporal) - ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to -1 + ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid } ep.Payload = vp8Packet + case "video/vp9": + if ep.DependencyDescriptor == nil { + var vp9Packet codecs.VP9Packet + _, err := vp9Packet.Unmarshal(rtpPacket.Payload) + if err != nil { + b.logger.Warnw("could not unmarshal VP9 packet", err) + return nil + } + ep.VideoLayer = VideoLayer{ + Spatial: int32(vp9Packet.SID), + Temporal: int32(vp9Packet.TID), + } + ep.Payload = vp9Packet + } + ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload) case "video/h264": - ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload) + ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload) case "video/av1": - ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload) + ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload) } if ep.KeyFrame { @@ -599,8 +634,8 @@ func (b *Buffer) doNACKs() { } } -func (b *Buffer) doReports(arrivalTime int64) { - timeDiff := arrivalTime - b.lastReport +func (b *Buffer) doReports(arrivalTime time.Time) { + timeDiff := arrivalTime.Sub(b.lastReport) if timeDiff < ReportDelta { return } @@ -638,7 +673,7 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { srData := &RTCPSenderReportData{ RTPTimestamp: rtpTime, NTPTimestamp: mediatransportutil.NtpTime(ntpTime), - ArrivalTime: time.Now(), + At: time.Now(), } b.RLock() @@ -652,15 +687,15 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { } } -func (b *Buffer) GetSenderReportDataExt() *RTCPSenderReportDataExt { +func (b *Buffer) GetSenderReportData() (*RTCPSenderReportData, *RTCPSenderReportData) { b.RLock() defer b.RUnlock() if b.rtpStats != nil { - return b.rtpStats.GetRtcpSenderReportDataExt() + return b.rtpStats.GetRtcpSenderReportData() } - return nil + return nil, nil } func (b *Buffer) SetLastFractionLostReport(lost uint8) { @@ -763,12 +798,6 @@ func (b *Buffer) GetAudioLevel() (float64, bool) { return b.audioLevel.GetLevel() } -// TODO : now we rely on stream tracker for layer change, dependency still -// work for that too. Do we keep it unchange or use both methods? -func (b *Buffer) OnMaxLayerChanged(fn func(int32, int32)) { - b.maxLayerChangedCB = fn -} - func (b *Buffer) OnFpsChanged(f func()) { b.Lock() b.onFpsChanged = f diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index a3207336e..7f1e97357 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -6,11 +20,12 @@ import ( "testing" "time" - "github.com/livekit/mediatransportutil/pkg/nack" "github.com/pion/rtcp" "github.com/pion/rtp" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" + + "github.com/livekit/mediatransportutil/pkg/nack" ) var vp8Codec = webrtc.RTPCodecParameters{ @@ -68,7 +83,7 @@ func TestNack(t *testing.T) { continue } if i < 14 { - time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.BackoffFactor, float64(i))+10) * time.Millisecond) + time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond) } else { time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries } @@ -127,7 +142,7 @@ func TestNack(t *testing.T) { continue } if i < 14 { - time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.BackoffFactor, float64(i))+10) * time.Millisecond) + time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond) } else { time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries } diff --git a/pkg/sfu/buffer/datastats.go b/pkg/sfu/buffer/datastats.go index 880e172af..515341cc0 100644 --- a/pkg/sfu/buffer/datastats.go +++ b/pkg/sfu/buffer/datastats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/datastats_test.go b/pkg/sfu/buffer/datastats_test.go index 72992d35f..5803fe9d9 100644 --- a/pkg/sfu/buffer/datastats_test.go +++ b/pkg/sfu/buffer/datastats_test.go @@ -1,12 +1,27 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( "testing" "time" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + + "github.com/livekit/protocol/livekit" ) func TestDataStats(t *testing.T) { diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 755e2cd4d..a3a2be794 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -1,102 +1,181 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( "fmt" + "sort" "github.com/pion/rtp" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" ) type DependencyDescriptorParser struct { structure *dd.FrameDependencyStructure - ddExt uint8 + ddExtID uint8 logger logger.Logger onMaxLayerChanged func(int32, int32) - decodeTargetLayer []VideoLayer + decodeTargets []DependencyDescriptorDecodeTarget + + seqWrapAround *utils.WrapAround[uint16, uint64] + frameWrapAround *utils.WrapAround[uint16, uint64] + structureExtSeq uint64 + activeDecodeTargetsExtSeq uint64 + activeDecodeTargetsMask uint32 + frameChecker *FrameIntegrityChecker } -func NewDependencyDescriptorParser(ddExt uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { - logger.Infow("creating dependency descriptor parser", "ddExt", ddExt) +func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { + logger.Infow("creating dependency descriptor parser", "ddExtID", ddExtID) return &DependencyDescriptorParser{ - ddExt: ddExt, + ddExtID: ddExtID, logger: logger, onMaxLayerChanged: onMaxLayerChanged, + seqWrapAround: utils.NewWrapAround[uint16, uint64](), + frameWrapAround: utils.NewWrapAround[uint16, uint64](), + frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video } } -func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*dd.DependencyDescriptor, VideoLayer, error) { +type ExtDependencyDescriptor struct { + Descriptor *dd.DependencyDescriptor + + DecodeTargets []DependencyDescriptorDecodeTarget + StructureUpdated bool + ActiveDecodeTargetsUpdated bool + Integrity bool + ExtFrameNum uint64 +} + +func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) { var videoLayer VideoLayer - if ddBuf := pkt.GetExtension(r.ddExt); ddBuf != nil { - var ddVal dd.DependencyDescriptor - ext := &dd.DependencyDescriptorExtension{ - Descriptor: &ddVal, - Structure: r.structure, - } - _, err := ext.Unmarshal(ddBuf) - if err != nil { - // r.logger.Debugw("failed to parse generic dependency descriptor", "err", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf)) - return nil, videoLayer, err - } + ddBuf := pkt.GetExtension(r.ddExtID) + if ddBuf == nil { + return nil, videoLayer, nil + } - if ddVal.FrameDependencies != nil { - videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) - } - if ddVal.AttachedStructure != nil && !ddVal.FirstPacketInFrame { - // r.logger.Debugw("ignoring non-first packet in frame with attached structure") - return nil, videoLayer, nil - } + var ddVal dd.DependencyDescriptor + ext := &dd.DependencyDescriptorExtension{ + Descriptor: &ddVal, + Structure: r.structure, + } + _, err := ext.Unmarshal(ddBuf) + if err != nil { + // r.logger.Debugw("failed to parse generic dependency descriptor", "err", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf)) + return nil, videoLayer, err + } - if ddVal.AttachedStructure != nil { - var maxSpatial, maxTemporal int32 + extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal + + if ddVal.FrameDependencies != nil { + videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) + } + + extFN := r.frameWrapAround.Update(ddVal.FrameNumber).ExtendedVal + r.frameChecker.AddPacket(extSeq, extFN, &ddVal) + + extDD := &ExtDependencyDescriptor{ + Descriptor: &ddVal, + ExtFrameNum: extFN, + Integrity: r.frameChecker.FrameIntegrity(extFN), + } + + if ddVal.AttachedStructure != nil { + r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String())) + if extSeq > r.structureExtSeq { r.structure = ddVal.AttachedStructure - r.decodeTargetLayer = r.decodeTargetLayer[:0] - for target := 0; target < r.structure.NumDecodeTargets; target++ { - layer := VideoLayer{0, 0} - for _, t := range r.structure.Templates { - if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { - if layer.Spatial < int32(t.SpatialId) { - layer.Spatial = int32(t.SpatialId) - } - if layer.Temporal < int32(t.TemporalId) { - layer.Temporal = int32(t.TemporalId) - } - } - } - if layer.Spatial > maxSpatial { - maxSpatial = layer.Spatial - } - if layer.Temporal > maxTemporal { - maxTemporal = layer.Temporal - } - r.decodeTargetLayer = append(r.decodeTargetLayer, layer) - } - r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal) - go r.onMaxLayerChanged(maxSpatial, maxTemporal) + r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure) + r.structureExtSeq = extSeq + extDD.StructureUpdated = true + extDD.ActiveDecodeTargetsUpdated = true + // The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present, + // so don't need to notify max layer change here. } + } - if ddVal.AttachedStructure != nil && ddVal.FirstPacketInFrame { - r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String())) - } - - if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil { + if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil && extSeq > r.activeDecodeTargetsExtSeq { + r.activeDecodeTargetsExtSeq = extSeq + if *mask != r.activeDecodeTargetsMask { + r.activeDecodeTargetsMask = *mask + extDD.ActiveDecodeTargetsUpdated = true var maxSpatial, maxTemporal int32 - for dt, layer := range r.decodeTargetLayer { - if *mask&(1<= int32(len(f.frameRates)) { f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal) @@ -113,9 +123,9 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { return f.calc() } -func (f *FrameRateCalculatorVP8) calc() bool { +func (f *frameRateCalculatorVPx) calc() bool { var rateCounter int - for currentTemporal := int32(0); currentTemporal <= int32(DefaultMaxLayerTemporal); currentTemporal++ { + for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ { if f.frameRates[currentTemporal] > 0 { rateCounter++ continue @@ -156,14 +166,13 @@ func (f *FrameRateCalculatorVP8) calc() bool { if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 { f.frameRates[1] = f.frameRates[2] / 2 } - f.logger.Debugw("frame rate calculated", "rate", f.frameRates) f.reset() return true } return false } -func (f *FrameRateCalculatorVP8) reset() { +func (f *frameRateCalculatorVPx) reset() { for i := range f.firstFrames { f.firstFrames[i] = nil f.secondFrames[i] = nil @@ -175,20 +184,146 @@ func (f *FrameRateCalculatorVP8) reset() { f.baseFrame = nil } -func (f *FrameRateCalculatorVP8) GetFrameRate() (bool, []float32) { +func (f *frameRateCalculatorVPx) GetFrameRate() (bool, []float32) { return f.completed, f.frameRates[:] } // ----------------------------- -// FrameRateCalculator based on Dependency descriptor +// FrameRateCalculator based on PictureID in VP8 +type FrameRateCalculatorVP8 struct { + *frameRateCalculatorVPx + logger logger.Logger +} + +func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 { + return &FrameRateCalculatorVP8{ + frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger), + logger: logger, + } +} + +func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { + if f.frameRateCalculatorVPx.Completed() { + return true + } + + vp8, ok := ep.Payload.(VP8) + if !ok { + f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) + return false + } + success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID) + + if f.frameRateCalculatorVPx.Completed() { + _, rate := f.frameRateCalculatorVPx.GetFrameRate() + f.logger.Debugw("frame rate calculated", "rate", rate) + } + + return success +} + +// ----------------------------- + +// FrameRateCalculator based on PictureID in VP9 +type FrameRateCalculatorVP9 struct { + logger logger.Logger + completed bool + + // VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX. + // Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured + frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx +} + +func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 { + f := &FrameRateCalculatorVP9{ + logger: logger, + } + + for i := range f.frameRateCalculatorsVPx { + f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger) + } + + return f +} + +func (f *FrameRateCalculatorVP9) Completed() bool { + return f.completed +} + +func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool { + if f.completed { + return true + } + + vp9, ok := ep.Payload.(codecs.VP9Packet) + if !ok { + f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber) + return false + } + + if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil { + f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial) + return false + } + + success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID) + + completed := true + for _, frc := range f.frameRateCalculatorsVPx { + if !frc.Completed() { + completed = false + break + } + } + + if completed { + f.completed = true + + var frameRates [DefaultMaxLayerSpatial + 1][]float32 + for i := range f.frameRateCalculatorsVPx { + _, frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate() + } + f.logger.Debugw("frame rate calculated", "rate", frameRates) + } + + return success +} + +func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) (bool, []float32) { + if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil { + return false, nil + } + return f.frameRateCalculatorsVPx[spatial].GetFrameRate() +} + +func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer { + return &FrameRateCalculatorForVP9Layer{ + FrameRateCalculatorVP9: f, + spatial: spatial, + } +} + +// ----------------------------- + +type FrameRateCalculatorForVP9Layer struct { + *FrameRateCalculatorVP9 + spatial int32 +} + +func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() (bool, []float32) { + return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial) +} + +// ----------------------------------------------- + +// FrameRateCalculator based on Dependency descriptor type FrameRateCalculatorDD struct { frameRates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32 clockRate uint32 logger logger.Logger firstFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo - spatial int fnReceived [256]*frameInfo baseFrame *frameInfo completed bool @@ -222,7 +357,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { } if ep.DependencyDescriptor == nil { - f.logger.Infow("dependency descriptor is nil") + f.logger.Debugw("dependency descriptor is nil") return false } @@ -237,7 +372,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { return false } - fn := ep.DependencyDescriptor.FrameNumber + fn := ep.DependencyDescriptor.Descriptor.FrameNumber if f.baseFrame == nil { f.baseFrame = &frameInfo{seq: ep.Packet.SequenceNumber, ts: ep.Packet.Timestamp, fn: fn} f.fnReceived[0] = f.baseFrame @@ -277,7 +412,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { fn: fn, temporal: temporal, spatial: spatial, - frameDiff: ep.DependencyDescriptor.FrameDependencies.FrameDiffs, + frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs, } f.fnReceived[baseDiff] = fi @@ -291,7 +426,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { if chain.Len() == 0 { chain.PushBack(fn) } - for _, fdiff := range ep.DependencyDescriptor.FrameDependencies.FrameDiffs { + for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs { dependFrame := fn - uint16(fdiff) // frame too old, ignore if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 { @@ -385,7 +520,7 @@ func (f *FrameRateCalculatorDD) calc() bool { f.completed = true f.close() - f.logger.Debugw("frame rate calculated", "spatial", f.spatial, "rate", f.frameRates) + f.logger.Debugw("frame rate calculated", "rate", f.frameRates) return true } return false @@ -424,6 +559,8 @@ func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) } } +// ----------------------------------------------- + type FrameRateCalculatorForDDLayer struct { *FrameRateCalculatorDD spatial int32 @@ -432,3 +569,5 @@ type FrameRateCalculatorForDDLayer struct { func (f *FrameRateCalculatorForDDLayer) GetFrameRate() (bool, []float32) { return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial) } + +// ----------------------------------------------- diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index a28f1073b..4a85d2808 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -31,10 +45,12 @@ func (f *testFrameInfo) toVP8() *ExtPacket { func (f *testFrameInfo) toDD() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - DependencyDescriptor: &dependencydescriptor.DependencyDescriptor{ - FrameNumber: f.framenumber, - FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ - FrameDiffs: f.frameDiff, + DependencyDescriptor: &ExtDependencyDescriptor{ + Descriptor: &dependencydescriptor.DependencyDescriptor{ + FrameNumber: f.framenumber, + FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ + FrameDiffs: f.frameDiff, + }, }, }, VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)}, @@ -147,7 +163,7 @@ func TestFpsVP8(t *testing.T) { testCase := c t.Run(name, func(t *testing.T) { fps := testCase.fps - frames := [][]*testFrameInfo{} + frames := make([][]*testFrameInfo, 0) vp8calcs := make([]*FrameRateCalculatorVP8, len(fps)) for i := range vp8calcs { vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger()) @@ -178,7 +194,7 @@ func TestFpsVP8(t *testing.T) { } t.Run("packet lost and duplicate", func(t *testing.T) { fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}} - frames := [][]*testFrameInfo{} + frames := make([][]*testFrameInfo, 0) vp8calcs := make([]*FrameRateCalculatorVP8, len(fps)) for i := range vp8calcs { vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger()) diff --git a/pkg/sfu/buffer/frameintegrity.go b/pkg/sfu/buffer/frameintegrity.go new file mode 100644 index 000000000..1b712652e --- /dev/null +++ b/pkg/sfu/buffer/frameintegrity.go @@ -0,0 +1,211 @@ +package buffer + +import ( + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +type FrameEntity struct { + startSeq *uint64 + endSeq *uint64 + integrity bool + + packetsConsective func(uint64, uint64) bool +} + +func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) { + // duplicate packet + if fe.integrity { + return + } + + if fe.startSeq == nil && ddVal.FirstPacketInFrame { + fe.startSeq = &extSeq + } + if fe.endSeq == nil && ddVal.LastPacketInFrame { + fe.endSeq = &extSeq + } + + if fe.startSeq != nil && fe.endSeq != nil { + if fe.packetsConsective(*fe.startSeq, *fe.endSeq) { + fe.integrity = true + } + } +} + +func (fe *FrameEntity) Reset() { + fe.integrity = false + fe.startSeq, fe.endSeq = nil, nil +} + +func (fe *FrameEntity) Integrity() bool { + return fe.integrity +} + +// ------------------------------ + +type PacketHistory struct { + base uint64 + last uint64 + bits []uint64 + packetCount int + inited bool +} + +func NewPacketHistory(packetCount int) *PacketHistory { + packetCount = (packetCount + 63) / 64 * 64 + return &PacketHistory{ + bits: make([]uint64, packetCount/64), + packetCount: packetCount, + } +} + +func (ph *PacketHistory) AddPacket(extSeq uint64) { + if !ph.inited { + ph.inited = true + ph.base = uint64(extSeq) + // set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped + if ph.base > 100 { + ph.base -= 100 + } else { + ph.base = 0 + } + ph.last = uint64(extSeq) + ph.set(extSeq, true) + return + } + + if extSeq <= ph.base { + // too old + return + } + + if extSeq <= ph.last { + if ph.last-extSeq < uint64(ph.packetCount) { + ph.set(extSeq, true) + } + return + } + + for i := ph.last + 1; i < extSeq; i++ { + ph.set(i, false) + } + + ph.set(extSeq, true) + ph.last = extSeq +} + +func (ph *PacketHistory) getPos(seq uint64) (index, offset int) { + idx := (seq - ph.base) % uint64(ph.packetCount) + return int(idx >> 6), int(idx % 64) +} + +func (ph *PacketHistory) set(seq uint64, received bool) { + idx, offset := ph.getPos(seq) + if !received { + ph.bits[idx] &= ^(1 << offset) + } else { + ph.bits[idx] |= 1 << (offset) + } +} + +func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool { + if start > end { + return false + } + + if end-start >= uint64(ph.packetCount) { + return false + } + + startIndex, startOffset := ph.getPos(start) + endIndex, endOffset := ph.getPos(end) + + if startIndex == endIndex && end-start <= 64 { + testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset + return ph.bits[startIndex]&testBits == testBits + } + + if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) { + return false + } + + for i := startIndex + 1; i != endIndex; i++ { + if i == len(ph.bits) { + i = 0 + if i == endIndex { + break + } + } + if ph.bits[i]+1 != 0 { + return false + } + } + + testBits := uint64((1 << (endOffset + 1)) - 1) + return ph.bits[endIndex]&testBits == testBits +} + +// ------------------------------ + +type FrameIntegrityChecker struct { + frameCount int + frames []FrameEntity + base uint64 + last uint64 + + pktHistory *PacketHistory + inited bool +} + +func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker { + fc := &FrameIntegrityChecker{ + frames: make([]FrameEntity, frameCount), + pktHistory: NewPacketHistory(packetCount), + frameCount: frameCount, + } + + for i := range fc.frames { + fc.frames[i].packetsConsective = fc.pktHistory.PacketsConsecutive + fc.frames[i].Reset() + } + return fc +} + +func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) { + fc.pktHistory.AddPacket(extSeq) + + if !fc.inited { + fc.inited = true + fc.base = extFrameNum + fc.last = extFrameNum + } + + if extFrameNum < fc.base { + // frame too old + return + } + + if extFrameNum <= fc.last { + if fc.last-extFrameNum >= uint64(fc.frameCount) { + // frame too old + return + } + fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal) + return + } + + // reset missing frames + for i := fc.last + 1; i <= extFrameNum; i++ { + fc.frames[int(i-fc.base)%fc.frameCount].Reset() + } + fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal) + fc.last = extFrameNum +} + +func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool { + if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) { + return false + } + + return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity() +} diff --git a/pkg/sfu/buffer/frameintegrity_test.go b/pkg/sfu/buffer/frameintegrity_test.go new file mode 100644 index 000000000..3e505efdd --- /dev/null +++ b/pkg/sfu/buffer/frameintegrity_test.go @@ -0,0 +1,72 @@ +package buffer + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +func TestFrameIntegrityChecker(t *testing.T) { + fc := NewFrameIntegrityChecker(100, 1000) + + // first frame out of order + fc.AddPacket(10, 10, &dependencydescriptor.DependencyDescriptor{}) + require.False(t, fc.FrameIntegrity(10)) + fc.AddPacket(9, 10, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(10)) + fc.AddPacket(11, 10, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(10)) + + // single packet frame + fc.AddPacket(100, 100, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(100)) + require.False(t, fc.FrameIntegrity(101)) + require.False(t, fc.FrameIntegrity(99)) + + // frame too old than first frame + fc.AddPacket(99, 99, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + + // multiple packet frame, out of order + fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + require.False(t, fc.FrameIntegrity(2001)) + require.False(t, fc.FrameIntegrity(1999)) + // out of frame count(100) + require.False(t, fc.FrameIntegrity(100)) + require.False(t, fc.FrameIntegrity(1900)) + + fc.AddPacket(2000, 2001, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(2001)) + fc.AddPacket(2002, 2001, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(2001)) + // duplicate packet + fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + require.True(t, fc.FrameIntegrity(2001)) + + // frame too old + fc.AddPacket(900, 1900, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(1900)) + + for frame := uint64(2002); frame < 2102; frame++ { + // large frame (1000 packets) out of order / retransmitted + firstFrame := uint64(3000 + (frame-2002)*1000) + lastFrame := uint64(3999 + (frame-2002)*1000) + frames := make([]uint64, 0, lastFrame-firstFrame+1) + for i := firstFrame; i <= lastFrame; i++ { + frames = append(frames, i) + } + require.False(t, fc.FrameIntegrity(frame)) + rand.Seed(int64(frame)) + rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] }) + for i, f := range frames { + fc.AddPacket(f, frame, &dependencydescriptor.DependencyDescriptor{ + FirstPacketInFrame: f == firstFrame, + LastPacketInFrame: f == lastFrame, + }) + require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i) + } + require.True(t, fc.FrameIntegrity(frame)) + } +} diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index cacc7ad1f..f52464b4c 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -33,22 +47,23 @@ var ( */ type VP8 struct { FirstByte byte + S bool - PictureIDPresent int - PictureID uint16 /* 8 or 16 bits, picture ID */ - MBit bool + I bool + M bool + PictureID uint16 /* 8 or 16 bits, picture ID */ - TL0PICIDXPresent int - TL0PICIDX uint8 /* 8 bits temporal level zero index */ + L bool + TL0PICIDX uint8 /* 8 bits temporal level zero index */ // Optional Header If either of the T or K bits are set to 1, // the TID/Y/KEYIDX extension field MUST be present. - TIDPresent int - TID uint8 /* 2 bits temporal layer idx */ - Y uint8 + T bool + TID uint8 /* 2 bits temporal layer idx */ + Y bool - KEYIDXPresent int - KEYIDX uint8 /* 5 bits of key frame idx */ + K bool + KEYIDX uint8 /* 5 bits of key frame idx */ HeaderSize int @@ -63,96 +78,94 @@ func (v *VP8) Unmarshal(payload []byte) error { } payloadLen := len(payload) - if payloadLen < 1 { return errShortPacket } idx := 0 v.FirstByte = payload[idx] - S := payload[idx]&0x10 > 0 + v.S = payload[idx]&0x10 > 0 // Check for extended bit control if payload[idx]&0x80 > 0 { idx++ if payloadLen < idx+1 { return errShortPacket } - I := payload[idx]&0x80 > 0 - L := payload[idx]&0x40 > 0 - T := payload[idx]&0x20 > 0 - K := payload[idx]&0x10 > 0 - if L && !T { + v.I = payload[idx]&0x80 > 0 + v.L = payload[idx]&0x40 > 0 + v.T = payload[idx]&0x20 > 0 + v.K = payload[idx]&0x10 > 0 + if v.L && !v.T { return errInvalidPacket } - // Check for PictureID - if I { + + if v.I { idx++ if payloadLen < idx+1 { return errShortPacket } - v.PictureIDPresent = 1 pid := payload[idx] & 0x7f - // Check if m is 1, then Picture ID is 15 bits - if payload[idx]&0x80 > 0 { + // if m is 1, then Picture ID is 15 bits + v.M = payload[idx]&0x80 > 0 + if v.M { idx++ if payloadLen < idx+1 { return errShortPacket } - v.MBit = true v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]}) } else { v.PictureID = uint16(pid) } } - // Check if TL0PICIDX is present - if L { + + if v.L { idx++ if payloadLen < idx+1 { return errShortPacket } - v.TL0PICIDXPresent = 1 - - if idx >= payloadLen { - return errShortPacket - } v.TL0PICIDX = payload[idx] } - if T || K { + + if v.T || v.K { idx++ if payloadLen < idx+1 { return errShortPacket } - if T { - v.TIDPresent = 1 + + if v.T { v.TID = (payload[idx] & 0xc0) >> 6 - v.Y = (payload[idx] & 0x20) >> 5 + v.Y = (payload[idx] & 0x20) > 0 } - if K { - v.KEYIDXPresent = 1 + + if v.K { v.KEYIDX = payload[idx] & 0x1f } } - if idx >= payloadLen { - return errShortPacket - } idx++ if payloadLen < idx+1 { return errShortPacket } + // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && S + v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S } else { idx++ if payloadLen < idx+1 { return errShortPacket } // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && S + v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S } v.HeaderSize = idx return nil } +func (v *VP8) Marshal() ([]byte, error) { + buf := make([]byte, v.HeaderSize) + err := v.MarshalTo(buf) + return buf, err +} + func (v *VP8) MarshalTo(buf []byte) error { if len(buf) < v.HeaderSize { return errShortPacket @@ -160,13 +173,17 @@ func (v *VP8) MarshalTo(buf []byte) error { idx := 0 buf[idx] = v.FirstByte - if (v.PictureIDPresent + v.TL0PICIDXPresent + v.TIDPresent + v.KEYIDXPresent) != 0 { + if v.I || v.L || v.T || v.K { buf[idx] |= 0x80 // X bit idx++ - buf[idx] = byte(v.PictureIDPresent<<7) | byte(v.TL0PICIDXPresent<<6) | byte(v.TIDPresent<<5) | byte(v.KEYIDXPresent<<4) + + xpos := idx + xval := byte(0) + idx++ - if v.PictureIDPresent == 1 { - if v.MBit { + if v.I { + xval |= (1 << 7) + if v.M { buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f) buf[idx+1] = byte(v.PictureID & 0xff) idx += 2 @@ -175,20 +192,31 @@ func (v *VP8) MarshalTo(buf []byte) error { idx++ } } - if v.TL0PICIDXPresent == 1 { + + if v.L { + xval |= (1 << 6) buf[idx] = v.TL0PICIDX idx++ } - if v.TIDPresent == 1 || v.KEYIDXPresent == 1 { + + if v.T || v.K { buf[idx] = 0 - if v.TIDPresent == 1 { - buf[idx] = v.TID<<6 | v.Y<<5 + if v.T { + xval |= (1 << 5) + buf[idx] = v.TID << 6 + if v.Y { + buf[idx] |= (1 << 5) + } } - if v.KEYIDXPresent == 1 { + + if v.K { + xval |= (1 << 4) buf[idx] |= v.KEYIDX & 0x1f } idx++ } + + buf[xpos] = xval } else { buf[idx] &^= 0x80 // X bit idx++ @@ -197,7 +225,9 @@ func (v *VP8) MarshalTo(buf []byte) error { return nil } -func VP8PictureIdSizeDiff(mBit1 bool, mBit2 bool) int { +// ------------------------------------- + +func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int { if mBit1 == mBit2 { return 0 } @@ -209,10 +239,12 @@ func VP8PictureIdSizeDiff(mBit1 bool, mBit2 bool) int { return -1 } -// IsH264Keyframe detects if h264 payload is a keyframe +// ------------------------------------- + +// IsH264KeyFrame detects if h264 payload is a keyframe // this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45 // all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsH264Keyframe(payload []byte) bool { +func IsH264KeyFrame(payload []byte) bool { if len(payload) < 1 { return false } @@ -276,10 +308,65 @@ func IsH264Keyframe(payload []byte) bool { return false } -// IsAV1Keyframe detects if av1 payload is a keyframe +// ------------------------------------- + +func IsVP9KeyFrame(payload []byte) bool { + payloadLen := len(payload) + if payloadLen < 1 { + return false + } + + idx := 0 + I := payload[idx]&0x80 > 0 + P := payload[idx]&0x40 > 0 + L := payload[idx]&0x20 > 0 + F := payload[idx]&0x10 > 0 + B := payload[idx]&0x08 > 0 + + if F && !I { + return false + } + + // Check for PictureID + if I { + idx++ + if payloadLen < idx+1 { + return false + } + // Check if m is 1, then Picture ID is 15 bits + if payload[idx]&0x80 > 0 { + idx++ + if payloadLen < idx+1 { + return false + } + } + } + + // Check if TL0PICIDX is present + sid := -1 + if L { + idx++ + if payloadLen < idx+1 { + return false + } + + tid := (payload[idx] >> 5) & 0x7 + if !P && tid != 0 { + return false + } + + sid = int((payload[idx] >> 1) & 0x7) + } + + return !P && (!L || (L && sid == 0)) && B +} + +// ------------------------------------- + +// IsAV1KeyFrame detects if av1 payload is a keyframe // taken from https://github.com/jech/galene/blob/master/codecs/codecs.go // all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsAV1Keyframe(payload []byte) bool { +func IsAV1KeyFrame(payload []byte) bool { if len(payload) < 2 { return false } diff --git a/pkg/sfu/buffer/helpers_test.go b/pkg/sfu/buffer/helpers_test.go index d52bce5c5..378bfbebf 100644 --- a/pkg/sfu/buffer/helpers_test.go +++ b/pkg/sfu/buffer/helpers_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -75,7 +89,7 @@ func TestVP8Helper_Unmarshal(t *testing.T) { t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) } if tt.checkTemporal { - require.Equal(t, tt.temporalSupport, p.TIDPresent == 1) + require.Equal(t, tt.temporalSupport, p.T) } if tt.checkKeyFrame { require.Equal(t, tt.keyFrame, p.IsKeyFrame) diff --git a/pkg/sfu/buffer/rtcpreader.go b/pkg/sfu/buffer/rtcpreader.go index 5d322abcd..1f8f365df 100644 --- a/pkg/sfu/buffer/rtcpreader.go +++ b/pkg/sfu/buffer/rtcpreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 9a0ba637a..cd9c96066 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1,6 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( + "errors" "fmt" "math" "sync" @@ -19,11 +34,32 @@ const ( GapHistogramNumBins = 101 NumSequenceNumbers = 65536 FirstSnapshotId = 1 - SnInfoSize = 2048 + SnInfoSize = 8192 SnInfoMask = SnInfoSize - 1 - TooLargeOWD = 400 * time.Millisecond ) +// ------------------------------------------------------- + +type driftResult struct { + timeSinceFirst time.Duration + rtpDiffSinceFirst uint64 + driftSamples int64 + driftMs float64 + sampleRate float64 +} + +func (d driftResult) String() string { + return fmt.Sprintf("time: %+v, rtp: %d, driftSamples: %d, driftMs: %.02f, sampleRate: %.02f", + d.timeSinceFirst, + d.rtpDiffSinceFirst, + d.driftSamples, + d.driftMs, + d.sampleRate, + ) +} + +// ------------------------------------------------------- + type RTPFlowState struct { HasLoss bool LossStartInclusive uint16 @@ -38,6 +74,7 @@ type IntervalStats struct { bytesPadding uint64 headerBytesPadding uint64 packetsLost uint32 + packetsOutOfOrder uint32 frames uint32 } @@ -55,6 +92,7 @@ type RTPDeltaInfo struct { HeaderBytesPadding uint64 PacketsLost uint32 PacketsMissing uint32 + PacketsOutOfOrder uint32 Frames uint32 RttMax uint32 JitterMax float64 @@ -66,6 +104,7 @@ type RTPDeltaInfo struct { type Snapshot struct { startTime time.Time extStartSN uint32 + extStartSNOverridden uint32 packetsDuplicate uint32 bytesDuplicate uint64 headerBytesDuplicate uint64 @@ -79,22 +118,18 @@ type Snapshot struct { } type SnInfo struct { - pktTime int64 hdrSize uint16 pktSize uint16 isPaddingOnly bool marker bool + isOutOfOrder bool } type RTCPSenderReportData struct { - RTPTimestamp uint32 - NTPTimestamp mediatransportutil.NtpTime - ArrivalTime time.Time -} - -type RTCPSenderReportDataExt struct { - SenderReportData RTCPSenderReportData - SmoothedOWD time.Duration + RTPTimestamp uint32 + RTPTimestampExt uint64 + NTPTimestamp mediatransportutil.NtpTime + At time.Time } type RTPStatsParams struct { @@ -123,10 +158,15 @@ type RTPStats struct { lastRRTime time.Time lastRR rtcp.ReceptionReport - highestTS uint32 - highestTime int64 + extStartTS uint64 + highestTS uint32 + tsCycles uint32 - lastTransit uint32 + firstTime time.Time + highestTime time.Time + + lastTransit uint32 + lastJitterRTP uint32 bytes uint64 headerBytes uint64 @@ -174,7 +214,8 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srDataExt *RTCPSenderReportDataExt + srFirst *RTCPSenderReportData + srNewest *RTCPSenderReportData nextSnapshotId uint32 snapshots map[uint32]*Snapshot @@ -201,7 +242,7 @@ func (r *RTPStats) Seed(from *RTPStats) { r.resyncOnNextPacket = from.resyncOnNextPacket r.startTime = from.startTime - // do not clone endTime as a non-zero endTime indiacates an ended object + // do not clone endTime as a non-zero endTime indicates an ended object r.extStartSN = from.extStartSN r.highestSN = from.highestSN @@ -211,10 +252,15 @@ func (r *RTPStats) Seed(from *RTPStats) { r.lastRRTime = from.lastRRTime r.lastRR = from.lastRR + r.extStartTS = from.extStartTS r.highestTS = from.highestTS + r.tsCycles = from.tsCycles + + r.firstTime = from.firstTime r.highestTime = from.highestTime r.lastTransit = from.lastTransit + r.lastJitterRTP = from.lastJitterRTP r.bytes = from.bytes r.headerBytes = from.headerBytes @@ -262,13 +308,17 @@ func (r *RTPStats) Seed(from *RTPStats) { r.rtt = from.rtt r.maxRtt = from.maxRtt - if from.srDataExt != nil { - r.srDataExt = &RTCPSenderReportDataExt{ - SenderReportData: from.srDataExt.SenderReportData, - SmoothedOWD: from.srDataExt.SmoothedOWD, - } + if from.srFirst != nil { + srFirst := *from.srFirst + r.srFirst = &srFirst } else { - r.srDataExt = nil + r.srFirst = nil + } + if from.srNewest != nil { + srNewest := *from.srNewest + r.srNewest = &srNewest + } else { + r.srNewest = nil } r.nextSnapshotId = from.nextSnapshotId @@ -295,7 +345,11 @@ func (r *RTPStats) NewSnapshotId() uint32 { id := r.nextSnapshotId if r.initialized { - r.snapshots[id] = &Snapshot{startTime: time.Now(), extStartSN: r.extStartSN} + r.snapshots[id] = &Snapshot{ + startTime: time.Now(), + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, + } } r.nextSnapshotId++ @@ -310,7 +364,7 @@ func (r *RTPStats) IsActive() bool { return r.initialized && r.endTime.IsZero() } -func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, packetTime int64) (flowState RTPFlowState) { +func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, packetTime time.Time) (flowState RTPFlowState) { r.lock.Lock() defer r.lock.Unlock() @@ -324,18 +378,26 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.startTime = time.Now() - r.highestSN = rtph.SequenceNumber - 1 - r.highestTS = rtph.Timestamp - r.highestTime = packetTime - r.extStartSN = uint32(rtph.SequenceNumber) + r.highestSN = rtph.SequenceNumber - 1 r.cycles = 0 + r.extStartTS = uint64(rtph.Timestamp) + r.highestTS = rtph.Timestamp + r.tsCycles = 0 + + r.firstTime = packetTime + r.highestTime = packetTime + first = true // initialize snapshots if any for i := uint32(FirstSnapshotId); i < r.nextSnapshotId; i++ { - r.snapshots[i] = &Snapshot{startTime: r.startTime, extStartSN: r.extStartSN} + r.snapshots[i] = &Snapshot{ + startTime: r.startTime, + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, + } } } @@ -359,7 +421,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa } // adjust start to account for out-of-order packets before a cycle completes - if !r.maybeAdjustStartSN(rtph, packetTime, pktSize, hdrSize, payloadSize) { + if !r.maybeAdjustStartSN(rtph, pktSize, hdrSize, payloadSize) { if !r.isSnInfoLost(rtph.SequenceNumber) { r.bytesDuplicate += pktSize r.headerBytesDuplicate += hdrSize @@ -367,7 +429,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa isDuplicate = true } else { r.packetsLost-- - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, true) } } @@ -386,14 +448,23 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.clearSnInfos(r.highestSN+1, rtph.SequenceNumber) r.packetsLost += uint32(diff - 1) - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, false) if rtph.SequenceNumber < r.highestSN && !first { r.cycles++ } r.highestSN = rtph.SequenceNumber - r.highestTS = rtph.Timestamp - r.highestTime = packetTime + + if rtph.Timestamp != r.highestTS { + if rtph.Timestamp < r.highestTS && !first { + r.tsCycles++ + } + r.highestTS = rtph.Timestamp + + // update only on first packet as same timestamp could be in multiple packets. + // NOTE: this may not be the first packet with this time stamp if there is packet loss. + r.highestTime = packetTime + } } if !isDuplicate { @@ -423,7 +494,7 @@ func (r *RTPStats) ResyncOnNextPacket() { r.resyncOnNextPacket = true } -func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, packetTime int64, pktSize uint64, hdrSize uint64, payloadSize int) bool { +func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, pktSize uint64, hdrSize uint64, payloadSize int) bool { if (r.getExtHighestSN() - r.extStartSN + 1) >= (NumSequenceNumbers / 2) { return false } @@ -436,7 +507,7 @@ func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, packetTime int64, pktSiz beforeAdjust := r.extStartSN r.extStartSN = uint32(rtph.SequenceNumber) - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, true) for _, s := range r.snapshots { if s.extStartSN == beforeAdjust { @@ -473,16 +544,21 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 r.lock.Lock() defer r.lock.Unlock() - if !r.endTime.IsZero() || !r.params.IsReceiverReportDriven { + if !r.initialized || !r.endTime.IsZero() || !r.params.IsReceiverReportDriven || rr.LastSequenceNumber < r.extStartSN { + // it is possible that the `LastSequenceNumber` in the receiver report is before the starting + // sequence number when dummy packets are used to trigger Pion's OnTrack path. return } - rtt, err := mediatransportutil.GetRttMsFromReceiverReportOnly(&rr) - if err == nil { - isRttChanged = rtt != r.rtt - } else { - if err != mediatransportutil.ErrRttNoLastSenderReport { - r.logger.Warnw("error getting rtt", err) + var err error + if r.srNewest != nil { + rtt, err = mediatransportutil.GetRttMs(&rr, r.srNewest.NTPTimestamp, r.srNewest.At) + if err == nil { + isRttChanged = rtt != r.rtt + } else { + if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) && !errors.Is(err, mediatransportutil.ErrRttNoLastSenderReport) { + r.logger.Warnw("error getting rtt", err) + } } } @@ -527,6 +603,13 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 return } +func (r *RTPStats) LastReceiverReport() time.Time { + r.lock.RLock() + defer r.lock.RUnlock() + + return r.lastRRTime +} + func (r *RTPStats) UpdateNack(nackCount uint32) { r.lock.Lock() defer r.lock.Unlock() @@ -684,87 +767,239 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { r.lock.Lock() defer r.lock.Unlock() - if srData == nil { - r.srDataExt = nil + if srData == nil || !r.initialized { return } // prevent against extreme case of anachronous sender reports - if r.srDataExt != nil && r.srDataExt.SenderReportData.NTPTimestamp > srData.NTPTimestamp { + if r.srNewest != nil && r.srNewest.NTPTimestamp > srData.NTPTimestamp { + r.logger.Infow( + "received anachronous sender report", + "current", srData.NTPTimestamp.Time(), + "last", r.srNewest.NTPTimestamp.Time(), + ) return } - // Low pass filter one-way-delay (owd) to normalize time stamp to local time base when sending RTCP Sender Report. - // Forwarding RTCP Sender Report would be ideal. But, there are a couple of issues with that - // 1. Senders could have different clocks. - // 2. Adjusting to current time as required by RTCP spec. - // By normalizing to local clock, these issues can be addressed. However, normalization is not straightforward - // as it is not possible to know the propagation delay and processing delay at both ends (send side processing - // after time stamping the RTCP packet and receive side processing after reading packet off the wire). - // Smoothed version of OWD is used to alleviate irregularities somewhat. - owd := srData.ArrivalTime.Sub(srData.NTPTimestamp.Time()) - if r.srDataExt != nil { - prevOwd := r.srDataExt.SenderReportData.ArrivalTime.Sub(r.srDataExt.SenderReportData.NTPTimestamp.Time()) - if time.Duration(math.Abs(float64(owd)-float64(prevOwd))) > TooLargeOWD { - r.logger.Infow("large one-way-delay", "owd", owd, "prevOwd", prevOwd) + cycles := uint64(0) + if r.srNewest != nil { + cycles = r.srNewest.RTPTimestampExt & 0xFF_FF_FF_FF_00_00_00_00 + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { + cycles += (1 << 32) } } - smoothedOwd := owd - if r.srDataExt != nil { - smoothedOwd = r.srDataExt.SmoothedOWD + srDataCopy := *srData + srDataCopy.RTPTimestampExt = uint64(srDataCopy.RTPTimestamp) + cycles + + // monitor and log RTP timestamp anomalies + var ntpDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + var arrivalDiffSinceLast time.Duration + var expectedTimeDiffSinceLast float64 + var isWarped bool + if r.srNewest != nil { + if srDataCopy.RTPTimestampExt < r.srNewest.RTPTimestampExt { + // This can happen when a track is replaced with a null and then restored - + // i. e. muting replacing with null and unmute restoring the original track. + // Under such a condition reset the sender reports to start from this point. + // Resetting will ensure sample rate calculations do not go haywire due to negative time. + r.logger.Infow( + "received sender report, out-of-order, resetting", + "prevTSExt", r.srNewest.RTPTimestampExt, + "prevNTP", r.srNewest.NTPTimestamp.Time().String(), + "currTSExt", srDataCopy.RTPTimestampExt, + "currNTP", srDataCopy.NTPTimestamp.Time().String(), + ) + r.srFirst = &srDataCopy + r.srNewest = &srDataCopy + } + + ntpDiffSinceLast = srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + rtpDiffSinceLast = srDataCopy.RTPTimestamp - r.srNewest.RTPTimestamp + arrivalDiffSinceLast = srDataCopy.At.Sub(r.srNewest.At) + expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + isWarped = true + } } - r.srDataExt = &RTCPSenderReportDataExt{ - SenderReportData: *srData, - SmoothedOWD: (owd + smoothedOwd) / 2, + + r.srNewest = &srDataCopy + if r.srFirst == nil { + r.srFirst = &srDataCopy + } + + if isWarped { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Infow( + "received sender report, time warp", + "ntp", srData.NTPTimestamp.Time().String(), + "rtp", srData.RTPTimestamp, + "arrival", srData.At.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) } } -func (r *RTPStats) GetRtcpSenderReportDataExt() *RTCPSenderReportDataExt { - r.lock.Lock() - defer r.lock.Unlock() - - if r.srDataExt == nil { - return nil - } - - return &RTCPSenderReportDataExt{ - SenderReportData: r.srDataExt.SenderReportData, - SmoothedOWD: r.srDataExt.SmoothedOWD, - } -} - -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportDataExt) *rtcp.SenderReport { +func (r *RTPStats) GetRtcpSenderReportData() (srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) { r.lock.RLock() defer r.lock.RUnlock() + if r.srFirst != nil { + srFirstCopy := *r.srFirst + srFirst = &srFirstCopy + } + + if r.srNewest != nil { + srNewestCopy := *r.srNewest + srNewest = &srNewestCopy + } + return +} + +func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt uint64, err error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.initialized { + err = errors.New("uninitilaized") + return + } + + timeDiff := at.Sub(r.firstTime) + expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 + expectedTSExt = r.extStartTS + uint64(expectedRTPDiff) + return +} + +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) *rtcp.SenderReport { + r.lock.Lock() + defer r.lock.Unlock() + if !r.initialized { return nil } - if srDataExt == nil || srDataExt.SenderReportData.NTPTimestamp == 0 || srDataExt.SenderReportData.ArrivalTime.IsZero() { - // no sender report from publisher - return nil + // construct current time based on monotonic clock + timeSinceFirst := time.Since(r.firstTime) + now := r.firstTime.Add(timeSinceFirst) + nowNTP := mediatransportutil.ToNtpTime(now) + + timeSinceHighest := now.Sub(r.highestTime) + nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) + + // It is possible that publisher is pacing at a slower rate. + // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. + // Check for that using calculated clock rate and use the later time stamp if applicable. + tsCycles := r.tsCycles + if nowRTP < r.highestTS { + tsCycles++ + } + nowRTPExt := getExtTS(nowRTP, tsCycles) + var nowRTPExtUsingRate uint64 + if calculatedClockRate != 0 { + nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) + if nowRTPExtUsingRate > nowRTPExt { + nowRTPExt = nowRTPExtUsingRate + nowRTP = uint32(nowRTPExtUsingRate) + } } - // NTP timestamp in sender report from publisher side could have a different base, - // i. e. although it should be wall clock at time of send, have observed instances of older timer. - // It is not possible to accurately calculate current time in the NTP time base of the publisher side. - // So, using a smoothed version of one way delay for use in sender reports. - now := time.Now() - nowNTP := mediatransportutil.ToNtpTime(now) - nowRTP := r.highestTS - - smoothedLocalTimeOfLatestSenderReportNTP := srDataExt.SenderReportData.NTPTimestamp.Time().Add(srDataExt.SmoothedOWD) - if smoothedLocalTimeOfLatestSenderReportNTP.After(now) { - r.logger.Infow("smoothed time of NTP is ahead", - "now", now, - "smoothed", smoothedLocalTimeOfLatestSenderReportNTP, - "diff", smoothedLocalTimeOfLatestSenderReportNTP.Sub(now), + if r.srNewest != nil && nowRTPExt < r.srNewest.RTPTimestampExt { + // If report being generated is behind, use the time different and clock rate of codec to produce next report. + // Current report could be behind due to the following + // - Publisher pacing + // - Due to above, report from publisher side is ahead of packet timestamps. + // Note that report will map wall clock to timestamp at capture time and happens before the pacer. + // - Pause/Mute followed by resume, some combination of events that could + // result in this module not having calculated clock rate of publisher side. + // - When the above happens, current will be generated using highestTS which could be behind. + // That could end up behind the last report's timestamp in extreme cases + r.logger.Infow( + "sending sender report, out-of-order, repairing", + "prevTSExt", r.srNewest.RTPTimestampExt, + "prevNTP", r.srNewest.NTPTimestamp.Time().String(), + "currTSExt", nowRTPExt, + "currNTP", nowNTP.Time().String(), + ) + ntpDiffSinceLast := nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) + nowRTPExt = r.srNewest.RTPTimestampExt + uint64(ntpDiffSinceLast.Seconds()*float64(r.params.ClockRate)) + } + + // monitor and log RTP timestamp anomalies + var ntpDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + var departureDiffSinceLast time.Duration + var expectedTimeDiffSinceLast float64 + var isWarped bool + if r.srNewest != nil { + ntpDiffSinceLast = nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) + rtpDiffSinceLast = nowRTP - r.srNewest.RTPTimestamp + departureDiffSinceLast = now.Sub(r.srNewest.At) + + expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + isWarped = true + } + } + + r.srNewest = &RTCPSenderReportData{ + NTPTimestamp: nowNTP, + RTPTimestamp: nowRTP, + RTPTimestampExt: nowRTPExt, + At: now, + } + if r.srFirst == nil { + r.srFirst = r.srNewest + } + + if isWarped { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Infow( + "sending sender report, time warp", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "departure", now.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + "calculatedClockRate", calculatedClockRate, + "nowRTPExt", nowRTPExt, + "nowRTPExtUsingRate", nowRTPExtUsingRate, ) - nowRTP += uint32(now.Sub(time.Unix(0, r.highestTime)).Milliseconds() * int64(r.params.ClockRate) / 1000) } else { - nowRTP = srDataExt.SenderReportData.RTPTimestamp + uint32(now.Sub(smoothedLocalTimeOfLatestSenderReportNTP).Milliseconds()*int64(r.params.ClockRate)/1000) + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Debugw( + "sending sender report", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "departure", now.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + "calculatedClockRate", calculatedClockRate, + "nowRTPExt", nowRTPExt, + "nowRTPExtUsingRate", nowRTPExtUsingRate, + ) } return &rtcp.SenderReport{ @@ -778,7 +1013,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotId uint32) *rtcp.ReceptionReport { r.lock.Lock() - then, now := r.getAndResetSnapshot(snapshotId) + then, now := r.getAndResetSnapshot(snapshotId, false) r.lock.Unlock() if now == nil || then == nil { @@ -800,17 +1035,8 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, return nil } - packetsLost := uint32(0) - if r.params.IsReceiverReportDriven { - // should not be set for streams that need to generate reception report - packetsLost = now.packetsLostOverridden - then.packetsLostOverridden - if int32(packetsLost) < 0 { - packetsLost = 0 - } - } else { - intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) - packetsLost = intervalStats.packetsLost - } + intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) + packetsLost := intervalStats.packetsLost lossRate := float32(packetsLost) / float32(packetsExpected) fracLost := uint8(lossRate * 256.0) if proxyFracLost > fracLost { @@ -818,28 +1044,22 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, } var dlsr uint32 - if r.srDataExt != nil && !r.srDataExt.SenderReportData.ArrivalTime.IsZero() { - delayMS := uint32(time.Since(r.srDataExt.SenderReportData.ArrivalTime).Milliseconds()) + if r.srNewest != nil && !r.srNewest.At.IsZero() { + delayMS := uint32(time.Since(r.srNewest.At).Milliseconds()) dlsr = (delayMS / 1e3) << 16 dlsr |= (delayMS % 1e3) * 65536 / 1000 } - jitter := r.jitter - if r.params.IsReceiverReportDriven { - // should not be set for streams that need to generate reception report - jitter = r.jitterOverridden - } - lastSR := uint32(0) - if r.srDataExt != nil { - lastSR = uint32(r.srDataExt.SenderReportData.NTPTimestamp >> 16) + if r.srNewest != nil { + lastSR = uint32(r.srNewest.NTPTimestamp >> 16) } return &rtcp.ReceptionReport{ SSRC: ssrc, FractionLost: fracLost, TotalLost: r.packetsLost, LastSequenceNumber: now.extStartSN, - Jitter: uint32(jitter), + Jitter: uint32(r.jitter), LastSenderReport: lastSR, Delay: dlsr, } @@ -847,7 +1067,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { r.lock.Lock() - then, now := r.getAndResetSnapshot(snapshotId) + then, now := r.getAndResetSnapshot(snapshotId, false) r.lock.Unlock() if now == nil || then == nil { @@ -862,56 +1082,99 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { packetsExpected := now.extStartSN - then.extStartSN if packetsExpected > NumSequenceNumbers { - r.logger.Warnw( + r.logger.Errorw( "too many packets expected in delta", fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSN, now.extStartSN, packetsExpected), ) return nil } if packetsExpected == 0 { - if r.params.IsReceiverReportDriven { - // not received RTCP RR - return nil - } - return &RTPDeltaInfo{ StartTime: startTime, Duration: endTime.Sub(startTime), } } - packetsLost := uint32(0) - packetsMissing := uint32(0) intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) - if r.params.IsReceiverReportDriven { - packetsMissing = intervalStats.packetsLost + return &RTPDeltaInfo{ + StartTime: startTime, + Duration: endTime.Sub(startTime), + Packets: packetsExpected - intervalStats.packetsPadding, + Bytes: intervalStats.bytes, + HeaderBytes: intervalStats.headerBytes, + PacketsDuplicate: now.packetsDuplicate - then.packetsDuplicate, + BytesDuplicate: now.bytesDuplicate - then.bytesDuplicate, + HeaderBytesDuplicate: now.headerBytesDuplicate - then.headerBytesDuplicate, + PacketsPadding: intervalStats.packetsPadding, + BytesPadding: intervalStats.bytesPadding, + HeaderBytesPadding: intervalStats.headerBytesPadding, + PacketsLost: intervalStats.packetsLost, + Frames: intervalStats.frames, + RttMax: then.maxRtt, + JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6, + Nacks: now.nacks - then.nacks, + Plis: now.plis - then.plis, + Firs: now.firs - then.firs, + } +} - packetsLost = now.packetsLostOverridden - then.packetsLostOverridden - if int32(packetsLost) < 0 { - packetsLost = 0 - } - - if packetsLost > packetsExpected { - r.logger.Warnw( - "unexpected number of packets lost", - fmt.Errorf( - "start: %d, end: %d, expected: %d, lost: report: %d, interval: %d", - then.extStartSN, - now.extStartSN, - packetsExpected, - now.packetsLostOverridden-then.packetsLostOverridden, - intervalStats.packetsLost, - ), - ) - packetsLost = packetsExpected - } - } else { - packetsLost = intervalStats.packetsLost +func (r *RTPStats) DeltaInfoOverridden(snapshotId uint32) *RTPDeltaInfo { + if !r.params.IsReceiverReportDriven { + return nil } - maxJitter := then.maxJitter - if r.params.IsReceiverReportDriven { - maxJitter = then.maxJitterOverridden + r.lock.Lock() + then, now := r.getAndResetSnapshot(snapshotId, true) + r.lock.Unlock() + + if now == nil || then == nil { + return nil + } + + r.lock.RLock() + defer r.lock.RUnlock() + + startTime := then.startTime + endTime := now.startTime + + packetsExpected := now.extStartSNOverridden - then.extStartSNOverridden + if packetsExpected > NumSequenceNumbers { + r.logger.Warnw( + "too many packets expected in delta (overridden)", + fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSNOverridden, now.extStartSNOverridden, packetsExpected), + ) + return nil + } + if packetsExpected == 0 { + // not received RTCP RR (OR) publisher is not producing any data + return nil + } + + intervalStats := r.getIntervalStats(uint16(then.extStartSNOverridden), uint16(now.extStartSNOverridden)) + packetsLost := now.packetsLostOverridden - then.packetsLostOverridden + if int32(packetsLost) < 0 { + packetsLost = 0 + } + + if packetsLost > packetsExpected { + r.logger.Warnw( + "unexpected number of packets lost", + fmt.Errorf( + "start: %d, end: %d, expected: %d, lost: report: %d, interval: %d", + then.extStartSNOverridden, + now.extStartSNOverridden, + packetsExpected, + now.packetsLostOverridden-then.packetsLostOverridden, + intervalStats.packetsLost, + ), + ) + packetsLost = packetsExpected + } + + // discount jitter from publisher side + internal processing + maxJitter := then.maxJitterOverridden - then.maxJitter + if maxJitter < 0.0 { + maxJitter = 0.0 } maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 @@ -928,7 +1191,8 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { BytesPadding: intervalStats.bytesPadding, HeaderBytesPadding: intervalStats.headerBytesPadding, PacketsLost: packetsLost, - PacketsMissing: packetsMissing, + PacketsMissing: intervalStats.packetsLost, + PacketsOutOfOrder: intervalStats.packetsOutOfOrder, Frames: intervalStats.frames, RttMax: then.maxRtt, JitterMax: maxJitterTime, @@ -952,7 +1216,7 @@ func (r *RTPStats) ToString() string { str := fmt.Sprintf("t: %+v|%+v|%.2fs", p.StartTime.AsTime().Format(time.UnixDate), p.EndTime.AsTime().Format(time.UnixDate), p.Duration) - str += fmt.Sprintf(" sn: %d|%d", r.extStartSN, r.getExtHighestSN()) + str += fmt.Sprintf(", sn: %d|%d", r.extStartSN, r.getExtHighestSN()) str += fmt.Sprintf(", ep: %d|%.2f/s", expectedPackets, expectedPacketRate) str += fmt.Sprintf(", p: %d|%.2f/s", p.Packets, p.PacketRate) @@ -971,6 +1235,7 @@ func (r *RTPStats) ToString() string { jitter := r.jitter maxJitter := r.maxJitter if r.params.IsReceiverReportDriven { + // NOTE: jitter includes jitter from publisher and from processing jitter = r.jitterOverridden maxJitter = r.maxJitterOverridden } @@ -1004,6 +1269,12 @@ func (r *RTPStats) ToString() string { str += ", rtt(ms):" str += fmt.Sprintf("%d|%d", p.RttCurrent, p.RttMax) + str += ", drift(ms):" + str += fmt.Sprintf("%.2f", p.DriftMs) + + str += ", sr(Hz):" + str += fmt.Sprintf("%.2f", p.SampleRate) + return str } @@ -1044,12 +1315,15 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { jitter := r.jitter maxJitter := r.maxJitter if r.params.IsReceiverReportDriven { + // NOTE: jitter includes jitter from publisher and from processing jitter = r.jitterOverridden maxJitter = r.maxJitterOverridden } jitterTime := jitter / float64(r.params.ClockRate) * 1e6 maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 + packetDrift, _ := r.getDrift() + p := &livekit.RTPStats{ StartTime: timestamppb.New(r.startTime), EndTime: timestamppb.New(endTime), @@ -1091,6 +1365,8 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { LastFir: timestamppb.New(r.lastFir), RttCurrent: r.rtt, RttMax: r.maxRtt, + DriftMs: packetDrift.driftMs, + SampleRate: packetDrift.sampleRate, } gapsPresent := false @@ -1152,7 +1428,7 @@ func (r *RTPStats) getSnInfoOutOfOrderPtr(sn uint16) int { return (r.snInfoWritePtr - int(offset) - 1) & SnInfoMask } -func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadSize uint16, marker bool) { +func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadSize uint16, marker bool, isOutOfOrder bool) { writePtr := 0 ooo := (sn - r.highestSN) > (1 << 15) if !ooo { @@ -1170,6 +1446,7 @@ func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadS snInfo.hdrSize = hdrSize snInfo.isPaddingOnly = payloadSize == 0 snInfo.marker = marker + snInfo.isOutOfOrder = isOutOfOrder } func (r *RTPStats) clearSnInfos(startInclusive uint16, endExclusive uint16) { @@ -1217,6 +1494,9 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) intervalStats.packets++ intervalStats.bytes += uint64(snInfo.pktSize) intervalStats.headerBytes += uint64(snInfo.hdrSize) + if snInfo.isOutOfOrder { + intervalStats.packetsOutOfOrder++ + } } if snInfo.marker { @@ -1236,18 +1516,31 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) } if packetsNotFound != 0 { - r.logger.Warnw( + r.logger.Errorw( "could not find some packets", nil, "start", startInclusive, "end", endExclusive, "count", packetsNotFound, + "highestSN", r.highestSN, ) } return } -func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime int64) { - packetTimeRTP := uint32(packetTime / 1e6 * int64(r.params.ClockRate/1e3)) +func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime time.Time) { + // Do not update jitter on multiple packets of same frame. + // All packets of a frame have the same time stamp. + // NOTE: This does not protect against using more than one packet of the same frame + // if packets arrive out-of-order. For example, + // p1f1 -> p1f2 -> p2f1 + // In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation + // although it is the second packet of a frame because of out-of-order receival. + if r.lastJitterRTP == rtph.Timestamp { + return + } + + timeSinceFirst := packetTime.Sub(r.firstTime) + packetTimeRTP := uint32(timeSinceFirst.Nanoseconds() * int64(r.params.ClockRate) / 1e9) transit := packetTimeRTP - rtph.Timestamp if r.lastTransit != 0 { @@ -1262,12 +1555,34 @@ func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime int64) { for _, s := range r.snapshots { if r.jitter > s.maxJitter { - r.maxJitter = r.jitter + s.maxJitter = r.jitter } } } r.lastTransit = transit + r.lastJitterRTP = rtph.Timestamp +} + +func (r *RTPStats) getDrift() (packetDrift driftResult, reportDrift driftResult) { + packetDrift.timeSinceFirst = r.highestTime.Sub(r.firstTime) + packetDrift.rtpDiffSinceFirst = getExtTS(r.highestTS, r.tsCycles) - r.extStartTS + packetDrift.driftSamples = int64(packetDrift.rtpDiffSinceFirst - uint64(packetDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + packetDrift.driftMs = (float64(packetDrift.driftSamples) * 1000) / float64(r.params.ClockRate) + if packetDrift.timeSinceFirst.Seconds() != 0 { + packetDrift.sampleRate = float64(packetDrift.rtpDiffSinceFirst) / packetDrift.timeSinceFirst.Seconds() + } + + if r.srFirst != nil && r.srNewest != nil && r.srFirst.RTPTimestamp != r.srNewest.RTPTimestamp { + reportDrift.timeSinceFirst = r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) + reportDrift.rtpDiffSinceFirst = r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt + reportDrift.driftSamples = int64(reportDrift.rtpDiffSinceFirst - uint64(reportDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + reportDrift.driftMs = (float64(reportDrift.driftSamples) * 1000) / float64(r.params.ClockRate) + if reportDrift.timeSinceFirst.Seconds() != 0 { + reportDrift.sampleRate = float64(reportDrift.rtpDiffSinceFirst) / reportDrift.timeSinceFirst.Seconds() + } + } + return } func (r *RTPStats) updateGapHistogram(gap int) { @@ -1283,24 +1598,33 @@ func (r *RTPStats) updateGapHistogram(gap int) { } } -func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) { - if !r.initialized || (r.params.IsReceiverReportDriven && r.lastRRTime.IsZero()) { +func (r *RTPStats) getAndResetSnapshot(snapshotId uint32, override bool) (*Snapshot, *Snapshot) { + if !r.initialized || (override && r.lastRRTime.IsZero()) { return nil, nil } then := r.snapshots[snapshotId] if then == nil { then = &Snapshot{ - startTime: r.startTime, - extStartSN: r.extStartSN, + startTime: r.startTime, + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, } r.snapshots[snapshotId] = then } + var startTime time.Time + if override { + startTime = r.lastRRTime + } else { + startTime = time.Now() + } + // snapshot now r.snapshots[snapshotId] = &Snapshot{ - startTime: time.Now(), - extStartSN: r.getExtHighestSNAdjusted() + 1, + startTime: startTime, + extStartSN: r.getExtHighestSN() + 1, + extStartSNOverridden: r.getExtHighestSNAdjusted() + 1, packetsDuplicate: r.packetsDuplicate, bytesDuplicate: r.bytesDuplicate, headerBytesDuplicate: r.headerBytesDuplicate, @@ -1308,9 +1632,9 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) nacks: r.nacks, plis: r.plis, firs: r.firs, - maxJitter: 0.0, - maxJitterOverridden: 0.0, - maxRtt: 0, + maxJitter: r.jitter, + maxJitterOverridden: r.jitterOverridden, + maxRtt: r.rtt, } // make a copy so that it can be used independently now := *r.snapshots[snapshotId] @@ -1320,6 +1644,10 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) // ---------------------------------- +func getExtTS(ts uint32, cycles uint32) uint64 { + return (uint64(cycles) << 32) | uint64(ts) +} + func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { if len(statsList) == 0 { return nil @@ -1357,6 +1685,8 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { lastFir := time.Time{} rtt := uint32(0) maxRtt := uint32(0) + driftMs := float64(0.0) + sampleRate := float64(0.0) for _, stats := range statsList { if startTime.IsZero() || startTime.After(stats.StartTime.AsTime()) { @@ -1423,6 +1753,9 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { if stats.RttMax > maxRtt { maxRtt = stats.RttMax } + + driftMs += stats.DriftMs + sampleRate += stats.SampleRate } if endTime.IsZero() { @@ -1485,5 +1818,114 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { LastFir: timestamppb.New(lastFir), RttCurrent: rtt / uint32(len(statsList)), RttMax: maxRtt, + DriftMs: driftMs / float64(len(statsList)), + SampleRate: sampleRate / float64(len(statsList)), } } + +func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { + if len(deltaInfoList) == 0 { + return nil + } + + startTime := time.Time{} + endTime := time.Time{} + + packets := uint32(0) + bytes := uint64(0) + headerBytes := uint64(0) + + packetsDuplicate := uint32(0) + bytesDuplicate := uint64(0) + headerBytesDuplicate := uint64(0) + + packetsPadding := uint32(0) + bytesPadding := uint64(0) + headerBytesPadding := uint64(0) + + packetsLost := uint32(0) + packetsMissing := uint32(0) + packetsOutOfOrder := uint32(0) + + frames := uint32(0) + + maxRtt := uint32(0) + maxJitter := float64(0) + + nacks := uint32(0) + plis := uint32(0) + firs := uint32(0) + + for _, deltaInfo := range deltaInfoList { + if deltaInfo == nil { + continue + } + + if startTime.IsZero() || startTime.After(deltaInfo.StartTime) { + startTime = deltaInfo.StartTime + } + + endedAt := deltaInfo.StartTime.Add(deltaInfo.Duration) + if endTime.IsZero() || endTime.Before(endedAt) { + endTime = endedAt + } + + packets += deltaInfo.Packets + bytes += deltaInfo.Bytes + headerBytes += deltaInfo.HeaderBytes + + packetsDuplicate += deltaInfo.PacketsDuplicate + bytesDuplicate += deltaInfo.BytesDuplicate + headerBytesDuplicate += deltaInfo.HeaderBytesDuplicate + + packetsPadding += deltaInfo.PacketsPadding + bytesPadding += deltaInfo.BytesPadding + headerBytesPadding += deltaInfo.HeaderBytesPadding + + packetsLost += deltaInfo.PacketsLost + packetsMissing += deltaInfo.PacketsMissing + packetsOutOfOrder += deltaInfo.PacketsOutOfOrder + + frames += deltaInfo.Frames + + if deltaInfo.RttMax > maxRtt { + maxRtt = deltaInfo.RttMax + } + + if deltaInfo.JitterMax > maxJitter { + maxJitter = deltaInfo.JitterMax + } + + nacks += deltaInfo.Nacks + plis += deltaInfo.Plis + firs += deltaInfo.Firs + } + if startTime.IsZero() || endTime.IsZero() { + return nil + } + + return &RTPDeltaInfo{ + StartTime: startTime, + Duration: endTime.Sub(startTime), + Packets: packets, + Bytes: bytes, + HeaderBytes: headerBytes, + PacketsDuplicate: packetsDuplicate, + BytesDuplicate: bytesDuplicate, + HeaderBytesDuplicate: headerBytesDuplicate, + PacketsPadding: packetsPadding, + BytesPadding: bytesPadding, + HeaderBytesPadding: headerBytesPadding, + PacketsLost: packetsLost, + PacketsMissing: packetsMissing, + PacketsOutOfOrder: packetsOutOfOrder, + Frames: frames, + RttMax: maxRtt, + JitterMax: maxJitter, + Nacks: nacks, + Plis: plis, + Firs: firs, + } +} + +// ------------------------------------------------------------------- diff --git a/pkg/sfu/buffer/rtpstats_test.go b/pkg/sfu/buffer/rtpstats_test.go index b1be40650..c5c4138ba 100644 --- a/pkg/sfu/buffer/rtpstats_test.go +++ b/pkg/sfu/buffer/rtpstats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -43,7 +57,7 @@ func TestRTPStats(t *testing.T) { timestamp += uint32(now.Sub(lastFrameTime).Seconds() * float64(clockRate)) for i := 0; i < packetsPerFrame; i++ { packet := getPacket(sequenceNumber, timestamp, packetSize) - r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) if (sequenceNumber % 100) == 0 { jump := uint16(rand.Float64() * 120.0) sequenceNumber += jump @@ -70,7 +84,7 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber := uint16(rand.Float64() * float64(1<<16)) timestamp := uint32(rand.Float64() * float64(1<<32)) packet := getPacket(sequenceNumber, timestamp, 1000) - flowState := r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState := r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.True(t, r.initialized) require.Equal(t, sequenceNumber, r.highestSN) @@ -80,14 +94,14 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber++ timestamp += 3000 packet = getPacket(sequenceNumber, timestamp, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) // out-of-order packet = getPacket(sequenceNumber-10, timestamp-30000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) @@ -96,7 +110,7 @@ func TestRTPStats_Update(t *testing.T) { // duplicate packet = getPacket(sequenceNumber-10, timestamp-30000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) @@ -107,7 +121,7 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber += 10 timestamp += 30000 packet = getPacket(sequenceNumber, timestamp, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.True(t, flowState.HasLoss) require.Equal(t, sequenceNumber-9, flowState.LossStartInclusive) require.Equal(t, sequenceNumber, flowState.LossEndExclusive) @@ -115,7 +129,7 @@ func TestRTPStats_Update(t *testing.T) { // out-of-order should decrement number of lost packets packet = getPacket(sequenceNumber-15, timestamp-45000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) diff --git a/pkg/sfu/buffer/streamstats.go b/pkg/sfu/buffer/streamstats.go index 04c02a65b..cdd8e1333 100644 --- a/pkg/sfu/buffer/streamstats.go +++ b/pkg/sfu/buffer/streamstats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer type StreamStatsWithLayers struct { diff --git a/pkg/sfu/buffer/videolayer.go b/pkg/sfu/buffer/videolayer.go index dce60597f..761cc1c13 100644 --- a/pkg/sfu/buffer/videolayer.go +++ b/pkg/sfu/buffer/videolayer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import "fmt" @@ -11,10 +25,15 @@ const ( ) var ( - InvalidLayers = VideoLayer{ + InvalidLayer = VideoLayer{ Spatial: InvalidLayerSpatial, Temporal: InvalidLayerTemporal, } + + DefaultMaxLayer = VideoLayer{ + Spatial: DefaultMaxLayerSpatial, + Temporal: DefaultMaxLayerTemporal, + } ) type VideoLayer struct { diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index b2bc54f23..b18c83d84 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( @@ -84,7 +98,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { logger.Warnw("unexpected rid f with only two qualities, low and high", nil) return 1 case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, medum and high", nil) + logger.Warnw("unexpected rid f with only two qualities, medium and high", nil) return 1 default: @@ -156,7 +170,7 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil) return HalfResolution case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected layer 2 with only two qualities, medum and high", nil) + logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil) return HalfResolution default: diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index 7e3b08792..bb103f72c 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -1,10 +1,25 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( "testing" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" ) func TestRidConversion(t *testing.T) { @@ -21,123 +36,123 @@ func TestRidConversion(t *testing.T) { "no track info", nil, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, } @@ -169,114 +184,114 @@ func TestQualityConversion(t *testing.T) { "no track info", nil, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_LOW, layer: 0}, }, }, { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, }, }, { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 0}, }, }, { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, }, }, { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1}, }, }, { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1}, }, }, { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, } @@ -322,7 +337,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[livekit.VideoQuality]string{ @@ -335,7 +350,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]string{ @@ -348,7 +363,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -361,8 +376,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]string{ @@ -375,8 +390,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -389,8 +404,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -403,9 +418,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go new file mode 100644 index 000000000..850cecb8d --- /dev/null +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -0,0 +1,39 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codecmunger + +import ( + "errors" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +var ( + ErrNotVP8 = errors.New("not VP8") + ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache") + ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer") +) + +type CodecMunger interface { + GetState() interface{} + SeedState(state interface{}) + + SetLast(extPkt *buffer.ExtPacket) + UpdateOffsets(extPkt *buffer.ExtPacket) + + UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) + + UpdateAndGetPadding(newPicture bool) ([]byte, error) +} diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go new file mode 100644 index 000000000..32e3d9ee9 --- /dev/null +++ b/pkg/sfu/codecmunger/null.go @@ -0,0 +1,54 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codecmunger + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type Null struct { + seededState interface{} +} + +func NewNull(_logger logger.Logger) *Null { + return &Null{} +} + +func (n *Null) GetState() interface{} { + return nil +} + +func (n *Null) SeedState(state interface{}) { + n.seededState = state +} + +func (n *Null) GetSeededState() interface{} { + return n.seededState +} + +func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { +} + +func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) { +} + +func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) { + return nil, nil +} + +func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) { + return nil, nil +} diff --git a/pkg/sfu/vp8munger.go b/pkg/sfu/codecmunger/vp8.go similarity index 55% rename from pkg/sfu/vp8munger.go rename to pkg/sfu/codecmunger/vp8.go index 1dfa24d12..bb270387d 100644 --- a/pkg/sfu/vp8munger.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -1,4 +1,18 @@ -package sfu +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codecmunger import ( "fmt" @@ -10,65 +24,68 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" ) -// VP8 munger -type TranslationParamsVP8 struct { - Header *buffer.VP8 -} +const ( + missingPictureIdsThreshold = 50 + droppedPictureIdsThreshold = 20 + exemptedPictureIdsThreshold = 20 +) // ----------------------------------------------------------- -type VP8MungerState struct { +type VP8State struct { ExtLastPictureId int32 - PictureIdUsed int + PictureIdUsed bool LastTl0PicIdx uint8 - Tl0PicIdxUsed int - TidUsed int + Tl0PicIdxUsed bool + TidUsed bool LastKeyIdx uint8 - KeyIdxUsed int + KeyIdxUsed bool } -func (v VP8MungerState) String() string { - return fmt.Sprintf("VP8MungerState{extLastPictureId: %d, pictureIdUsed: %+v, lastTl0PicIdx: %d, tl0PicIdxUsed: %+v, tidUsed: %+v, lastKeyIdx: %d, keyIdxUsed: %+v)", +func (v VP8State) String() string { + return fmt.Sprintf("VP8State{extLastPictureId: %d, pictureIdUsed: %+v, lastTl0PicIdx: %d, tl0PicIdxUsed: %+v, tidUsed: %+v, lastKeyIdx: %d, keyIdxUsed: %+v)", v.ExtLastPictureId, v.PictureIdUsed, v.LastTl0PicIdx, v.Tl0PicIdxUsed, v.TidUsed, v.LastKeyIdx, v.KeyIdxUsed) } // ----------------------------------------------------------- -type VP8MungerParams struct { +type VP8 struct { + logger logger.Logger + pictureIdWrapHandler VP8PictureIdWrapHandler extLastPictureId int32 pictureIdOffset int32 - pictureIdUsed int + pictureIdUsed bool lastTl0PicIdx uint8 tl0PicIdxOffset uint8 - tl0PicIdxUsed int - tidUsed int + tl0PicIdxUsed bool + tidUsed bool lastKeyIdx uint8 keyIdxOffset uint8 - keyIdxUsed int + keyIdxUsed bool - missingPictureIds *orderedmap.OrderedMap[int32, int32] - lastDroppedPictureId int32 + missingPictureIds *orderedmap.OrderedMap[int32, int32] + droppedPictureIds *orderedmap.OrderedMap[int32, bool] + exemptedPictureIds *orderedmap.OrderedMap[int32, bool] } -type VP8Munger struct { - logger logger.Logger - - VP8MungerParams -} - -func NewVP8Munger(logger logger.Logger) *VP8Munger { - return &VP8Munger{ - logger: logger, - VP8MungerParams: VP8MungerParams{ - missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), - lastDroppedPictureId: -1, - }, +func NewVP8(logger logger.Logger) *VP8 { + return &VP8{ + logger: logger, + missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), + droppedPictureIds: orderedmap.NewOrderedMap[int32, bool](), + exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](), } } -func (v *VP8Munger) GetLast() VP8MungerState { - return VP8MungerState{ +func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 { + v := NewVP8(logger) + v.SeedState(cm.(*Null).GetSeededState()) + return v +} + +func (v *VP8) GetState() interface{} { + return VP8State{ ExtLastPictureId: v.extLastPictureId, PictureIdUsed: v.pictureIdUsed, LastTl0PicIdx: v.lastTl0PicIdx, @@ -79,78 +96,78 @@ func (v *VP8Munger) GetLast() VP8MungerState { } } -func (v *VP8Munger) SeedLast(state VP8MungerState) { - v.extLastPictureId = state.ExtLastPictureId - v.pictureIdUsed = state.PictureIdUsed - v.lastTl0PicIdx = state.LastTl0PicIdx - v.tl0PicIdxUsed = state.Tl0PicIdxUsed - v.tidUsed = state.TidUsed - v.lastKeyIdx = state.LastKeyIdx - v.keyIdxUsed = state.KeyIdxUsed +func (v *VP8) SeedState(seed interface{}) { + if state, ok := seed.(VP8State); ok { + v.extLastPictureId = state.ExtLastPictureId + v.pictureIdUsed = state.PictureIdUsed + v.lastTl0PicIdx = state.LastTl0PicIdx + v.tl0PicIdxUsed = state.Tl0PicIdxUsed + v.tidUsed = state.TidUsed + v.lastKeyIdx = state.LastKeyIdx + v.keyIdxUsed = state.KeyIdxUsed + } } -func (v *VP8Munger) SetLast(extPkt *buffer.ExtPacket) { +func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return } - v.pictureIdUsed = vp8.PictureIDPresent - if v.pictureIdUsed == 1 { - v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + v.pictureIdUsed = vp8.I + if v.pictureIdUsed { + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M) v.extLastPictureId = int32(vp8.PictureID) } - v.tl0PicIdxUsed = vp8.TL0PICIDXPresent - if v.tl0PicIdxUsed == 1 { + v.tl0PicIdxUsed = vp8.L + if v.tl0PicIdxUsed { v.lastTl0PicIdx = vp8.TL0PICIDX } - v.tidUsed = vp8.TIDPresent + v.tidUsed = vp8.T - v.keyIdxUsed = vp8.KEYIDXPresent - if v.keyIdxUsed == 1 { + v.keyIdxUsed = vp8.K + if v.keyIdxUsed { v.lastKeyIdx = vp8.KEYIDX } - - v.lastDroppedPictureId = -1 } -func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { +func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return } - if v.pictureIdUsed == 1 { - v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + if v.pictureIdUsed { + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M) v.pictureIdOffset = int32(vp8.PictureID) - v.extLastPictureId - 1 } - if v.tl0PicIdxUsed == 1 { + if v.tl0PicIdxUsed { v.tl0PicIdxOffset = vp8.TL0PICIDX - v.lastTl0PicIdx - 1 } - if v.keyIdxUsed == 1 { + if v.keyIdxUsed { v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f } - // clear missing picture ids on layer switch + // clear picture id caches on layer switch v.missingPictureIds = orderedmap.NewOrderedMap[int32, int32]() - - v.lastDroppedPictureId = -1 + v.droppedPictureIds = orderedmap.NewOrderedMap[int32, bool]() + v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } -func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumberOrdering, maxTemporalLayer int32) (*TranslationParamsVP8, error) { +func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) ([]byte, error) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return nil, ErrNotVP8 } - extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.MBit) + extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M) // if out-of-order, look up missing picture id cache - if ordering == SequenceNumberOrderingOutOfOrder { + if snOutOfOrder { pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId) if !ok { return nil, ErrOutOfOrderVP8PictureIdCacheMiss @@ -164,27 +181,25 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff) vp8Packet := &buffer.VP8{ - FirstByte: vp8.FirstByte, - PictureIDPresent: vp8.PictureIDPresent, - PictureID: mungedPictureId, - MBit: mungedPictureId > 127, - TL0PICIDXPresent: vp8.TL0PICIDXPresent, - TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset, - TIDPresent: vp8.TIDPresent, - TID: vp8.TID, - Y: vp8.Y, - KEYIDXPresent: vp8.KEYIDXPresent, - KEYIDX: vp8.KEYIDX - v.keyIdxOffset, - IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + FirstByte: vp8.FirstByte, + I: vp8.I, + M: mungedPictureId > 127, + PictureID: mungedPictureId, + L: vp8.L, + TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset, + T: vp8.T, + TID: vp8.TID, + Y: vp8.Y, + K: vp8.K, + KEYIDX: vp8.KEYIDX - v.keyIdxOffset, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return &TranslationParamsVP8{ - Header: vp8Packet, - }, nil + return vp8Packet.Marshal() } prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() - v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.MBit) + v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.M) // if there is a gap in sequence number, record possible pictures that // the missing packets can belong to in missing picture id cache. @@ -199,35 +214,57 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb // it is possible to deduce that (for example by looking at previous packet's RTP marker // and check if that was the last packet of Picture 10), it could get complicated when // the gap is larger. - if ordering == SequenceNumberOrderingGap { - // can drop packet if it belongs to the last dropped picture. - // Example: - // o Packet 10 - Picture 11 - TID that should be dropped - // o Packet 11 - missing - // o Packet 12 - Picture 11 - will be reported as GAP, but belongs to a picture that was dropped and hence can be dropped - // If Packet 11 comes around, it will be reported as OUT_OF_ORDER, but the missing - // picture id cache will not have an entry and hence will be dropped. - if extPictureId == v.lastDroppedPictureId { - return nil, ErrFilteredVP8TemporalLayer - } else { - for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ { + if snHasGap { + for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ { + // Record missing only if picture id was not dropped. This is to avoid a subsequent packet of dropped frame going through. + // A sequence like this + // o Packet 10 - Picture 11 - TID that should be dropped + // o Packet 11 - missing - belongs to Picture 11 still + // o Packet 12 - Picture 12 - will be reported as GAP, so missing picture id mapping will be set up for Picture 11 also. + // o Next packet - Packet 11 - this will use the wrong offset from missing pictures cache + _, ok := v.droppedPictureIds.Get(lostPictureId) + if !ok { v.missingPictureIds.Set(lostPictureId, v.pictureIdOffset) } + } + // trim cache if necessary + for v.missingPictureIds.Len() > missingPictureIdsThreshold { + el := v.missingPictureIds.Front() + v.missingPictureIds.Delete(el.Key) + } + + // if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined + // which layer the missing packets belong to. A layer could have multiple packets. So, keep track + // of pictures that are forwarded even though they will be filterd out based on temporal layer + // requirements. That allows forwarding of the complete picture. + if vp8.T && vp8.TID > uint8(maxTemporalLayer) { + v.exemptedPictureIds.Set(extPictureId, true) // trim cache if necessary - for v.missingPictureIds.Len() > 50 { - el := v.missingPictureIds.Front() - v.missingPictureIds.Delete(el.Key) + for v.exemptedPictureIds.Len() > exemptedPictureIdsThreshold { + el := v.exemptedPictureIds.Front() + v.exemptedPictureIds.Delete(el.Key) } } } else { - if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { - // adjust only once per picture as a picture could have multiple packets - if vp8.PictureIDPresent == 1 && prevMaxPictureId != extPictureId { - v.lastDroppedPictureId = extPictureId - v.pictureIdOffset += 1 + if vp8.T && vp8.TID > uint8(maxTemporalLayer) { + // drop only if not exempted + _, ok := v.exemptedPictureIds.Get(extPictureId) + if !ok { + // adjust only once per picture as a picture could have multiple packets + if vp8.I && prevMaxPictureId != extPictureId { + // keep track of dropped picture ids so that they do not get into the missing picture cache + v.droppedPictureIds.Set(extPictureId, true) + // trim cache if necessary + for v.droppedPictureIds.Len() > droppedPictureIdsThreshold { + el := v.droppedPictureIds.Front() + v.droppedPictureIds.Delete(el.Key) + } + + v.pictureIdOffset += 1 + } + return nil, ErrFilteredVP8TemporalLayer } - return nil, ErrFilteredVP8TemporalLayer } } @@ -247,38 +284,36 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb v.lastKeyIdx = mungedKeyIdx vp8Packet := &buffer.VP8{ - FirstByte: vp8.FirstByte, - PictureIDPresent: vp8.PictureIDPresent, - PictureID: mungedPictureId, - MBit: mungedPictureId > 127, - TL0PICIDXPresent: vp8.TL0PICIDXPresent, - TL0PICIDX: mungedTl0PicIdx, - TIDPresent: vp8.TIDPresent, - TID: vp8.TID, - Y: vp8.Y, - KEYIDXPresent: vp8.KEYIDXPresent, - KEYIDX: mungedKeyIdx, - IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + FirstByte: vp8.FirstByte, + I: vp8.I, + M: mungedPictureId > 127, + PictureID: mungedPictureId, + L: vp8.L, + TL0PICIDX: mungedTl0PicIdx, + T: vp8.T, + TID: vp8.TID, + Y: vp8.Y, + K: vp8.K, + KEYIDX: mungedKeyIdx, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return &TranslationParamsVP8{ - Header: vp8Packet, - }, nil + return vp8Packet.Marshal() } -func (v *VP8Munger) UpdateAndGetPadding(newPicture bool) *buffer.VP8 { +func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { offset := 0 if newPicture { offset = 1 } headerSize := 1 - if (v.pictureIdUsed + v.tl0PicIdxUsed + v.tidUsed + v.keyIdxUsed) != 0 { + if v.pictureIdUsed || v.tl0PicIdxUsed || v.tidUsed || v.keyIdxUsed { headerSize += 1 } extPictureId := v.extLastPictureId - if v.pictureIdUsed == 1 { + if v.pictureIdUsed { extPictureId = v.extLastPictureId + int32(offset) v.extLastPictureId = extPictureId v.pictureIdOffset -= int32(offset) @@ -291,44 +326,44 @@ func (v *VP8Munger) UpdateAndGetPadding(newPicture bool) *buffer.VP8 { pictureId := uint16(extPictureId & 0x7fff) tl0PicIdx := uint8(0) - if v.tl0PicIdxUsed == 1 { + if v.tl0PicIdxUsed { tl0PicIdx = v.lastTl0PicIdx + uint8(offset) v.lastTl0PicIdx = tl0PicIdx v.tl0PicIdxOffset -= uint8(offset) headerSize += 1 } - if (v.tidUsed + v.keyIdxUsed) != 0 { + if v.tidUsed || v.keyIdxUsed { headerSize += 1 } keyIdx := uint8(0) - if v.keyIdxUsed == 1 { + if v.keyIdxUsed { keyIdx = (v.lastKeyIdx + uint8(offset)) & 0x1f v.lastKeyIdx = keyIdx v.keyIdxOffset -= uint8(offset) } vp8Packet := &buffer.VP8{ - FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame - PictureIDPresent: v.pictureIdUsed, - PictureID: pictureId, - MBit: pictureId > 127, - TL0PICIDXPresent: v.tl0PicIdxUsed, - TL0PICIDX: tl0PicIdx, - TIDPresent: v.tidUsed, - TID: 0, - Y: 1, - KEYIDXPresent: v.keyIdxUsed, - KEYIDX: keyIdx, - IsKeyFrame: true, - HeaderSize: headerSize, + FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame + I: v.pictureIdUsed, + M: pictureId > 127, + PictureID: pictureId, + L: v.tl0PicIdxUsed, + TL0PICIDX: tl0PicIdx, + T: v.tidUsed, + TID: 0, + Y: true, + K: v.keyIdxUsed, + KEYIDX: keyIdx, + IsKeyFrame: true, + HeaderSize: headerSize, } - return vp8Packet + return vp8Packet.Marshal() } // for testing only -func (v *VP8Munger) PictureIdOffset(extPictureId int32) (int32, bool) { +func (v *VP8) PictureIdOffset(extPictureId int32) (int32, bool) { return v.missingPictureIds.Get(extPictureId) } diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go new file mode 100644 index 000000000..c72965189 --- /dev/null +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -0,0 +1,524 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codecmunger + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/testutils" +) + +func compare(expected *VP8, actual *VP8) bool { + return reflect.DeepEqual(expected.pictureIdWrapHandler, actual.pictureIdWrapHandler) && + expected.extLastPictureId == actual.extLastPictureId && + expected.pictureIdOffset == actual.pictureIdOffset && + expected.pictureIdUsed == actual.pictureIdUsed && + expected.lastTl0PicIdx == actual.lastTl0PicIdx && + expected.tl0PicIdxOffset == actual.tl0PicIdxOffset && + expected.tl0PicIdxUsed == actual.tl0PicIdxUsed && + expected.tidUsed == actual.tidUsed && + expected.lastKeyIdx == actual.lastKeyIdx && + expected.keyIdxOffset == actual.keyIdxOffset && + expected.keyIdxUsed == actual.keyIdxUsed +} + +func newVP8() *VP8 { + return NewVP8(logger.GetLogger()) +} + +func TestSetLast(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, err := testutils.GetTestExtPacketVP8(params, vp8) + require.NoError(t, err) + require.NotNil(t, extPkt) + + expectedVP8 := VP8{ + pictureIdWrapHandler: VP8PictureIdWrapHandler{ + maxPictureId: 13466, + maxMBit: true, + totalWrap: 0, + lastWrap: 0, + }, + extLastPictureId: 13467, + pictureIdOffset: 0, + pictureIdUsed: true, + lastTl0PicIdx: 233, + tl0PicIdxOffset: 0, + tl0PicIdxUsed: true, + tidUsed: true, + lastKeyIdx: 23, + keyIdxOffset: 0, + keyIdxUsed: true, + } + + v.SetLast(extPkt) + require.True(t, compare(&expectedVP8, v)) +} + +func TestUpdateOffsets(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) + v.SetLast(extPkt) + + params = &testutils.TestExtPacketParams{ + SequenceNumber: 56789, + Timestamp: 0xabcdef, + SSRC: 0x87654321, + } + vp8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 345, + L: true, + TL0PICIDX: 12, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 4, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + v.UpdateOffsets(extPkt) + + expectedVP8 := VP8{ + pictureIdWrapHandler: VP8PictureIdWrapHandler{ + maxPictureId: 344, + maxMBit: true, + totalWrap: 0, + lastWrap: 0, + }, + extLastPictureId: 13467, + pictureIdOffset: 345 - 13467 - 1, + pictureIdUsed: true, + lastTl0PicIdx: 233, + tl0PicIdxOffset: (12 - 233 - 1) & 0xff, + tl0PicIdxUsed: true, + tidUsed: true, + lastKeyIdx: 23, + keyIdxOffset: (4 - 23 - 1) & 0x1f, + keyIdxUsed: true, + } + require.True(t, compare(&expectedVP8, v)) +} + +func TestOutOfOrderPictureId(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) + v.SetLast(extPkt) + v.UpdateAndGet(extPkt, false, false, 2) + + // out-of-order sequence number not in the missing picture id cache + vp8.PictureID = 13466 + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + codecBytes, err := v.UpdateAndGet(extPkt, true, false, 2) + require.Error(t, err) + require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) + require.Nil(t, codecBytes) + + // create a hole in picture id + vp8.PictureID = 13469 + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) + + // all three, the last, the current and the in-between should have been added to missing picture id cache + value, ok := v.PictureIdOffset(13467) + require.True(t, ok) + require.EqualValues(t, 0, value) + + value, ok = v.PictureIdOffset(13468) + require.True(t, ok) + require.EqualValues(t, 0, value) + + value, ok = v.PictureIdOffset(13469) + require.True(t, ok) + require.EqualValues(t, 0, value) + + // out-of-order sequence number should be in the missing picture id cache + vp8.PictureID = 13468 + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + codecBytes, err = v.UpdateAndGet(extPkt, true, false, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) +} + +func TestTemporalLayerFiltering(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) + v.SetLast(extPkt) + + // translate + tp, err := v.UpdateAndGet(extPkt, false, false, 0) + require.Error(t, err) + require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) + require.Nil(t, tp) + dropped, _ := v.droppedPictureIds.Get(13467) + require.True(t, dropped) + require.EqualValues(t, 1, v.pictureIdOffset) + + // another packet with the same picture id. + // It should be dropped, but offset should not be updated. + params.SequenceNumber = 23334 + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + tp, err = v.UpdateAndGet(extPkt, false, false, 0) + require.Error(t, err) + require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) + require.Nil(t, tp) + dropped, _ = v.droppedPictureIds.Get(13467) + require.True(t, dropped) + require.EqualValues(t, 1, v.pictureIdOffset) + + // another packet with the same picture id, but a gap in sequence number. + // It should be dropped, but offset should not be updated. + params.SequenceNumber = 23337 + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + tp, err = v.UpdateAndGet(extPkt, false, false, 0) + require.Error(t, err) + require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) + require.Nil(t, tp) + dropped, _ = v.droppedPictureIds.Get(13467) + require.True(t, dropped) + require.EqualValues(t, 1, v.pictureIdOffset) +} + +func TestGapInSequenceNumberSamePicture(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 65533, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 33, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) + v.SetLast(extPkt) + + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + codecBytes, err := v.UpdateAndGet(extPkt, false, false, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) + + // telling there is a gap in sequence number will add pictures to missing picture cache + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) + + value, ok := v.PictureIdOffset(13467) + require.True(t, ok) + require.EqualValues(t, 0, value) +} + +func TestUpdateAndGetPadding(t *testing.T) { + v := newVP8() + + params := &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + } + vp8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) + + v.SetLast(extPkt) + + // getting padding with repeat of last picture + blankBytes, err := v.UpdateAndGetPadding(false) + require.NoError(t, err) + expectedVP8 := buffer.VP8{ + FirstByte: 16, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankBytes) + + // getting padding with new picture + blankBytes, err = v.UpdateAndGetPadding(true) + require.NoError(t, err) + expectedVP8 = buffer.VP8{ + FirstByte: 16, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankBytes) +} + +func TestVP8PictureIdWrapHandler(t *testing.T) { + v := &VP8PictureIdWrapHandler{} + + v.Init(109, false) + require.Equal(t, int32(109), v.MaxPictureId()) + require.False(t, v.maxMBit) + + v.UpdateMaxPictureId(109350, true) + require.Equal(t, int32(109350), v.MaxPictureId()) + require.True(t, v.maxMBit) + + // start with something close to the 15-bit wrap around point + v.Init(32766, true) + + // out-of-order, do not wrap + extPictureId := v.Unwrap(32750, true) + require.Equal(t, int32(32750), extPictureId) + require.Equal(t, int32(0), v.totalWrap) + require.Equal(t, int32(0), v.lastWrap) + + // wrap at 15-bits + extPictureId = v.Unwrap(5, false) + require.Equal(t, int32(32773), extPictureId) // 15-bit wrap at 32768 + 5 = 32773 + require.Equal(t, int32(32768), v.totalWrap) + require.Equal(t, int32(32768), v.lastWrap) + + // set things near 7-bit wrap point + v.UpdateMaxPictureId(32893, false) // 32768 + 125 + + // wrap at 7-bits + extPictureId = v.Unwrap(5, true) + require.Equal(t, int32(32901), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 5 = 32901 + require.Equal(t, int32(32896), v.totalWrap) // one 15-bit wrap + one 7-bit wrap + require.Equal(t, int32(128), v.lastWrap) + + // a new picture in 7-bit mode much with a gap in between. + // A big enough gap which would have been treated as out-of-order in 7-bit mode. + v.UpdateMaxPictureId(32901, false) + extPictureId = v.Unwrap(73, false) + require.Equal(t, int32(32841), extPictureId) // 15-bit wrap at 32768 + 73 = 32841 + + // a new picture in 15-bit mode much with a gap in between. + // A big enough gap which would have been treated as out-of-order in 7-bit mode. + v.UpdateMaxPictureId(32901, true) + v.lastWrap = int32(32768) + extPictureId = v.Unwrap(73, false) + require.Equal(t, int32(32969), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 73 = 32969 +} diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index ac6506bd8..565eccabb 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( @@ -6,37 +20,41 @@ import ( "time" "github.com/frostbyte73/core" + "github.com/pion/webrtc/v3" "go.uber.org/atomic" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/pion/webrtc/v3" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( - UpdateInterval = 5 * time.Second - processThreshold = 0.95 + UpdateInterval = 5 * time.Second + noReceiverReportTooLongThreshold = 30 * time.Second ) type ConnectionStatsParams struct { - UpdateInterval time.Duration - MimeType string - IsFECEnabled bool - GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers - Logger logger.Logger + UpdateInterval time.Duration + MimeType string + IsFECEnabled bool + IncludeRTT bool + IncludeJitter bool + GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers + GetDeltaStatsOverridden func() map[uint32]*buffer.StreamStatsWithLayers + GetLastReceiverReportTime func() time.Time + Logger logger.Logger } type ConnectionStats struct { - params ConnectionStatsParams - isVideo atomic.Bool + params ConnectionStatsParams + + isStarted atomic.Bool + isVideo atomic.Bool onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat) - lock sync.RWMutex - lastStatsAt time.Time - statsInProcess bool + lock sync.RWMutex + streamingStartedAt time.Time scorer *qualityScorer @@ -48,20 +66,37 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { params: params, scorer: newQualityScorer(qualityScorerParams{ PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? + IncludeRTT: params.IncludeRTT, + IncludeJitter: params.IncludeJitter, Logger: params.Logger, }), done: core.NewFuse(), } } -func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo, at time.Time) { +func (cs *ConnectionStats) start(trackInfo *livekit.TrackInfo) { cs.isVideo.Store(trackInfo.Type == livekit.TrackType_VIDEO) - - cs.scorer.Start(at) - go cs.updateStatsWorker() } +func (cs *ConnectionStats) StartAt(trackInfo *livekit.TrackInfo, at time.Time) { + if cs.isStarted.Swap(true) { + return + } + + cs.scorer.StartAt(at) + cs.start(trackInfo) +} + +func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo) { + if cs.isStarted.Swap(true) { + return + } + + cs.scorer.Start() + cs.start(trackInfo) +} + func (cs *ConnectionStats) Close() { cs.done.Break() } @@ -70,106 +105,209 @@ func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *live cs.onStatsUpdate = fn } -func (cs *ConnectionStats) UpdateMute(isMuted bool, at time.Time) { - cs.scorer.UpdateMute(isMuted, at) +func (cs *ConnectionStats) UpdateMuteAt(isMuted bool, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateMuteAt(isMuted, at) } -func (cs *ConnectionStats) AddBitrateTransition(bitrate int64, at time.Time) { - cs.scorer.AddBitrateTransition(bitrate, at) +func (cs *ConnectionStats) UpdateMute(isMuted bool) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateMute(isMuted) } -func (cs *ConnectionStats) AddLayerTransition(distance float64, at time.Time) { - cs.scorer.AddLayerTransition(distance, at) +func (cs *ConnectionStats) AddBitrateTransitionAt(bitrate int64, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddBitrateTransitionAt(bitrate, at) +} + +func (cs *ConnectionStats) AddBitrateTransition(bitrate int64) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddBitrateTransition(bitrate) +} + +func (cs *ConnectionStats) UpdateLayerMuteAt(isMuted bool, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateLayerMuteAt(isMuted, at) +} + +func (cs *ConnectionStats) UpdateLayerMute(isMuted bool) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateLayerMute(isMuted) +} + +func (cs *ConnectionStats) UpdatePauseAt(isPaused bool, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdatePauseAt(isPaused, at) +} + +func (cs *ConnectionStats) UpdatePause(isPaused bool) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdatePause(isPaused) +} + +func (cs *ConnectionStats) AddLayerTransitionAt(distance float64, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddLayerTransitionAt(distance, at) +} + +func (cs *ConnectionStats) AddLayerTransition(distance float64) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddLayerTransition(distance) } func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQuality) { return cs.scorer.GetMOSAndQuality() } -func (cs *ConnectionStats) ReceiverReportReceived(at time.Time) { - cs.getStat(at) -} - -func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { +func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at time.Time) float32 { var stat windowStat - for _, s := range streams { - if stat.startedAt.IsZero() || stat.startedAt.After(s.RTPStats.StartTime) { - stat.startedAt = s.RTPStats.StartTime - } - if stat.duration < s.RTPStats.Duration { - stat.duration = s.RTPStats.Duration - } - stat.packetsExpected += s.RTPStats.Packets + s.RTPStats.PacketsPadding - stat.packetsLost += s.RTPStats.PacketsLost - stat.packetsMissing += s.RTPStats.PacketsMissing - if stat.rttMax < s.RTPStats.RttMax { - stat.rttMax = s.RTPStats.RttMax - } - if stat.jitterMax < s.RTPStats.JitterMax { - stat.jitterMax = s.RTPStats.JitterMax - } - stat.bytes += s.RTPStats.Bytes - s.RTPStats.HeaderBytes // only use media payload size + if agg != nil { + stat.startedAt = agg.StartTime + stat.duration = agg.Duration + stat.packetsExpected = agg.Packets + agg.PacketsPadding + stat.packetsLost = agg.PacketsLost + stat.packetsMissing = agg.PacketsMissing + stat.packetsOutOfOrder = agg.PacketsOutOfOrder + stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size + stat.rttMax = agg.RttMax + stat.jitterMax = agg.JitterMax + } + if at.IsZero() { + cs.scorer.Update(&stat) + } else { + cs.scorer.UpdateAt(&stat, at) } - cs.scorer.Update(&stat, at) mos, _ := cs.scorer.GetMOSAndQuality() return mos } -func (cs *ConnectionStats) maybeMarkInProcess() bool { - cs.lock.Lock() - defer cs.lock.Unlock() - - if cs.statsInProcess { - // already running - return false +func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { + if cs.params.GetDeltaStatsOverridden == nil || cs.params.GetLastReceiverReportTime == nil { + return MinMOS, nil } - interval := cs.params.UpdateInterval - if interval == 0 { - interval = UpdateInterval + cs.lock.RLock() + streamingStartedAt := cs.streamingStartedAt + cs.lock.RUnlock() + if streamingStartedAt.IsZero() { + // not streaming, just return current score + mos, _ := cs.scorer.GetMOSAndQuality() + return mos, nil } - if cs.lastStatsAt.IsZero() || time.Since(cs.lastStatsAt) > time.Duration(processThreshold*float64(interval)) { - cs.statsInProcess = true - return true + streams := cs.params.GetDeltaStatsOverridden() + if len(streams) == 0 { + // check for receiver report not received for a while + marker := cs.params.GetLastReceiverReportTime() + if marker.IsZero() || streamingStartedAt.After(marker) { + marker = streamingStartedAt + } + if time.Since(marker) > noReceiverReportTooLongThreshold { + // have not received receiver report for a long time when streaming, run with nil stat + return cs.updateScoreWithAggregate(nil, at), nil + } + + // wait for receiver report, return current score + mos, _ := cs.scorer.GetMOSAndQuality() + return mos, nil } - return false + // delta stat duration could be large due to not receiving receiver report for a long time (for example, due to mute), + // adjust to streaming start if necessary + agg := toAggregateDeltaInfo(streams) + if streamingStartedAt.After(cs.params.GetLastReceiverReportTime()) { + // last receiver report was before streaming started, wait for next one + mos, _ := cs.scorer.GetMOSAndQuality() + return mos, streams + } + + if streamingStartedAt.After(agg.StartTime) { + agg.Duration = agg.StartTime.Add(agg.Duration).Sub(streamingStartedAt) + agg.StartTime = streamingStartedAt + } + return cs.updateScoreWithAggregate(agg, at), streams } -func (cs *ConnectionStats) updateInProcess(isAvailable bool, at time.Time) { - cs.lock.Lock() - defer cs.lock.Unlock() - - cs.statsInProcess = false - if isAvailable { - cs.lastStatsAt = at - } -} - -func (cs *ConnectionStats) getStat(at time.Time) { +func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { if cs.params.GetDeltaStats == nil { - return - } - - if !cs.maybeMarkInProcess() { - // not yet time to process - return + return MinMOS, nil } streams := cs.params.GetDeltaStats() if len(streams) == 0 { - cs.updateInProcess(false, at) - return + mos, _ := cs.scorer.GetMOSAndQuality() + return mos, nil } - // stats available, update last stats time - cs.updateInProcess(true, at) + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) + for _, s := range streams { + deltaInfoList = append(deltaInfoList, s.RTPStats) + } + agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) + if agg != nil && agg.Packets > 0 { + // not very accurate as streaming could have started part way in the window, but don't need accurate time + cs.maybeSetStreamingStart(agg.StartTime) + } else { + cs.clearStreamingStart() + } - score := cs.updateScore(streams, at) + if cs.params.GetDeltaStatsOverridden != nil { + // receiver report based quality scoring, use stats from receiver report for scoring + return cs.updateScoreFromReceiverReport(at) + } - if cs.onStatsUpdate != nil { + return cs.updateScoreWithAggregate(agg, at), streams +} + +func (cs *ConnectionStats) maybeSetStreamingStart(at time.Time) { + cs.lock.Lock() + if cs.streamingStartedAt.IsZero() { + cs.streamingStartedAt = at + } + cs.lock.Unlock() +} + +func (cs *ConnectionStats) clearStreamingStart() { + cs.lock.Lock() + cs.streamingStartedAt = time.Time{} + cs.lock.Unlock() +} + +func (cs *ConnectionStats) getStat() { + score, streams := cs.updateScoreAt(time.Time{}) + + if cs.onStatsUpdate != nil && len(streams) != 0 { analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams)) for ssrc, stream := range streams { as := toAnalyticsStream(ssrc, stream.RTPStats) @@ -181,7 +319,10 @@ func (cs *ConnectionStats) getStat(at time.Time) { // if (len(streams) > 1 || len(stream.Layers) > 1) && cs.isVideo.Load() { for layer, layerStats := range stream.Layers { - as.VideoLayers = append(as.VideoLayers, toAnalyticsVideoLayer(layer, layerStats)) + avl := toAnalyticsVideoLayer(layer, layerStats) + if avl != nil { + as.VideoLayers = append(as.VideoLayers, avl) + } } } @@ -211,11 +352,11 @@ func (cs *ConnectionStats) updateStatsWorker() { return case <-tk.C: - if cs.done.IsClosed() { + if cs.done.IsBroken() { return } - cs.getStat(time.Now()) + cs.getStat() } } } @@ -225,39 +366,56 @@ func (cs *ConnectionStats) updateStatsWorker() { // how much weight to give to packet loss rate when calculating score. // It is codec dependent. // For audio: -// o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight -// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight +// +// o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight +// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight // // For video: -// o No in-built codec repair available, hence same for all codecs +// +// o No in-built codec repair available, hence same for all codecs func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { - plw := float64(0.0) + var plw float64 switch { case strings.EqualFold(mimeType, webrtc.MimeTypeOpus): - // 2.5%: fall to GOOD, 5%: fall to POOR + // 2.5%: fall to GOOD, 7.5%: fall to POOR plw = 8.0 if isFecEnabled { - // 3.75%: fall to GOOD, 7.5%: fall to POOR + // 3.75%: fall to GOOD, 11.25%: fall to POOR plw /= 1.5 } case strings.EqualFold(mimeType, "audio/red"): - // 6.66%: fall to GOOD, 13.33%: fall to POOR - plw = 3.0 + // 10%: fall to GOOD, 30.0%: fall to POOR + plw = 2.0 if isFecEnabled { - // 10%: fall to GOOD, 20%: fall to POOR + // 15%: fall to GOOD, 45.0%: fall to POOR plw /= 1.5 } case strings.HasPrefix(strings.ToLower(mimeType), "video/"): - // 2%: fall to GOOD, 4%: fall to POOR + // 2%: fall to GOOD, 6%: fall to POOR plw = 10.0 } return plw } +func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers) *buffer.RTPDeltaInfo { + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) + for _, s := range streams { + deltaInfoList = append(deltaInfoList, s.RTPStats) + } + return buffer.AggregateRTPDeltaInfo(deltaInfoList) +} + func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.AnalyticsStream { + // discount the feed side loss when reporting forwarded track stats + packetsLost := deltaStats.PacketsLost + if deltaStats.PacketsMissing > packetsLost { + packetsLost = 0 + } else { + packetsLost -= deltaStats.PacketsMissing + } return &livekit.AnalyticsStream{ Ssrc: ssrc, PrimaryPackets: deltaStats.Packets, @@ -266,7 +424,7 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An RetransmitBytes: deltaStats.BytesDuplicate, PaddingPackets: deltaStats.PacketsPadding, PaddingBytes: deltaStats.BytesPadding, - PacketsLost: deltaStats.PacketsLost, + PacketsLost: packetsLost, Frames: deltaStats.Frames, Rtt: deltaStats.RttMax, Jitter: uint32(deltaStats.JitterMax), @@ -277,10 +435,15 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An } func toAnalyticsVideoLayer(layer int32, layerStats *buffer.RTPDeltaInfo) *livekit.AnalyticsVideoLayer { - return &livekit.AnalyticsVideoLayer{ + avl := &livekit.AnalyticsVideoLayer{ Layer: layer, Packets: layerStats.Packets + layerStats.PacketsDuplicate + layerStats.PacketsPadding, Bytes: layerStats.Bytes + layerStats.BytesDuplicate + layerStats.BytesPadding, Frames: layerStats.Frames, } + if avl.Packets == 0 || avl.Bytes == 0 || avl.Frames == 0 { + return nil + } + + return avl } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 355a4e270..a8aec4b81 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( @@ -5,38 +19,52 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/stretchr/testify/require" ) -func newConnectionStats(mimeType string, isFECEnabled bool) *ConnectionStats { +func newConnectionStats( + mimeType string, + isFECEnabled bool, + includeRTT bool, + includeJitter bool, + getDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers, +) *ConnectionStats { return NewConnectionStats(ConnectionStatsParams{ - MimeType: mimeType, - IsFECEnabled: isFECEnabled, - Logger: logger.GetLogger(), + MimeType: mimeType, + IsFECEnabled: isFECEnabled, + IncludeRTT: includeRTT, + IncludeJitter: includeJitter, + GetDeltaStats: getDeltaStats, + Logger: logger.GetLogger(), }) } func TestConnectionQuality(t *testing.T) { - t.Run("quality scorer state machine", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false) + t.Run("quality scorer operation", func(t *testing.T) { + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) - cs.UpdateMute(false, now.Add(-1*time.Second)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) // no data and not enough unmute time should return default state which is EXCELLENT quality - cs.updateScore(nil, now) + cs.updateScoreAt(now) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // best conditions (no loss, jitter/rtt = 0) - quality should stay EXCELLENT - streams := map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -44,7 +72,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -52,25 +80,34 @@ func TestConnectionQuality(t *testing.T) { // introduce loss and the score should drop - 12% loss for Opus -> POOR now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, - Packets: 250, + Packets: 120, PacketsLost: 30, }, }, + 2: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 130, + PacketsLost: 0, + }, + }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) - // should stay at POOR quality for one iteration even if the conditions improve - // due to significant loss (12%) in the previous window + // should climb to GOOD quality in one iteration if the conditions improve. + // although significant loss (12%) in the previous window, lowest score is + // bound so that climbing back does not take too long even under excellent conditions. now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -78,15 +115,15 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) - require.Equal(t, livekit.ConnectionQuality_POOR, quality) + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) - // should climb up to GOOD if conditions continue to be good + // should stay at GOOD if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -94,7 +131,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -102,7 +139,7 @@ func TestConnectionQuality(t *testing.T) { // should climb up to EXCELLENT if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -110,7 +147,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -118,7 +155,7 @@ func TestConnectionQuality(t *testing.T) { // introduce loss and the score should drop - 5% loss for Opus -> GOOD now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -127,7 +164,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -135,7 +172,7 @@ func TestConnectionQuality(t *testing.T) { // should stay at GOOD quality for another iteration even if the conditions improve now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -143,7 +180,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -151,7 +188,7 @@ func TestConnectionQuality(t *testing.T) { // should climb up to EXCELLENT if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -159,7 +196,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -167,32 +204,32 @@ func TestConnectionQuality(t *testing.T) { // mute when quality is POOR should return quality to EXCELLENT now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, Packets: 250, - PacketsLost: 25, + PacketsLost: 30, }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // unmute at time so that next window does not satisfy the unmute time threshold. // that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality - cs.UpdateMute(false, now.Add(3*time.Second)) + cs.UpdateMuteAt(false, now.Add(3*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -200,7 +237,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -208,7 +245,7 @@ func TestConnectionQuality(t *testing.T) { // next update with no packets should knock quality down now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -216,20 +253,20 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // with lesser number of packet (simulating DTX). - // even higher loss (like 10%) should only knock down quality to GOOD, typically would be POOR at that loss rate + // even higher loss (like 10%) should not knock down quality due to quadratic weighting of packet loss ratio streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -238,63 +275,190 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(4.1), mos) - require.Equal(t, livekit.ConnectionQuality_GOOD, quality) + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // RTT and jitter can knock quality down. // at 2% loss, quality should stay at EXCELLENT purely based on loss, but with added RTT/jitter, should drop to GOOD streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, Packets: 250, PacketsLost: 5, - RttMax: 300, + RttMax: 400, JitterMax: 30000, }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // bitrate based calculation can drop quality even if there is no loss - cs.AddBitrateTransition(1_000_000, now) - cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + cs.AddBitrateTransitionAt(1_000_000, now) + cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, Packets: 250, - Bytes: 8_000_000 / 8 / 4, + Bytes: 8_000_000 / 8 / 5, }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) - // a transition to 0 (all layers stopped) should flip quality to EXCELLENT - now = now.Add(duration) - cs.AddBitrateTransition(0, now) + // test layer mute via UpdateLayerMute API + cs.AddBitrateTransitionAt(1_000_000, now) + cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) + + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 5, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) + + now = now.Add(duration) + cs.UpdateLayerMuteAt(true, now) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + + // unmute layer + cs.UpdateLayerMuteAt(false, now.Add(2*time.Second)) + + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 5, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + + // pause + now = now.Add(duration) + cs.UpdatePauseAt(true, now) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // resume + cs.UpdatePauseAt(false, now.Add(2*time.Second)) + + // although conditions are perfect, climbing back from POOR (because of pause above) + // will only climb to GOOD. + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 5, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) + }) + + t.Run("quality scorer dependent rtt", func(t *testing.T) { + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, false, true, getDeltaStats) + + duration := 5 * time.Second + now := time.Now() + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) + + // RTT does not knock quality down because it is dependent and hence not taken into account + // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms) + // quality should drop to GOOD if RTT were taken into consideration + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + PacketsLost: 5, + RttMax: 700, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) + mos, quality := cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + }) + + t.Run("quality scorer dependent jitter", func(t *testing.T) { + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, true, false, getDeltaStats) + + duration := 5 * time.Second + now := time.Now() + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) + + // Jitter does not knock quality down because it is dependent and hence not taken into account + // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms) + // quality should drop to GOOD if jitter were taken into consideration + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + PacketsLost: 5, + JitterMax: 200, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) + mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) }) @@ -313,7 +477,7 @@ func TestConnectionQuality(t *testing.T) { expectedQualities []expectedQuality }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact - // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 5%: GOOD, >= 5%: POOR + // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR { name: "audio/opus - no fec", mimeType: "audio/opus", @@ -331,13 +495,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.2, - expectedMOS: 3.2, + packetLossPercentage: 9.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 7.5%: GOOD, >= 7.5%: POOR + // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR { name: "audio/opus - fec", mimeType: "audio/opus", @@ -350,47 +514,23 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 4.1, + packetLossPercentage: 4.4, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 8.2, - expectedMOS: 3.2, + packetLossPercentage: 15.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 13.33%: GOOD, >= 13.33%: POOR + // "audio/red" - no fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR { name: "audio/red - no fec", mimeType: "audio/red", isFECEnabled: false, packetsExpected: 200, - expectedQualities: []expectedQuality{ - { - packetLossPercentage: 6.0, - expectedMOS: 4.6, - expectedQuality: livekit.ConnectionQuality_EXCELLENT, - }, - { - packetLossPercentage: 10.0, - expectedMOS: 4.1, - expectedQuality: livekit.ConnectionQuality_GOOD, - }, - { - packetLossPercentage: 16.0, - expectedMOS: 3.2, - expectedQuality: livekit.ConnectionQuality_POOR, - }, - }, - }, - // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 20%: GOOD, >= 20%: POOR - { - name: "audio/red - fec", - mimeType: "audio/red", - isFECEnabled: true, - packetsExpected: 200, expectedQualities: []expectedQuality{ { packetLossPercentage: 8.0, @@ -398,18 +538,42 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 18.0, + packetLossPercentage: 12.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 22.0, - expectedMOS: 3.2, + packetLossPercentage: 39.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 4%: GOOD, >= 4%: POOR + // "audio/red" - fec - 0 <= loss < 15%: EXCELLENT, 15% <= loss < 45%: GOOD, >= 45%: POOR + { + name: "audio/red - fec", + mimeType: "audio/red", + isFECEnabled: true, + packetsExpected: 200, + expectedQualities: []expectedQuality{ + { + packetLossPercentage: 12.0, + expectedMOS: 4.6, + expectedQuality: livekit.ConnectionQuality_EXCELLENT, + }, + { + packetLossPercentage: 20.0, + expectedMOS: 4.1, + expectedQuality: livekit.ConnectionQuality_GOOD, + }, + { + packetLossPercentage: 60.0, + expectedMOS: 2.1, + expectedQuality: livekit.ConnectionQuality_POOR, + }, + }, + }, + // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR { name: "video/*", mimeType: "video/vp8", @@ -422,13 +586,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 2.5, + packetLossPercentage: 3.5, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.0, - expectedMOS: 3.2, + packetLossPercentage: 8.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, @@ -437,15 +601,19 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) for _, eq := range tc.expectedQualities { - streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -454,7 +622,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, eq.expectedMOS, mos) require.Equal(t, eq.expectedQuality, quality) @@ -479,8 +647,8 @@ func TestConnectionQuality(t *testing.T) { }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact // 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT - // ~2.7 <= expectedBits / actualBits < ~7.5 = GOOD - // expectedBits / actualBits >= ~7.5 = POOR + // ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD + // expectedBits / actualBits >= ~20.1 = POOR { name: "excellent", transitions: []transition{ @@ -507,7 +675,7 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(7_000_000.0 / 8.0 / 3.5)), + bytes: uint64(math.Ceil(7_000_000.0 / 8.0 / 4.2)), expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, @@ -522,26 +690,30 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 13.0)), - expectedMOS: 3.2, + bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 75.0)), + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("video/vp8", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) for _, tr := range tc.transitions { - cs.AddBitrateTransition(tr.bitrate, now.Add(tr.offset)) + cs.AddBitrateTransitionAt(tr.bitrate, now.Add(tr.offset)) } - streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -550,7 +722,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) @@ -606,29 +778,33 @@ func TestConnectionQuality(t *testing.T) { distance: 2.0, }, { - distance: 2.7, + distance: 2.6, offset: 1 * time.Second, }, }, - expectedMOS: 3.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("video/vp8", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) for _, tr := range tc.transitions { - cs.AddLayerTransition(tr.distance, now.Add(tr.offset)) + cs.AddLayerTransitionAt(tr.distance, now.Add(tr.offset)) } - streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -636,7 +812,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 1cca32299..340f32368 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( @@ -8,18 +22,21 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" ) const ( MaxMOS = float32(4.5) + MinMOS = float32(1.0) maxScore = float64(100.0) - poorScore = float64(50.0) + poorScore = float64(30.0) + minScore = float64(20.0) - increaseFactor = float64(0.4) // slow increase - decreaseFactor = float64(0.8) // fast decrease + increaseFactor = float64(0.4) // slower increase, i. e. when score is recovering move up slower -> conservative + decreaseFactor = float64(0.7) // faster decrease, i. e. when score is dropping move down faster -> aggressive to be responsive to quality drops - distanceWeight = float64(20.0) // each spatial layer missed drops a quality level + distanceWeight = float64(35.0) // each spatial layer missed drops a quality level unmuteTimeThreshold = float64(0.5) ) @@ -27,31 +44,64 @@ const ( // ------------------------------------------ type windowStat struct { - startedAt time.Time - duration time.Duration - packetsExpected uint32 - packetsLost uint32 - packetsMissing uint32 - bytes uint64 - rttMax uint32 - jitterMax float64 + startedAt time.Time + duration time.Duration + packetsExpected uint32 + packetsLost uint32 + packetsMissing uint32 + packetsOutOfOrder uint32 + bytes uint64 + rttMax uint32 + jitterMax float64 } -func (w *windowStat) calculatePacketScore(plw float64) float64 { +func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJitter bool) float64 { // this is based on simplified E-model based on packet loss, rtt, jitter as // outlined at https://www.pingman.com/kb/article/how-is-mos-calculated-in-pingplotter-pro-50.html. - effectiveDelay := (float64(w.rttMax) / 2.0) + ((w.jitterMax * 2.0) / 1000.0) + effectiveDelay := 0.0 + // discount the dependent factors if dependency indicated. + // for example, + // 1. in the up stream, RTT cannot be measured without RTCP-XR, it is using down stream RTT. + // 2. in the down stream, up stream jitter affects it. although jitter can be adjusted to account for up stream + // jitter, this lever can be used to discount jitter in scoring. + if includeRTT { + effectiveDelay += float64(w.rttMax) / 2.0 + } + if includeJitter { + effectiveDelay += (w.jitterMax * 2.0) / 1000.0 + } delayEffect := effectiveDelay / 40.0 if effectiveDelay > 160.0 { delayEffect = (effectiveDelay - 120.0) / 10.0 } - actualLost := w.packetsLost - w.packetsMissing + // discount out-of-order packets from loss to deal with a scenario like + // 1. up stream has loss + // 2. down stream forwards with loss/hole in sequence number + // 3. down stream client reports a certain number of loss + // 4. while processing that, up stream could have retransmitted missing packets + // 5. those retransmitted packets are forwarded, + // - server's view: it has forwarded those packets + // - client's view: it had not seen those packets when sending RTCP RR + // so those retransmitted packets appear like down stream loss to server. + // + // retransmitted packets would have arrived out-of-order. So, discounting them + // will account for it. + // + // Note that packets can arrive out-of-order in the upstream during regular + // streaming as well, i. e. without loss + NACK + retransmit. Those will be + // discounted too. And that will skew the real loss. For example, let + // us say that 40 out of 100 packets were reported lost by down stream. + // These could be real losses. In the same window, 40 packets could have been + // delivered out-of-order by the up stream, thus cancelling out the real loss. + // But, those situations should be rare and is a compromise for not letting + // up stream loss penalise down stream. + actualLost := w.packetsLost - w.packetsMissing - w.packetsOutOfOrder if int32(actualLost) < 0 { actualLost = 0 } - lossEffect := float64(0.0) + var lossEffect float64 if w.packetsExpected > 0 { lossEffect = float64(actualLost) * 100.0 / float64(w.packetsExpected) } @@ -71,11 +121,11 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { return maxScore } - score := float64(0.0) + var score float64 if w.bytes != 0 { // using the ratio of expectedBitrate / actualBitrate // the quality inflection points are approximately - // GOOD at ~2.7x, POOR at ~7.5x + // GOOD at ~2.7x, POOR at ~20.1x score = maxScore - 20*math.Log(float64(expectedBitrate)/float64(w.bytes*8)) if score > maxScore { score = maxScore @@ -89,11 +139,13 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { } func (w *windowStat) String() string { - return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, b: %d, rtt: %d, jitter: %0.2f", + return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f", w.startedAt, w.duration, w.packetsExpected, w.packetsLost, + w.packetsMissing, + w.packetsOutOfOrder, w.bytes, w.rttMax, w.jitterMax, @@ -102,18 +154,10 @@ func (w *windowStat) String() string { // ------------------------------------------ -type bitrateTransition struct { - startedAt time.Time - bitrate int64 -} - -type layerTransition struct { - startedAt time.Time - distance float64 -} - type qualityScorerParams struct { PacketLossWeight float64 + IncludeRTT bool + IncludeJitter bool Logger logger.Logger } @@ -124,37 +168,55 @@ type qualityScorer struct { lastUpdateAt time.Time score float64 + stat windowStat mutedAt time.Time unmutedAt time.Time - bitrateMutedAt time.Time - bitrateUnmutedAt time.Time + layerMutedAt time.Time + layerUnmutedAt time.Time + + pausedAt time.Time + resumedAt time.Time maxPPS float64 - bitrateTransitions []bitrateTransition - layerTransitions []layerTransition + aggregateBitrate *utils.TimedAggregator[int64] + layerDistance *utils.TimedAggregator[float64] } func newQualityScorer(params qualityScorerParams) *qualityScorer { return &qualityScorer{ params: params, score: maxScore, + aggregateBitrate: utils.NewTimedAggregator[int64](utils.TimedAggregatorParams{ + CapNegativeValues: true, + }), + layerDistance: utils.NewTimedAggregator[float64](utils.TimedAggregatorParams{ + CapNegativeValues: true, + }), } } -func (q *qualityScorer) Start(at time.Time) { - q.lock.Lock() - defer q.lock.Unlock() - +func (q *qualityScorer) startAtLocked(at time.Time) { q.lastUpdateAt = at } -func (q *qualityScorer) UpdateMute(isMuted bool, at time.Time) { +func (q *qualityScorer) StartAt(at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.startAtLocked(at) +} + +func (q *qualityScorer) Start() { + q.lock.Lock() + defer q.lock.Unlock() + + q.startAtLocked(time.Now()) +} + +func (q *qualityScorer) updateMuteAtLocked(isMuted bool, at time.Time) { if isMuted { q.mutedAt = at q.score = maxScore @@ -163,61 +225,149 @@ func (q *qualityScorer) UpdateMute(isMuted bool, at time.Time) { } } -func (q *qualityScorer) AddBitrateTransition(bitrate int64, at time.Time) { +func (q *qualityScorer) UpdateMuteAt(isMuted bool, at time.Time) { q.lock.Lock() defer q.lock.Unlock() - q.bitrateTransitions = append(q.bitrateTransitions, bitrateTransition{ - startedAt: at, - bitrate: bitrate, - }) + q.updateMuteAtLocked(isMuted, at) +} - if bitrate == 0 { - q.bitrateMutedAt = at - q.score = maxScore +func (q *qualityScorer) UpdateMute(isMuted bool) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateMuteAtLocked(isMuted, time.Now()) +} + +func (q *qualityScorer) addBitrateTransitionAtLocked(bitrate int64, at time.Time) { + q.aggregateBitrate.AddSampleAt(bitrate, at) +} + +func (q *qualityScorer) AddBitrateTransitionAt(bitrate int64, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addBitrateTransitionAtLocked(bitrate, at) +} + +func (q *qualityScorer) AddBitrateTransition(bitrate int64) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addBitrateTransitionAtLocked(bitrate, time.Now()) +} + +func (q *qualityScorer) updateLayerMuteAtLocked(isMuted bool, at time.Time) { + if isMuted { + if !q.isLayerMuted() { + q.aggregateBitrate.Reset() + q.layerDistance.Reset() + + q.layerMutedAt = at + q.score = maxScore + } } else { - if q.bitrateUnmutedAt.IsZero() || q.bitrateMutedAt.After(q.bitrateUnmutedAt) { - q.bitrateUnmutedAt = at + if q.isLayerMuted() { + q.layerUnmutedAt = at } } } -func (q *qualityScorer) AddLayerTransition(distance float64, at time.Time) { +func (q *qualityScorer) UpdateLayerMuteAt(isMuted bool, at time.Time) { q.lock.Lock() defer q.lock.Unlock() - q.layerTransitions = append(q.layerTransitions, layerTransition{ - startedAt: at, - distance: distance, - }) + q.updateLayerMuteAtLocked(isMuted, at) } -func (q *qualityScorer) Update(stat *windowStat, at time.Time) { +func (q *qualityScorer) UpdateLayerMute(isMuted bool) { q.lock.Lock() defer q.lock.Unlock() + q.updateLayerMuteAtLocked(isMuted, time.Now()) +} + +func (q *qualityScorer) updatePauseAtLocked(isPaused bool, at time.Time) { + if isPaused { + if !q.isPaused() { + q.aggregateBitrate.Reset() + q.layerDistance.Reset() + + q.pausedAt = at + q.score = poorScore + } + } else { + if q.isPaused() { + q.resumedAt = at + } + } +} + +func (q *qualityScorer) UpdatePauseAt(isPaused bool, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updatePauseAtLocked(isPaused, at) +} + +func (q *qualityScorer) UpdatePause(isPaused bool) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updatePauseAtLocked(isPaused, time.Now()) +} + +func (q *qualityScorer) addLayerTransitionAtLocked(distance float64, at time.Time) { + q.layerDistance.AddSampleAt(distance, at) +} + +func (q *qualityScorer) AddLayerTransitionAt(distance float64, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addLayerTransitionAtLocked(distance, at) +} + +func (q *qualityScorer) AddLayerTransition(distance float64) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addLayerTransitionAtLocked(distance, time.Now()) +} + +func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // always update transitions - expectedBitrate := q.getExpectedBitsAndUpdateTransitions(at) - expectedDistance := q.getExpectedDistanceAndUpdateTransitions(at) + expectedBitrate, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at) + if err != nil { + q.params.Logger.Warnw("error getting expected bitrate", err) + } + expectedDistance, err := q.layerDistance.GetAverageAndRestartAt(at) + if err != nil { + q.params.Logger.Warnw("error getting expected distance", err) + } // nothing to do when muted or not unmuted for long enough // NOTE: it is possible that unmute -> mute -> unmute transition happens in the - // same analysis window. On a transition to mute, state immediately moves - // to stable and quality EXCELLENT for responsiveness. On an unmute, the - // entire window data is considered (as long as enough time has passed since - // unmute) including the data before mute. - if q.isMuted() || !q.isUnmutedEnough(at) || q.isBitrateMuted() { + // same analysis window. On a transition to mute, quality is immediately moved + // EXCELLENT for responsiveness. On an unmute, the entire window data is + // considered (as long as enough time has passed since unmute). + // + // Similarly, when paused (possibly due to congestion), score is immediately + // set to poorScore for responsiveness. The layer transision is reest. + // On a resume, quality climbs back up using normal operation. + if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() { q.lastUpdateAt = at return } + plw := q.getPacketLossWeight(stat) reason := "none" var score float64 if stat.packetsExpected == 0 { reason = "dry" score = poorScore } else { - packetScore := stat.calculatePacketScore(q.getPacketLossWeight(stat)) + packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) bitrateScore := stat.calculateBitrateScore(expectedBitrate) layerScore := math.Max(math.Min(maxScore, maxScore-(expectedDistance*distanceWeight)), 0.0) @@ -237,12 +387,19 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { reason = "layer" score = layerScore } + + factor := increaseFactor + if score < q.score { + factor = decreaseFactor + } + score = factor*score + (1.0-factor)*q.score } - factor := increaseFactor - if score < q.score { - factor = decreaseFactor + if score < minScore { + // lower bound to prevent score from becoming very small values due to extreme conditions. + // Without a lower bound, it can get so low that it takes a long time to climb back to + // better quality even under excellent conditions. + score = minScore } - score = factor*score + (1.0-factor)*q.score // WARNING NOTE: comparing protobuf enum values directly (livekit.ConnectionQuality) if scoreToConnectionQuality(q.score) > scoreToConnectionQuality(score) { q.params.Logger.Infow( @@ -250,18 +407,36 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { "reason", reason, "prevScore", q.score, "prevQuality", scoreToConnectionQuality(q.score), + "prevStat", &q.stat, "score", score, "quality", scoreToConnectionQuality(score), "stat", stat, + "packetLossWeight", plw, + "maxPPS", q.maxPPS, "expectedBitrate", expectedBitrate, "expectedDistance", expectedDistance, ) } q.score = score + q.stat = *stat q.lastUpdateAt = at } +func (q *qualityScorer) UpdateAt(stat *windowStat, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateAtLocked(stat, at) +} + +func (q *qualityScorer) Update(stat *windowStat) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateAtLocked(stat, time.Now()) +} + func (q *qualityScorer) isMuted() bool { return !q.mutedAt.IsZero() && (q.unmutedAt.IsZero() || q.mutedAt.After(q.unmutedAt)) } @@ -274,16 +449,16 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool { sinceUnmute = at.Sub(q.unmutedAt) } - var sinceLayersUnmute time.Duration - if q.bitrateUnmutedAt.IsZero() { - sinceLayersUnmute = at.Sub(q.lastUpdateAt) + var sinceLayerUnmute time.Duration + if q.layerUnmutedAt.IsZero() { + sinceLayerUnmute = at.Sub(q.lastUpdateAt) } else { - sinceLayersUnmute = at.Sub(q.bitrateUnmutedAt) + sinceLayerUnmute = at.Sub(q.layerUnmutedAt) } validDuration := sinceUnmute - if sinceLayersUnmute < validDuration { - validDuration = sinceLayersUnmute + if sinceLayerUnmute < validDuration { + validDuration = sinceLayerUnmute } sinceLastUpdate := at.Sub(q.lastUpdateAt) @@ -291,12 +466,16 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool { return validDuration.Seconds()/sinceLastUpdate.Seconds() > unmuteTimeThreshold } -func (q *qualityScorer) isBitrateMuted() bool { - return !q.bitrateMutedAt.IsZero() && (q.bitrateUnmutedAt.IsZero() || q.bitrateMutedAt.After(q.bitrateUnmutedAt)) +func (q *qualityScorer) isLayerMuted() bool { + return !q.layerMutedAt.IsZero() && (q.layerUnmutedAt.IsZero() || q.layerMutedAt.After(q.layerUnmutedAt)) +} + +func (q *qualityScorer) isPaused() bool { + return !q.pausedAt.IsZero() && (q.resumedAt.IsZero() || q.pausedAt.After(q.resumedAt)) } func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { - if stat == nil { + if stat == nil || stat.duration == 0 { return q.params.PacketLossWeight } @@ -308,103 +487,15 @@ func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { pps := float64(stat.packetsExpected) / stat.duration.Seconds() if pps > q.maxPPS { q.maxPPS = pps + q.params.Logger.Debugw("updating maxPPS", "expected", stat.packetsExpected, "duration", stat.duration.Seconds(), "pps", pps) } - return math.Sqrt(pps/q.maxPPS) * q.params.PacketLossWeight -} - -func (q *qualityScorer) getExpectedBitsAndUpdateTransitions(at time.Time) int64 { - if len(q.bitrateTransitions) == 0 { - return 0 + if q.maxPPS == 0 { + return q.params.PacketLossWeight } - var startedAt time.Time - totalBits := float64(0.0) - for idx := 0; idx < len(q.bitrateTransitions)-1; idx++ { - bt := &q.bitrateTransitions[idx] - btNext := &q.bitrateTransitions[idx+1] - - if bt.startedAt.After(q.lastUpdateAt) { - startedAt = bt.startedAt - } else { - startedAt = q.lastUpdateAt - } - totalBits += btNext.startedAt.Sub(startedAt).Seconds() * float64(bt.bitrate) - } - - // last transition - bt := &q.bitrateTransitions[len(q.bitrateTransitions)-1] - if bt.startedAt.After(q.lastUpdateAt) { - startedAt = bt.startedAt - } else { - startedAt = q.lastUpdateAt - } - totalBits += at.Sub(startedAt).Seconds() * float64(bt.bitrate) - - // set up last bit rate as the startig bit rate for next analysis window - q.bitrateTransitions = []bitrateTransition{bitrateTransition{ - startedAt: at, - bitrate: bt.bitrate, - }} - - return int64(totalBits) -} - -func (q *qualityScorer) getExpectedDistanceAndUpdateTransitions(at time.Time) float64 { - if len(q.layerTransitions) == 0 { - return 0 - } - - var startedAt time.Time - totalDistance := float64(0.0) - totalDuration := time.Duration(0) - for idx := 0; idx < len(q.layerTransitions)-1; idx++ { - lt := &q.layerTransitions[idx] - ltNext := &q.layerTransitions[idx+1] - - if lt.startedAt.After(q.lastUpdateAt) { - startedAt = lt.startedAt - } else { - startedAt = q.lastUpdateAt - } - dur := ltNext.startedAt.Sub(startedAt) - totalDuration += dur - - dist := lt.distance - if dist < 0.0 { - // negative distances are overshoot, that does not compensate for shortfalls, so use optimal, i. e. 0 distance when overshooting - dist = 0.0 - } - totalDistance += dur.Seconds() * float64(dist) - } - - // last transition - lt := &q.layerTransitions[len(q.layerTransitions)-1] - if lt.startedAt.After(q.lastUpdateAt) { - startedAt = lt.startedAt - } else { - startedAt = q.lastUpdateAt - } - dur := at.Sub(startedAt) - totalDuration += dur - - dist := lt.distance - if dist < 0.0 { - dist = 0.0 - } - totalDistance += dur.Seconds() * float64(dist) - - // set up last distance as the startig distance for next analysis window - q.layerTransitions = []layerTransition{layerTransition{ - startedAt: at, - distance: lt.distance, - }} - - if totalDuration == 0 { - return 0 - } - - return totalDistance / totalDuration.Seconds() + packetRatio := pps / q.maxPPS + return packetRatio * packetRatio * q.params.PacketLossWeight } func (q *qualityScorer) GetScoreAndQuality() (float32, livekit.ConnectionQuality) { @@ -424,13 +515,19 @@ func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality) // ------------------------------------------ func scoreToConnectionQuality(score float64) livekit.ConnectionQuality { - // R-factor -> livekit.ConnectionQuality scale mapping based on + // R-factor -> livekit.ConnectionQuality scale mapping roughly based on // https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm + // + // As there are only three levels in livekit.ConnectionQuality scale, + // using a larger range for middling quality. Empirical evidence suggests + // that a score of 60 does not correspond to `POOR` quality. Repair + // mechanisms and use of algorithms like de-jittering makes the experience + // better even under harsh conditions. if score > 80.0 { return livekit.ConnectionQuality_EXCELLENT } - if score > 60.0 { + if score > 40.0 { return livekit.ConnectionQuality_GOOD } diff --git a/pkg/sfu/dependencydescriptor/bitstreamreader.go b/pkg/sfu/dependencydescriptor/bitstreamreader.go index 84700f6b8..62f4d100c 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamreader.go +++ b/pkg/sfu/dependencydescriptor/bitstreamreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( @@ -6,17 +20,17 @@ import ( ) type BitStreamReader struct { - buf []byte - pos int - remaingBits int + buf []byte + pos int + remainingBits int } func NewBitStreamReader(buf []byte) *BitStreamReader { - return &BitStreamReader{buf: buf, remaingBits: len(buf) * 8} + return &BitStreamReader{buf: buf, remainingBits: len(buf) * 8} } -func (b *BitStreamReader) RemaningBits() int { - return b.remaingBits +func (b *BitStreamReader) RemainingBits() int { + return b.remainingBits } // Reads `bits` from the bitstream. `bits` must be in range [0, 64]. @@ -27,17 +41,17 @@ func (b *BitStreamReader) ReadBits(bits int) (uint64, error) { return 0, errors.New("invalid number of bits, expected 0-64") } - if b.remaingBits < bits { - b.remaingBits -= bits + if b.remainingBits < bits { + b.remainingBits -= bits return 0, io.EOF } - remainingBitsInFirstByte := b.remaingBits % 8 - b.remaingBits -= bits + remainingBitsInFirstByte := b.remainingBits % 8 + b.remainingBits -= bits if bits < remainingBitsInFirstByte { // Reading fewer bits than what's left in the current byte, just // return the portion of this byte that is needed. - offset := (remainingBitsInFirstByte - bits) + offset := remainingBitsInFirstByte - bits return uint64((b.buf[b.pos] >> offset) & ((1 << bits) - 1)), nil } var result uint64 @@ -69,11 +83,11 @@ func (b *BitStreamReader) ReadBool() (bool, error) { } func (b *BitStreamReader) Ok() bool { - return b.remaingBits >= 0 + return b.remainingBits >= 0 } func (b *BitStreamReader) Invalidate() { - b.remaingBits = -1 + b.remainingBits = -1 } // Reads value in range [0, `num_values` - 1]. @@ -107,8 +121,8 @@ func (b *BitStreamReader) ReadNonSymmetric(numValues uint32) (uint32, error) { return uint32((val << 1) + bit - uint64(numMinBitsValues)), nil } -func (b *BitStreamReader) ReadedBytes() int { - if b.remaingBits%8 > 0 { +func (b *BitStreamReader) BytesRead() int { + if b.remainingBits%8 > 0 { return b.pos + 1 } return b.pos diff --git a/pkg/sfu/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/dependencydescriptor/bitstreamwriter.go index 671fcdc50..1e5ffacbe 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamwriter.go +++ b/pkg/sfu/dependencydescriptor/bitstreamwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( @@ -27,7 +41,7 @@ func (w *BitStreamWriter) WriteBits(val uint64, bitCount int) error { totalBits := bitCount // push bits to the highest bits of uint64 - val <<= (64 - bitCount) + val <<= 64 - bitCount buf := w.buf[w.pos:] diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go index 8316b6b01..b573d7d60 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( @@ -9,23 +23,20 @@ import ( // DependencyDescriptorExtension is a extension payload format in // https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +func formatBitmask(b *uint32) string { + if b == nil { + return "-" + } + return strconv.FormatInt(int64(*b), 2) +} + +// ------------------------------------------------------------------------------ + type DependencyDescriptorExtension struct { Descriptor *DependencyDescriptor Structure *FrameDependencyStructure } -func (d *DependencyDescriptor) MarshalSize() (int, error) { - return d.MarshalSizeWithActiveChains(^uint32(0)) -} - -func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) { - writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d) - if err != nil { - return 0, err - } - return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil -} - func (d *DependencyDescriptorExtension) Marshal() ([]byte, error) { return d.MarshalWithActiveChains(^uint32(0)) } @@ -48,6 +59,8 @@ func (d *DependencyDescriptorExtension) Unmarshal(buf []byte) (int, error) { return reader.Parse() } +// ------------------------------------------------------------------------------ + const ( MaxSpatialIds = 4 MaxTemporalIds = 8 @@ -59,9 +72,11 @@ const ( ExtensionUrl = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" ) +// ------------------------------------------------------------------------------ + type DependencyDescriptor struct { - FirstPacketInFrame bool // = true; - LastPacketInFrame bool // = true; + FirstPacketInFrame bool + LastPacketInFrame bool FrameNumber uint16 FrameDependencies *FrameDependencyTemplate Resolution *RenderResolution @@ -69,11 +84,16 @@ type DependencyDescriptor struct { AttachedStructure *FrameDependencyStructure } -func formatBitmask(b *uint32) string { - if b == nil { - return "-" +func (d *DependencyDescriptor) MarshalSize() (int, error) { + return d.MarshalSizeWithActiveChains(^uint32(0)) +} + +func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) { + writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d) + if err != nil { + return 0, err } - return strconv.FormatInt(int64(*b), 2) + return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil } func (d *DependencyDescriptor) String() string { @@ -81,21 +101,23 @@ func (d *DependencyDescriptor) String() string { d.FirstPacketInFrame, d.LastPacketInFrame, d.FrameNumber, *d.FrameDependencies, *d.Resolution, formatBitmask(d.ActiveDecodeTargetsBitmask), d.AttachedStructure) } +// ------------------------------------------------------------------------------ + // Relationship of a frame to a Decode target. type DecodeTargetIndication int const ( - DecodeTargetNotPresent DecodeTargetIndication = iota // DecodeTargetInfo symbol '-' - DecodeTargetDiscadable // DecodeTargetInfo symbol 'D' - DecodeTargetSwitch // DecodeTargetInfo symbol 'S' - DecodeTargetRequired // DecodeTargetInfo symbol 'R' + DecodeTargetNotPresent DecodeTargetIndication = iota // DecodeTargetInfo symbol '-' + DecodeTargetDiscardable // DecodeTargetInfo symbol 'D' + DecodeTargetSwitch // DecodeTargetInfo symbol 'S' + DecodeTargetRequired // DecodeTargetInfo symbol 'R' ) func (i DecodeTargetIndication) String() string { switch i { case DecodeTargetNotPresent: return "-" - case DecodeTargetDiscadable: + case DecodeTargetDiscardable: return "D" case DecodeTargetSwitch: return "S" @@ -106,6 +128,8 @@ func (i DecodeTargetIndication) String() string { } } +// ------------------------------------------------------------------------------ + type FrameDependencyTemplate struct { SpatialId int TemporalId int @@ -132,6 +156,8 @@ func (t *FrameDependencyTemplate) Clone() *FrameDependencyTemplate { return t2 } +// ------------------------------------------------------------------------------ + type FrameDependencyStructure struct { StructureId int NumDecodeTargets int @@ -156,7 +182,11 @@ func (f *FrameDependencyStructure) String() string { return str } +// ------------------------------------------------------------------------------ + type RenderResolution struct { Width int Height int } + +// ------------------------------------------------------------------------------ diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go index 25cb1b3bc..95c3aba35 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( @@ -35,7 +49,7 @@ func TestDependencyDescriptorUnmarshal(t *testing.T) { } var ddVal DependencyDescriptor - var d DependencyDescriptorExtension = DependencyDescriptorExtension{ + var d = DependencyDescriptorExtension{ Structure: structure, Descriptor: &ddVal, } diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go index fdfff8f89..04ae1ce7c 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go @@ -1,6 +1,22 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor -import "errors" +import ( + "errors" +) type DependencyDescriptorReader struct { // Output. @@ -59,7 +75,7 @@ func (r *DependencyDescriptorReader) Parse() (int, error) { if err != nil { return 0, err } - return r.buffer.ReadedBytes(), nil + return r.buffer.BytesRead(), nil } func (r *DependencyDescriptorReader) readMandatoryFields() error { @@ -89,7 +105,6 @@ func (r *DependencyDescriptorReader) readMandatoryFields() error { } func (r *DependencyDescriptorReader) readExtendedFields() error { - templateDependencyStructurePresentFlag, err := r.buffer.ReadBool() if err != nil { return err @@ -384,7 +399,6 @@ func (r *DependencyDescriptorReader) readFrameDtis() error { } func (r *DependencyDescriptorReader) readFrameFdiffs() error { - r.descriptor.FrameDependencies.FrameDiffs = r.descriptor.FrameDependencies.FrameDiffs[:0] for { nexFdiffSize, err := r.buffer.ReadBits(2) diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go index 0f6972898..37ce7bcf8 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( @@ -47,7 +61,7 @@ func (w *DependencyDescriptorWriter) Write() error { return err } - if w.hasExtenedFields() { + if w.hasExtendedFields() { if err := w.writeExtendedFields(); err != nil { return err } @@ -57,15 +71,15 @@ func (w *DependencyDescriptorWriter) Write() error { } } - remaingBits := w.writer.RemainingBits() + remainingBits := w.writer.RemainingBits() // Zero remaining memory to avoid leaving it uninitialized. - if remaingBits%64 != 0 { - if err := w.writeBits(0, remaingBits%64); err != nil { + if remainingBits%64 != 0 { + if err := w.writeBits(0, remainingBits%64); err != nil { return err } } - for i := 0; i < remaingBits/64; i++ { + for i := 0; i < remainingBits/64; i++ { if err := w.writeBits(0, 64); err != nil { return err } @@ -101,10 +115,10 @@ func (w *DependencyDescriptorWriter) findBestTemplate() error { } // Search if there any better template that have small extra size. - w.bestTemplate = w.caculateMatch(firstSameLayerIdx, firstSameLayer) + w.bestTemplate = w.calculateMatch(firstSameLayerIdx, firstSameLayer) for i := firstSameLayerIdx + 1; i <= lastSameLayerIdx; i++ { t := w.structure.Templates[i] - match := w.caculateMatch(i, t) + match := w.calculateMatch(i, t) if match.ExtraSizeBits < w.bestTemplate.ExtraSizeBits { w.bestTemplate = match } @@ -127,7 +141,7 @@ func (w *DependencyDescriptorWriter) findBestTemplate() error { // return true // } -func (w *DependencyDescriptorWriter) caculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch { +func (w *DependencyDescriptorWriter) calculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch { var result TemplateMatch result.TemplateIdx = idx result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !reflect.DeepEqual(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs) @@ -191,13 +205,13 @@ func (w *DependencyDescriptorWriter) writeBool(val bool) error { } func (w *DependencyDescriptorWriter) writeBits(val uint64, bitCount int) error { - if err := w.writer.WriteBits(uint64(val), bitCount); err != nil { + if err := w.writer.WriteBits(val, bitCount); err != nil { return err } return nil } -func (w *DependencyDescriptorWriter) hasExtenedFields() bool { +func (w *DependencyDescriptorWriter) hasExtendedFields() bool { return w.bestTemplate.ExtraSizeBits > 0 || w.descriptor.AttachedStructure != nil || w.descriptor.ActiveDecodeTargetsBitmask != nil } @@ -448,7 +462,7 @@ const mandatoryFieldSize = 1 + 1 + 6 + 16 func (w *DependencyDescriptorWriter) ValueSizeBits() int { valueSizeBits := mandatoryFieldSize + w.bestTemplate.ExtraSizeBits - if w.hasExtenedFields() { + if w.hasExtendedFields() { valueSizeBits += 5 if w.descriptor.AttachedStructure != nil { valueSizeBits += w.structureSizeBits() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 034f7562d..975cdda74 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -22,6 +36,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/pacer" ) // TrackSender defines an interface send media to remote peer @@ -42,6 +57,8 @@ type TrackSender interface { HandleTrackFrameRateReport(payloadType webrtc.PayloadType, fps [][]float32) error } +// ------------------------------------------------------------------- + const ( RTPPaddingMaxPayloadSize = 255 RTPPaddingEstimatedHeaderSize = 20 @@ -60,17 +77,15 @@ const ( maxPaddingOnMuteDuration = 5 * time.Second ) +// ------------------------------------------------------------------- + var ( ErrUnknownKind = errors.New("unknown kind of codec") ErrOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") ErrPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") ErrDuplicatePacket = errors.New("duplicate packet") ErrPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") - ErrNotVP8 = errors.New("not VP8") - ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache") - ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer") ErrDownTrackAlreadyBound = errors.New("already bound") - ErrDownTrackClosed = errors.New("downtrack closed") ) var ( @@ -112,18 +127,25 @@ var ( // ------------------------------------------------------------------- type DownTrackState struct { - RTPStats *buffer.RTPStats - DeltaStatsSnapshotId uint32 - ForwarderState ForwarderState + RTPStats *buffer.RTPStats + DeltaStatsSnapshotId uint32 + DeltaStatsOverriddenSnapshotId uint32 + ForwarderState ForwarderState } func (d DownTrackState) String() string { - return fmt.Sprintf("DownTrackState{rtpStats: %s, delta: %d, forwarder: %s}", - d.RTPStats.ToString(), d.DeltaStatsSnapshotId, d.ForwarderState.String()) + return fmt.Sprintf("DownTrackState{rtpStats: %s, delta: %d, deltaOverridden: %d, forwarder: %s}", + d.RTPStats.ToString(), d.DeltaStatsSnapshotId, d.DeltaStatsOverriddenSnapshotId, d.ForwarderState.String()) } // ------------------------------------------------------------------- +type NackInfo struct { + Timestamp uint32 + SequenceNumber uint16 + Attempts uint8 +} + type DownTrackStreamAllocatorListener interface { // RTCP received OnREMB(dt *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) @@ -135,20 +157,29 @@ type DownTrackStreamAllocatorListener interface { // video layer bitrate availability changed OnBitrateAvailabilityChanged(dt *DownTrack) - // max published video layer changed - OnMaxPublishedLayerChanged(dt *DownTrack) + // max published spatial layer changed + OnMaxPublishedSpatialChanged(dt *DownTrack) + + // max published temporal layer changed + OnMaxPublishedTemporalChanged(dt *DownTrack) // subscription changed - mute/unmute OnSubscriptionChanged(dt *DownTrack) // subscribed max video layer changed - OnSubscribedLayersChanged(dt *DownTrack, layers VideoLayers) + OnSubscribedLayerChanged(dt *DownTrack, layers buffer.VideoLayer) - // target video layer reaached - OnTargetLayerReached(dt *DownTrack) + // stream resumed + OnResume(dt *DownTrack) // packet(s) sent OnPacketsSent(dt *DownTrack, size int) + + // NACKs received + OnNACK(dt *DownTrack, nackInfos []NackInfo) + + // RTCP Receiver Report received + OnRTCPReceiverReport(dt *DownTrack, rr rtcp.ReceptionReport) } type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) @@ -176,23 +207,22 @@ type DownTrack struct { forwarder *Forwarder - upstreamCodecs []webrtc.RTPCodecParameters - codec webrtc.RTPCodecCapability - rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter - absSendTimeID int - dependencyDescriptorID int - receiver TrackReceiver - transceiver *webrtc.RTPTransceiver - writeStream webrtc.TrackLocalWriter - rtcpReader *buffer.RTCPReader - onCloseHandler func(willBeResumed bool) - onBinding func() + upstreamCodecs []webrtc.RTPCodecParameters + codec webrtc.RTPCodecCapability + absSendTimeExtID int + transportWideExtID int + dependencyDescriptorExtID int + receiver TrackReceiver + transceiver *webrtc.RTPTransceiver + writeStream webrtc.TrackLocalWriter + rtcpReader *buffer.RTCPReader listenerLock sync.RWMutex receiverReportListeners []ReceiverReportListener - bindLock sync.Mutex - bound atomic.Bool + bindLock sync.Mutex + bound atomic.Bool + onBinding func(error) isClosed atomic.Bool connected atomic.Bool @@ -200,19 +230,15 @@ type DownTrack struct { rtpStats *buffer.RTPStats - statsLock sync.RWMutex - totalRepeatedNACKs uint32 + totalRepeatedNACKs atomic.Uint32 keyFrameRequestGeneration atomic.Uint32 blankFramesGeneration atomic.Uint32 - connectionStats *connectionquality.ConnectionStats - deltaStatsSnapshotId uint32 - - // Debug info - pktsDropped atomic.Uint32 - writeIOErrors atomic.Uint32 + connectionStats *connectionquality.ConnectionStats + deltaStatsSnapshotId uint32 + deltaStatsOverriddenSnapshotId uint32 isNACKThrottled atomic.Bool @@ -222,15 +248,20 @@ type DownTrack struct { streamAllocatorListener DownTrackStreamAllocatorListener streamAllocatorReportGeneration int streamAllocatorBytesCounter atomic.Uint32 + bytesSent atomic.Uint32 + bytesRetransmitted atomic.Uint32 - // update stats - onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) + pacer pacer.Pacer - // when max subscribed layer changes + maxLayerNotifierCh chan struct{} + + trailer []byte + + cbMu sync.RWMutex + onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) onMaxSubscribedLayerChanged func(dt *DownTrack, layer int32) - - // update rtt - onRttUpdate func(dt *DownTrack, rtt uint32) + onRttUpdate func(dt *DownTrack, rtt uint32) + onCloseHandler func(willBeResumed bool) } // NewDownTrack returns a DownTrack. @@ -240,6 +271,8 @@ func NewDownTrack( bf *buffer.Factory, subID livekit.ParticipantID, mt int, + pacer pacer.Pacer, + trailer []byte, logger logger.Logger, ) (*DownTrack, error) { var kind webrtc.RTPCodecType @@ -253,42 +286,57 @@ func NewDownTrack( } d := &DownTrack{ - logger: logger, - id: r.TrackID(), - subscriberID: subID, - maxTrack: mt, - streamID: r.StreamID(), - bufferFactory: bf, - receiver: r, - upstreamCodecs: codecs, - kind: kind, - codec: codecs[0].RTPCodecCapability, + logger: logger, + id: r.TrackID(), + subscriberID: subID, + maxTrack: mt, + streamID: r.StreamID(), + bufferFactory: bf, + receiver: r, + upstreamCodecs: codecs, + kind: kind, + codec: codecs[0].RTPCodecCapability, + pacer: pacer, + trailer: trailer, + maxLayerNotifierCh: make(chan struct{}, 20), } - d.forwarder = NewForwarder(d.kind, d.logger, d.receiver.GetReferenceLayerRTPTimestamp) - d.forwarder.OnParkedLayersExpired(func() { + d.forwarder = NewForwarder( + d.kind, + d.logger, + d.receiver.GetReferenceLayerRTPTimestamp, + d.getExpectedRTPTimestamp, + ) + d.forwarder.OnParkedLayerExpired(func() { if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) } }) - d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ - MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change - IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), - GetDeltaStats: d.getDeltaStats, - Logger: d.logger, - }) - d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { - if d.onStatsUpdate != nil { - d.onStatsUpdate(d, stat) - } - }) - d.rtpStats = buffer.NewRTPStats(buffer.RTPStatsParams{ ClockRate: d.codec.ClockRate, IsReceiverReportDriven: true, Logger: d.logger, }) d.deltaStatsSnapshotId = d.rtpStats.NewSnapshotId() + d.deltaStatsOverriddenSnapshotId = d.rtpStats.NewSnapshotId() + + d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ + MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change + IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), + GetDeltaStats: d.getDeltaStats, + GetDeltaStatsOverridden: d.getDeltaStatsOverridden, + GetLastReceiverReportTime: func() time.Time { return d.rtpStats.LastReceiverReport() }, + Logger: d.logger.WithValues("direction", "down"), + }) + d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { + if onStatsUpdate := d.getOnStatsUpdate(); onStatsUpdate != nil { + onStatsUpdate(d, stat) + } + }) + + if d.kind == webrtc.RTPCodecTypeVideo { + go d.maxLayerNotifierWorker() + } return d, nil } @@ -312,8 +360,14 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } if codec.MimeType == "" { + err := webrtc.ErrUnsupportedCodec + onBinding := d.onBinding d.bindLock.Unlock() - return webrtc.RTPCodecParameters{}, webrtc.ErrUnsupportedCodec + d.logger.Infow("bind error for unsupported codec", "codecs", d.upstreamCodecs, "remoteParameters", t.CodecParameters()) + if onBinding != nil { + onBinding(err) + } + return webrtc.RTPCodecParameters{}, err } // if a downtrack is closed before bind, it already unsubscribed from client, don't do subsequent operation and return here. @@ -342,13 +396,14 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } d.codec = codec.RTPCodecCapability - d.forwarder.DetermineCodec(d.codec) if d.onBinding != nil { - d.onBinding() + d.onBinding(nil) } d.bound.Store(true) d.bindLock.Unlock() + d.forwarder.DetermineCodec(d.codec, d.receiver.HeaderExtensions()) + d.logger.Debugw("downtrack bound") d.onBindAndConnected() @@ -367,7 +422,7 @@ func (d *DownTrack) TrackInfoAvailable() { if ti == nil { return } - d.connectionStats.Start(ti, time.Now()) + d.connectionStats.Start(ti) } func (d *DownTrack) SetStreamAllocatorListener(listener DownTrackStreamAllocatorListener) { @@ -447,13 +502,14 @@ func (d *DownTrack) SubscriberID() livekit.ParticipantID { return d.subscriberID // Sets RTP header extensions for this track func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { - d.rtpHeaderExtensions = rtpHeaderExtensions for _, ext := range rtpHeaderExtensions { switch ext.URI { case sdp.ABSSendTimeURI: - d.absSendTimeID = ext.ID + d.absSendTimeExtID = ext.ID + case sdp.TransportCCURI: + d.transportWideExtID = ext.ID case dd.ExtensionUrl: - d.dependencyDescriptorID = ext.ID + d.dependencyDescriptorExtID = ext.ID } } } @@ -496,7 +552,7 @@ func (d *DownTrack) maybeStartKeyFrameRequester() { // d.stopKeyFrameRequester() - // TODO : for svc, don't need pli/lrr when layer comes down + // SVC-TODO : don't need pli/lrr when layer comes down locked, layer := d.forwarder.CheckSync() if !locked { go d.keyFrameRequester(d.keyFrameRequestGeneration.Load(), layer) @@ -508,9 +564,10 @@ func (d *DownTrack) stopKeyFrameRequester() { } func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { - if d.IsClosed() || layer == InvalidLayerSpatial { + if d.IsClosed() || layer == buffer.InvalidLayerSpatial { return } + interval := 2 * d.rtpStats.GetRtt() if interval < keyFrameIntervalMin { interval = keyFrameIntervalMin @@ -520,7 +577,13 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { } ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) defer ticker.Stop() + for { + locked, _ := d.forwarder.CheckSync() + if locked { + return + } + if d.connected.Load() { d.logger.Debugw("sending PLI for layer lock", "generation", generation, "layer", layer) d.receiver.SendPLI(layer, false) @@ -535,108 +598,104 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { } } +func (d *DownTrack) postMaxLayerNotifierEvent() { + if d.IsClosed() { + return + } + + select { + case d.maxLayerNotifierCh <- struct{}{}: + default: + d.logger.Warnw("max layer notifier event queue full", nil) + } +} + +func (d *DownTrack) maxLayerNotifierWorker() { + more := true + for more { + _, more = <-d.maxLayerNotifierCh + + maxLayerSpatial := buffer.InvalidLayerSpatial + if more { + maxLayerSpatial = d.forwarder.GetMaxSubscribedSpatial() + } + if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { + d.logger.Infow("max subscribed layer changed", "maxLayerSpatial", maxLayerSpatial) + onMaxSubscribedLayerChanged(d, maxLayerSpatial) + } + } +} + // WriteRTP writes an RTP Packet to the DownTrack func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { - var pool *[]byte - defer func() { - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } - }() - if !d.bound.Load() || !d.connected.Load() { return nil } tp, err := d.forwarder.GetTranslationParams(extPkt, layer) if tp.shouldDrop { - if tp.isDroppingRelevant { - d.pktsDropped.Inc() - } if err != nil { d.logger.Errorw("write rtp packet failed", err) } return err } - payload := extPkt.Packet.Payload - if tp.vp8 != nil { - incomingVP8, _ := extPkt.Payload.(buffer.VP8) - pool = PacketFactory.Get().(*[]byte) - payload, err = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.vp8.Header, pool) - if err != nil { - d.pktsDropped.Inc() - d.logger.Errorw("write rtp packet failed", err) - return err + var payload []byte + pool := PacketFactory.Get().(*[]byte) + if len(tp.codecBytes) != 0 { + incomingVP8, ok := extPkt.Payload.(buffer.VP8) + if ok { + payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) } } - - var meta *packetMeta - if d.sequencer != nil { - meta = d.sequencer.push(extPkt.Packet.SequenceNumber, tp.rtp.sequenceNumber, tp.rtp.timestamp, int8(layer)) - if meta != nil && tp.vp8 != nil { - meta.packVP8(tp.vp8.Header) - } + if payload == nil { + payload = (*pool)[:len(extPkt.Packet.Payload)] + copy(payload, extPkt.Packet.Payload) } hdr, err := d.getTranslatedRTPHeader(extPkt, tp) if err != nil { - d.pktsDropped.Inc() d.logger.Errorw("write rtp packet failed", err) - return err - } - - if meta != nil && d.dependencyDescriptorID != 0 { - meta.ddBytes = hdr.GetExtension(uint8(d.dependencyDescriptorID)) - } - - _, err = d.writeStream.WriteRTP(hdr, payload) - if err != nil { - d.pktsDropped.Inc() - if errors.Is(err, io.ErrClosedPipe) { - writeIOErrors := d.writeIOErrors.Inc() - if (writeIOErrors % 100) == 1 { - d.logger.Errorw("write rtp packet failed", err, "count", writeIOErrors) - } - } else { - d.logger.Errorw("write rtp packet failed", err) + if pool != nil { + PacketFactory.Put(pool) } return err } - d.streamAllocatorBytesCounter.Add(uint32(hdr.MarshalSize() + len(payload))) - - if tp.isSwitchingToMaxLayer && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, layer) + if d.sequencer != nil { + d.sequencer.push( + extPkt.Packet.SequenceNumber, + tp.rtp.sequenceNumber, + tp.rtp.timestamp, + hdr.Marker, + int8(layer), + tp.codecBytes, + tp.ddBytes, + ) } - if extPkt.KeyFrame || tp.isSwitchingToTargetLayer { - d.isNACKThrottled.Store(false) - if extPkt.KeyFrame { - d.rtpStats.UpdateKeyFrame(1) - d.logger.Debugw("forwarding key frame", "layer", layer) - } - - locked, _ := d.forwarder.CheckSync() - if locked { - d.stopKeyFrameRequester() - } - - if tp.isSwitchingToTargetLayer { - if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnTargetLayerReached(d) - } - } - } - - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.pacer.Enqueue(pacer.Packet{ + Header: hdr, + Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}}, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + layer: layer, + arrival: extPkt.Arrival, + isKeyFrame: extPkt.KeyFrame, + tp: tp, + pool: pool, + }, + OnSent: d.packetSent, + }) return nil } // WritePaddingRTP tries to write as many padding only RTP packets as necessary // to satisfy given size to the DownTrack -func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { +func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMarker bool) int { if !d.rtpStats.IsActive() && !paddingOnMute { return 0 } @@ -668,7 +727,7 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { return 0 } - snts, err := d.forwarder.GetSnTsForPadding(num) + snts, err := d.forwarder.GetSnTsForPadding(num, forceMarker) if err != nil { return 0 } @@ -694,23 +753,23 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) - if err != nil { - return bytesSent - } - payload := make([]byte, RTPPaddingMaxPayloadSize) // last byte of padding has padding size including that byte payload[RTPPaddingMaxPayloadSize-1] = byte(RTPPaddingMaxPayloadSize) - _, err = d.writeStream.WriteRTP(&hdr, payload) - if err != nil { - return bytesSent - } - - if !paddingOnMute { - d.rtpStats.Update(&hdr, 0, len(payload), time.Now().UnixNano()) - } + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isPadding: true, + disableCounter: true, + disableRTPStats: paddingOnMute, + }, + OnSent: d.packetSent, + }) // // Register with sequencer with invalid layer so that NACKs for these can be filtered out. @@ -724,27 +783,28 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { bytesSent += hdr.MarshalSize() + len(payload) } + // STREAM_ALLOCATOR-TODO: change this to pull this counter from stream allocator so that counter can be update in pacer callback return bytesSent } // Mute enables or disables media forwarding - subscriber triggered func (d *DownTrack) Mute(muted bool) { - changed, maxLayers := d.forwarder.Mute(muted) - d.handleMute(muted, false, changed, maxLayers) + changed := d.forwarder.Mute(muted) + d.handleMute(muted, changed) } // PubMute enables or disables media forwarding - publisher side func (d *DownTrack) PubMute(pubMuted bool) { - changed, maxLayers := d.forwarder.PubMute(pubMuted) - d.handleMute(pubMuted, true, changed, maxLayers) + changed := d.forwarder.PubMute(pubMuted) + d.handleMute(pubMuted, changed) } -func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers VideoLayers) { +func (d *DownTrack) handleMute(muted bool, changed bool) { if !changed { return } - d.connectionStats.UpdateMute(d.forwarder.IsAnyMuted(), time.Now()) + d.connectionStats.UpdateMute(d.forwarder.IsAnyMuted()) // // Subscriber mute changes trigger a max layer notification. @@ -760,24 +820,13 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers V // 2. down track(s) notifying max layer // 3. out-of-band notification about max layer sent back to the publisher // 4. publisher starts layer(s) - // Ideally, on publisher mute, whatever layers were active reamin active and + // Ideally, on publisher mute, whatever layers were active remain active and // can be restarted by publisher immediately on unmute. // // Note that while publisher mute is active, subscriber changes can also happen // and that could turn on/off layers on publisher side. // - if !isPub && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - notifyLayer := InvalidLayerSpatial - if !muted { - // - // When unmuting, don't wait for layer lock as - // client might need to be notified to start layers - // before locking can happen in the forwarder. - // - notifyLayer = maxLayers.Spatial - } - d.onMaxSubscribedLayerChanged(d, notifyLayer) - } + d.postMaxLayerNotifierEvent() if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) @@ -808,7 +857,7 @@ func (d *DownTrack) Close() { d.CloseWithFlush(true) } -// Close track, flush used to indicate whether send blank frame to flush +// CloseWithFlush - flush used to indicate whether send blank frame to flush // decoder of client. // 1. When transceiver is reused by other participant's video track, // set flush=true to avoid previous video shows before new stream is displayed. @@ -846,13 +895,13 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.bound.Store(false) d.logger.Debugw("closing sender", "kind", d.kind) - d.receiver.DeleteDownTrack(d.subscriberID) + } + d.receiver.DeleteDownTrack(d.subscriberID) - if d.rtcpReader != nil { - d.logger.Debugw("downtrack close rtcp reader") - d.rtcpReader.Close() - d.rtcpReader.OnPacket(nil) - } + if d.rtcpReader != nil && flush { + d.logger.Debugw("downtrack close rtcp reader") + d.rtcpReader.Close() + d.rtcpReader.OnPacket(nil) } d.bindLock.Unlock() @@ -860,12 +909,10 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.rtpStats.Stop() d.logger.Infow("rtp stats", "direction", "downstream", "mime", d.mime, "ssrc", d.ssrc, "stats", d.rtpStats.ToString()) - if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, InvalidLayerSpatial) - } + close(d.maxLayerNotifierCh) - if d.onCloseHandler != nil { - d.onCloseHandler(!flush) + if onCloseHandler := d.getOnCloseHandler(); onCloseHandler != nil { + onCloseHandler(!flush) } d.stopKeyFrameRequester() @@ -873,53 +920,47 @@ func (d *DownTrack) CloseWithFlush(flush bool) { } func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { - changed, maxLayers, currentLayers := d.forwarder.SetMaxSpatialLayer(spatialLayer) + changed, maxLayer := d.forwarder.SetMaxSpatialLayer(spatialLayer) if !changed { return } - if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo && maxLayers.SpatialGreaterThanOrEqual(currentLayers) { - // - // Notify when new max is - // 1. Equal to current -> already locked to the new max - // 2. Greater than current -> two scenarios - // a. is higher than previous max -> client may need to start higher layer before forwarder can lock - // b. is lower than previous max -> client can stop higher layer(s) - // - d.onMaxSubscribedLayerChanged(d, maxLayers.Spatial) - } + d.postMaxLayerNotifierEvent() if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnSubscribedLayersChanged(d, maxLayers) + sal.OnSubscribedLayerChanged(d, maxLayer) } } func (d *DownTrack) SetMaxTemporalLayer(temporalLayer int32) { - changed, maxLayers, _ := d.forwarder.SetMaxTemporalLayer(temporalLayer) + changed, maxLayer := d.forwarder.SetMaxTemporalLayer(temporalLayer) if !changed { return } if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnSubscribedLayersChanged(d, maxLayers) + sal.OnSubscribedLayerChanged(d, maxLayer) } } -func (d *DownTrack) MaxLayers() VideoLayers { - return d.forwarder.MaxLayers() +func (d *DownTrack) MaxLayer() buffer.VideoLayer { + return d.forwarder.MaxLayer() } func (d *DownTrack) GetState() DownTrackState { - return DownTrackState{ - RTPStats: d.rtpStats, - DeltaStatsSnapshotId: d.deltaStatsSnapshotId, - ForwarderState: d.forwarder.GetState(), + dts := DownTrackState{ + RTPStats: d.rtpStats, + DeltaStatsSnapshotId: d.deltaStatsSnapshotId, + DeltaStatsOverriddenSnapshotId: d.deltaStatsOverriddenSnapshotId, + ForwarderState: d.forwarder.GetState(), } + return dts } func (d *DownTrack) SeedState(state DownTrackState) { d.rtpStats.Seed(state.RTPStats) d.deltaStatsSnapshotId = state.DeltaStatsSnapshotId + d.deltaStatsOverriddenSnapshotId = state.DeltaStatsOverriddenSnapshotId d.forwarder.SeedState(state.ForwarderState) } @@ -936,44 +977,61 @@ func (d *DownTrack) UpTrackBitrateAvailabilityChange() { } func (d *DownTrack) UpTrackMaxPublishedLayerChange(maxPublishedLayer int32) { - d.forwarder.SetMaxPublishedLayer(maxPublishedLayer) - - if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnMaxPublishedLayerChanged(d) + if d.forwarder.SetMaxPublishedLayer(maxPublishedLayer) { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnMaxPublishedSpatialChanged(d) + } } } func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32) { - d.forwarder.SetMaxTemporalLayerSeen(maxTemporalLayerSeen) + if d.forwarder.SetMaxTemporalLayerSeen(maxTemporalLayerSeen) { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnMaxPublishedTemporalChanged(d) + } + } } -func (d *DownTrack) maybeAddTransition(bitrate int64, distance float64) { +func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64, pauseReason VideoPauseReason) { if d.kind == webrtc.RTPCodecTypeAudio { return } - ti := d.receiver.TrackInfo() - if ti == nil { - return + if pauseReason == VideoPauseReasonBandwidth { + d.connectionStats.UpdatePause(true) + } else { + d.connectionStats.UpdatePause(false) + d.connectionStats.AddLayerTransition(distance) } - - if ti.Source == livekit.TrackSource_SCREEN_SHARE { - d.connectionStats.AddLayerTransition(distance, time.Now()) - } - - d.connectionStats.AddBitrateTransition(bitrate, time.Now()) } -func (d *DownTrack) UpTrackBitrateReport(_availableLayers []int32, bitrates Bitrates) { - d.maybeAddTransition(d.forwarder.GetOptimalBandwidthNeeded(bitrates), d.forwarder.DistanceToDesired(bitrates)) +func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) { + d.maybeAddTransition( + d.forwarder.GetOptimalBandwidthNeeded(bitrates), + d.forwarder.DistanceToDesired(availableLayers, bitrates), + d.forwarder.PauseReason(), + ) } // OnCloseHandler method to be called on remote tracked removed func (d *DownTrack) OnCloseHandler(fn func(willBeResumed bool)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onCloseHandler = fn } -func (d *DownTrack) OnBinding(fn func()) { +func (d *DownTrack) getOnCloseHandler() func(willBeResumed bool) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onCloseHandler +} + +func (d *DownTrack) OnBinding(fn func(error)) { + d.bindLock.Lock() + defer d.bindLock.Unlock() + d.onBinding = fn } @@ -985,17 +1043,47 @@ func (d *DownTrack) AddReceiverReportListener(listener ReceiverReportListener) { } func (d *DownTrack) OnStatsUpdate(fn func(dt *DownTrack, stat *livekit.AnalyticsStat)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onStatsUpdate = fn } +func (d *DownTrack) getOnStatsUpdate() func(dt *DownTrack, stat *livekit.AnalyticsStat) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onStatsUpdate +} + func (d *DownTrack) OnRttUpdate(fn func(dt *DownTrack, rtt uint32)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onRttUpdate = fn } +func (d *DownTrack) getOnRttUpdate() func(dt *DownTrack, rtt uint32) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onRttUpdate +} + func (d *DownTrack) OnMaxLayerChanged(fn func(dt *DownTrack, layer int32)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onMaxSubscribedLayerChanged = fn } +func (d *DownTrack) getOnMaxLayerChanged() func(dt *DownTrack, layer int32) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onMaxSubscribedLayerChanged +} + func (d *DownTrack) IsDeficient() bool { return d.forwarder.IsDeficient() } @@ -1006,24 +1094,28 @@ func (d *DownTrack) BandwidthRequested() int64 { } func (d *DownTrack) DistanceToDesired() float64 { - _, brs := d.receiver.GetLayeredBitrate() - return d.forwarder.DistanceToDesired(brs) + al, brs := d.receiver.GetLayeredBitrate() + return d.forwarder.DistanceToDesired(al, brs) } func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { al, brs := d.receiver.GetLayeredBitrate() allocation := d.forwarder.AllocateOptimal(al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } func (d *DownTrack) ProvisionalAllocatePrepare() { - _, brs := d.receiver.GetLayeredBitrate() - d.forwarder.ProvisionalAllocatePrepare(brs) + al, brs := d.receiver.GetLayeredBitrate() + d.forwarder.ProvisionalAllocatePrepare(al, brs) } -func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { +func (d *DownTrack) ProvisionalAllocateReset() { + d.forwarder.ProvisionalAllocateReset() +} + +func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { return d.forwarder.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) } @@ -1042,30 +1134,30 @@ func (d *DownTrack) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti func (d *DownTrack) ProvisionalAllocateCommit() VideoAllocation { allocation := d.forwarder.ProvisionalAllocateCommit() d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (VideoAllocation, bool) { - _, brs := d.receiver.GetLayeredBitrate() - allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, brs, allowOvershoot) + al, brs := d.receiver.GetLayeredBitrate() + allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation, available } func (d *DownTrack) GetNextHigherTransition(allowOvershoot bool) (VideoTransition, bool) { _, brs := d.receiver.GetLayeredBitrate() transition, available := d.forwarder.GetNextHigherTransition(brs, allowOvershoot) - d.logger.Debugw("stream: get next higher layer", "transition", transition, "available", available) + d.logger.Debugw("stream: get next higher layer", "transition", transition, "available", available, "bitrates", brs) return transition, available } func (d *DownTrack) Pause() VideoAllocation { - _, brs := d.receiver.GetLayeredBitrate() - allocation := d.forwarder.Pause(brs) + al, brs := d.receiver.GetLayeredBitrate() + allocation := d.forwarder.Pause(al, brs) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } @@ -1099,7 +1191,11 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetRTCPSenderReportDataExt(d.forwarder.GetReferenceLayerSpatial())) + clockLayer := d.forwarder.CurrentLayer().Spatial + if clockLayer == buffer.InvalidLayerSpatial { + clockLayer = d.forwarder.GetReferenceLayerSpatial() + } + return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetCalculatedClockRate(clockLayer)) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { @@ -1111,16 +1207,16 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return } - var writeBlankFrame func(*rtp.Header, bool) (int, error) + var getBlankFrame func(bool) ([]byte, error) switch d.mime { case "audio/opus": - writeBlankFrame = d.writeOpusBlankFrame + getBlankFrame = d.getOpusBlankFrame case "audio/red": - writeBlankFrame = d.writeOpusRedBlankFrame + getBlankFrame = d.getOpusRedBlankFrame case "video/vp8": - writeBlankFrame = d.writeVP8BlankFrame + getBlankFrame = d.getVP8BlankFrame case "video/h264": - writeBlankFrame = d.writeH264BlankFrame + getBlankFrame = d.getH264BlankFrame default: close(done) return @@ -1165,21 +1261,24 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) + payload, err := getBlankFrame(frameEndNeeded) if err != nil { - d.logger.Warnw("could not write header extension for blank frame", err) + d.logger.Warnw("could not get blank frame", err) close(done) return } - pktSize, err := writeBlankFrame(&hdr, frameEndNeeded) - if err != nil { - d.logger.Warnw("could not write blank frame", err) - close(done) - return - } - - d.streamAllocatorBytesCounter.Add(uint32(pktSize)) + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isBlankFrame: true, + }, + OnSent: d.packetSent, + }) // only the first frame will need frameEndNeeded to close out the // previous picture, rest are small key frames (for the video case) @@ -1194,24 +1293,30 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return done } -func (d *DownTrack) writeOpusBlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) maybeAddTrailer(buf []byte) int { + if len(buf) < len(d.trailer) { + d.logger.Warnw("trailer too big", nil, "bufLen", len(buf), "trailerLen", len(d.trailer)) + return 0 + } + + copy(buf, d.trailer) + return len(d.trailer) +} + +func (d *DownTrack) getOpusBlankFrame(_frameEndNeeded bool) ([]byte, error) { // silence frame // Used shortly after muting to ensure residual noise does not keep // generating noise at the decoder after the stream is stopped // i. e. comfort noise generation actually not producing something comfortable. - payload := make([]byte, len(OpusSilenceFrame)) + payload := make([]byte, 1000) copy(payload[0:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) - } - return hdr.MarshalSize() + len(payload), err + trailerLen := d.maybeAddTrailer(payload[len(OpusSilenceFrame):]) + return payload[:len(OpusSilenceFrame)+trailerLen], nil } -func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { // primary only silence frame for opus/red, there is no need to contain redundant silent frames - payload := make([]byte, len(OpusSilenceFrame)+1) + payload := make([]byte, 1000) // primary header // 0 1 2 3 4 5 6 7 @@ -1220,42 +1325,32 @@ func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) // +-+-+-+-+-+-+-+-+ payload[0] = opusPT copy(payload[1:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) - } - return hdr.MarshalSize() + len(payload), err + trailerLen := d.maybeAddTrailer(payload[1+len(OpusSilenceFrame):]) + return payload[:1+len(OpusSilenceFrame)+trailerLen], nil } -func (d *DownTrack) writeVP8BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { - blankVP8 := d.forwarder.GetPaddingVP8(frameEndNeeded) +func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { + blankVP8, err := d.forwarder.GetPadding(frameEndNeeded) + if err != nil { + return nil, err + } // 8x8 key frame // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) - payload := make([]byte, blankVP8.HeaderSize+len(VP8KeyFrame8x8)) - vp8Header := payload[:blankVP8.HeaderSize] - err := blankVP8.MarshalTo(vp8Header) - if err != nil { - return 0, err - } - - copy(payload[blankVP8.HeaderSize:], VP8KeyFrame8x8) - - _, err = d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) - } - return hdr.MarshalSize() + len(payload), err + payload := make([]byte, 1000) + copy(payload[:len(blankVP8)], blankVP8) + copy(payload[len(blankVP8):], VP8KeyFrame8x8) + trailerLen := d.maybeAddTrailer(payload[len(blankVP8)+len(VP8KeyFrame8x8):]) + return payload[:len(blankVP8)+len(VP8KeyFrame8x8)+trailerLen], nil } -func (d *DownTrack) writeH264BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { // TODO - Jie Zeng // now use STAP-A to compose sps, pps, idr together, most decoder support packetization-mode 1. // if client only support packetization-mode 0, use single nalu unit packet - buf := make([]byte, 1462) + buf := make([]byte, 1000) offset := 0 buf[0] = 0x18 // STAP-A offset++ @@ -1265,12 +1360,8 @@ func (d *DownTrack) writeH264BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (i copy(buf[offset:offset+len(payload)], payload) offset += len(payload) } - payload := buf[:offset] - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) - } - return hdr.MarshalSize() + offset, err + offset += d.maybeAddTrailer(buf[offset:]) + return buf[:offset], nil } func (d *DownTrack) handleRTCP(bytes []byte) { @@ -1283,10 +1374,10 @@ func (d *DownTrack) handleRTCP(bytes []byte) { pliOnce := true sendPliOnce := func() { if pliOnce { - targetLayers := d.forwarder.TargetLayers() - if targetLayers != InvalidLayers && !d.forwarder.IsAnyMuted() { - d.logger.Debugw("sending PLI RTCP", "layer", targetLayers.Spatial) - d.receiver.SendPLI(targetLayers.Spatial, false) + _, layer := d.forwarder.CheckSync() + if layer != buffer.InvalidLayerSpatial && !d.forwarder.IsAnyMuted() { + d.logger.Debugw("sending PLI RTCP", "layer", layer) + d.receiver.SendPLI(layer, false) d.isNACKThrottled.Store(true) d.rtpStats.UpdatePliTime() pliOnce = false @@ -1330,6 +1421,10 @@ func (d *DownTrack) handleRTCP(bytes []byte) { if isRttChanged { rttToReport = rtt } + + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnRTCPReceiverReport(d, r) + } } if len(rr.Reports) > 0 { d.listenerLock.RLock() @@ -1337,8 +1432,6 @@ func (d *DownTrack) handleRTCP(bytes []byte) { l(d, rr) } d.listenerLock.RUnlock() - - d.connectionStats.ReceiverReportReceived(time.Now()) } case *rtcp.TransportLayerNack: @@ -1368,8 +1461,8 @@ func (d *DownTrack) handleRTCP(bytes []byte) { d.sequencer.setRTT(rttToReport) } - if d.onRttUpdate != nil { - d.onRttUpdate(d, rttToReport) + if onRttUpdate := d.getOnRttUpdate(); onRttUpdate != nil { + onRttUpdate(d, rttToReport) } } } @@ -1401,31 +1494,24 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { return } - var pool *[]byte - defer func() { - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } - }() - src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) + nackInfos := make([]NackInfo, 0, len(filtered)) for _, meta := range d.sequencer.getPacketsMeta(filtered) { if disallowedLayers[meta.layer] { continue } nackAcks++ - - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } + nackInfos = append(nackInfos, NackInfo{ + SequenceNumber: meta.targetSeqNo, + Timestamp: meta.timestamp, + Attempts: meta.nacked, + }) pktBuff := *src n, err := d.receiver.ReadRTP(pktBuff, uint8(meta.layer), meta.sourceSeqNo) @@ -1443,89 +1529,65 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { var pkt rtp.Packet if err = pkt.Unmarshal(pktBuff[:n]); err != nil { + d.logger.Errorw("unmarshalling rtp packet failed in retransmit", err) continue } + pkt.Header.Marker = meta.marker pkt.Header.SequenceNumber = meta.targetSeqNo pkt.Header.Timestamp = meta.timestamp pkt.Header.SSRC = d.ssrc pkt.Header.PayloadType = d.payloadType - payload := pkt.Payload - if d.mime == "video/vp8" && len(pkt.Payload) > 0 { + var payload []byte + pool := PacketFactory.Get().(*[]byte) + if d.mime == "video/vp8" && len(pkt.Payload) > 0 && len(meta.codecBytes) != 0 { var incomingVP8 buffer.VP8 if err = incomingVP8.Unmarshal(pkt.Payload); err != nil { d.logger.Errorw("unmarshalling VP8 packet err", err) + PacketFactory.Put(pool) continue } - translatedVP8 := meta.unpackVP8() - pool = PacketFactory.Get().(*[]byte) - payload, err = d.translateVP8PacketTo(&pkt, &incomingVP8, translatedVP8, pool) - if err != nil { - d.logger.Errorw("translating VP8 packet err", err) - continue - } + payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) + } + if payload == nil { + payload = (*pool)[:len(pkt.Payload)] + copy(payload, pkt.Payload) } - var extraExtensions []extensionData - if len(meta.ddBytes) > 0 { - extraExtensions = append(extraExtensions, extensionData{ - id: uint8(d.dependencyDescriptorID), - payload: meta.ddBytes, - }) - } - err = d.writeRTPHeaderExtensions(&pkt.Header, extraExtensions...) - if err != nil { - d.logger.Errorw("writing rtp header extensions err", err) - continue - } - - if _, err = d.writeStream.WriteRTP(&pkt.Header, payload); err != nil { - d.logger.Errorw("writing rtx packet err", err) - } else { - d.streamAllocatorBytesCounter.Add(uint32(pkt.Header.MarshalSize() + len(payload))) - - d.rtpStats.Update(&pkt.Header, len(payload), 0, time.Now().UnixNano()) - } + d.pacer.Enqueue(pacer.Packet{ + Header: &pkt.Header, + Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: meta.ddBytes}}, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isRTX: true, + pool: pool, + }, + OnSent: d.packetSent, + }) } - d.statsLock.Lock() - d.totalRepeatedNACKs += numRepeatedNACKs - d.statsLock.Unlock() + d.totalRepeatedNACKs.Add(numRepeatedNACKs) d.rtpStats.UpdateNackProcessed(nackAcks, nackMisses, numRepeatedNACKs) -} - -type extensionData struct { - id uint8 - payload []byte -} - -// writes RTP header extensions of track -func (d *DownTrack) writeRTPHeaderExtensions(hdr *rtp.Header, extraExtensions ...extensionData) error { - // clear out extensions that may have been in the forwarded header - hdr.Extension = false - hdr.ExtensionProfile = 0 - hdr.Extensions = []rtp.Extension{} - - for _, ext := range extraExtensions { - hdr.SetExtension(uint8(ext.id), ext.payload) + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-START + // Need to check on the following + // - get all NACKs from sequencer even if SFU is not acknowledging, + // i. e. SFU does not acknowledge even same sequence number is NACKed too closely, + // but if sequencer return those also (even if not actually retransmitting), + // will that provide a signal? + // - get padding NACKs also? Maybe only look at them when their NACK count is 2? + // because padding runs in a separate path, it could get out of order with + // primary packets. So, it could be NACKed once. But, a repeat NACK means they + // were probably lost. But, as we do not retransmit padding packets, more than + // the second try does not provide any useful signal. + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-END + if sal := d.getStreamAllocatorListener(); sal != nil && len(nackInfos) != 0 { + sal.OnNACK(d, nackInfos) } - - if d.absSendTimeID != 0 { - sendTime := rtp.NewAbsSendTimeExtension(time.Now()) - b, err := sendTime.Marshal() - if err != nil { - return err - } - - err = hdr.SetExtension(uint8(d.absSendTimeID), b) - if err != nil { - return err - } - } - - return nil } func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (*rtp.Header, error) { @@ -1539,34 +1601,17 @@ func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *Transla hdr.Marker = tp.marker } - var extension []extensionData - if d.dependencyDescriptorID != 0 && tp.ddExtension != nil { - bytes, err := tp.ddExtension.Marshal() - if err != nil { - d.logger.Warnw("error marshalling dependency descriptor extension", err) - } else { - extension = append(extension, extensionData{ - id: uint8(d.dependencyDescriptorID), - payload: bytes, - }) - } - } - err := d.writeRTPHeaderExtensions(&hdr, extension...) - if err != nil { - return nil, err - } - return &hdr, nil } -func (d *DownTrack) translateVP8PacketTo(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 *buffer.VP8, outbuf *[]byte) ([]byte, error) { - buf := (*outbuf)[:len(pkt.Payload)+translatedVP8.HeaderSize-incomingVP8.HeaderSize] +func (d *DownTrack) translateVP8PacketTo(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 []byte, outbuf *[]byte) []byte { + buf := (*outbuf)[:len(pkt.Payload)+len(translatedVP8)-incomingVP8.HeaderSize] srcPayload := pkt.Payload[incomingVP8.HeaderSize:] - dstPayload := buf[translatedVP8.HeaderSize:] + dstPayload := buf[len(translatedVP8):] copy(dstPayload, srcPayload) - err := translatedVP8.MarshalTo(buf[:translatedVP8.HeaderSize]) - return buf, err + copy(buf[:len(translatedVP8)], translatedVP8) + return buf } func (d *DownTrack) DebugInfo() map[string]interface{} { @@ -1579,7 +1624,6 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { "TSOffset": rtpMungerParams.tsOffset, "LastMarker": rtpMungerParams.lastMarker, "LastPli": d.rtpStats.LastPli(), - "PacketsDropped": d.pktsDropped.Load(), } senderReport := d.CreateSenderReport() @@ -1598,11 +1642,15 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { "Bound": d.bound.Load(), "Muted": d.forwarder.IsMuted(), "PubMuted": d.forwarder.IsPubMuted(), - "CurrentSpatialLayer": d.forwarder.CurrentLayers().Spatial, + "CurrentSpatialLayer": d.forwarder.CurrentLayer().Spatial, "Stats": stats, } } +func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint64, error) { + return d.rtpStats.GetExpectedRTPTimestamp(at) +} + func (d *DownTrack) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) { return d.connectionStats.GetScoreAndQuality() } @@ -1611,39 +1659,45 @@ func (d *DownTrack) GetTrackStats() *livekit.RTPStats { return d.rtpStats.ToProto() } -func (d *DownTrack) getDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { - streamStats := make(map[uint32]*buffer.StreamStatsWithLayers, 1) - - deltaStats := d.rtpStats.DeltaInfo(d.deltaStatsSnapshotId) - if deltaStats == nil { +func (d *DownTrack) deltaStats(ds *buffer.RTPDeltaInfo) map[uint32]*buffer.StreamStatsWithLayers { + if ds == nil { return nil } + streamStats := make(map[uint32]*buffer.StreamStatsWithLayers, 1) streamStats[d.ssrc] = &buffer.StreamStatsWithLayers{ - RTPStats: deltaStats, + RTPStats: ds, Layers: map[int32]*buffer.RTPDeltaInfo{ - 0: deltaStats, + 0: ds, }, } return streamStats } +func (d *DownTrack) getDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { + return d.deltaStats(d.rtpStats.DeltaInfo(d.deltaStatsSnapshotId)) +} + +func (d *DownTrack) getDeltaStatsOverridden() map[uint32]*buffer.StreamStatsWithLayers { + return d.deltaStats(d.rtpStats.DeltaInfoOverridden(d.deltaStatsOverriddenSnapshotId)) +} + func (d *DownTrack) GetNackStats() (totalPackets uint32, totalRepeatedNACKs uint32) { totalPackets = d.rtpStats.GetTotalPacketsPrimary() - - d.statsLock.RLock() - totalRepeatedNACKs = d.totalRepeatedNACKs - d.statsLock.RUnlock() - + totalRepeatedNACKs = d.totalRepeatedNACKs.Load() return } +func (d *DownTrack) GetAndResetBytesSent() (uint32, uint32) { + return d.bytesSent.Swap(0), d.bytesRetransmitted.Swap(0) +} + func (d *DownTrack) onBindAndConnected() { if d.connected.Load() && d.bound.Load() && !d.bindAndConnectedOnce.Swap(true) { if d.kind == webrtc.RTPCodecTypeVideo { _, layer := d.forwarder.CheckSync() - if layer != InvalidLayerSpatial { + if layer != buffer.InvalidLayerSpatial { d.receiver.SendPLI(layer, true) } } @@ -1655,10 +1709,10 @@ func (d *DownTrack) onBindAndConnected() { } func (d *DownTrack) sendPaddingOnMute() { - d.logger.Debugw("sending padding on mute") // let uptrack have chance to send packet before we send padding time.Sleep(waitBeforeSendPaddingOnMute) + d.logger.Debugw("sending padding on mute") if d.kind == webrtc.RTPCodecTypeVideo { d.sendPaddingOnMuteForVideo() } else if d.mime == "audio/opus" { @@ -1673,7 +1727,7 @@ func (d *DownTrack) sendPaddingOnMuteForVideo() { if d.rtpStats.IsActive() || d.IsClosed() { return } - d.WritePaddingRTP(20, true) + d.WritePaddingRTP(20, true, true) time.Sleep(paddingOnMuteInterval) } } @@ -1703,20 +1757,24 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) + payload, err := d.getOpusBlankFrame(false) if err != nil { - d.logger.Warnw("could not write header extension for blank frame", err) + d.logger.Warnw("could not get blank frame", err) return } - payload := make([]byte, len(OpusSilenceFrame)) - copy(payload[0:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(&hdr, payload) - if err != nil { - d.logger.Warnw("could not write blank frame", err) - return - } + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isBlankFrame: true, + disableRTPStats: true, + }, + OnSent: d.packetSent, + }) } numFrames-- @@ -1731,3 +1789,81 @@ func (d *DownTrack) HandleRTCPSenderReportData(_payloadType webrtc.PayloadType, func (d *DownTrack) HandleTrackFrameRateReport(_payloadType webrtc.PayloadType, _fps [][]float32) error { return nil } + +type sendPacketMetadata struct { + layer int32 + arrival time.Time + isKeyFrame bool + isRTX bool + isPadding bool + isBlankFrame bool + disableCounter bool + disableRTPStats bool + tp *TranslationParams + pool *[]byte +} + +func (d *DownTrack) packetSent(md interface{}, hdr *rtp.Header, payloadSize int, sendTime time.Time, sendError error) { + spmd, ok := md.(sendPacketMetadata) + if !ok { + d.logger.Errorw("invalid send packet metadata", nil) + return + } + + if spmd.pool != nil { + PacketFactory.Put(spmd.pool) + } + + if sendError != nil { + return + } + + headerSize := hdr.MarshalSize() + if !spmd.disableCounter { + // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter + size := uint32(headerSize + payloadSize) + d.streamAllocatorBytesCounter.Add(size) + if spmd.isRTX { + d.bytesRetransmitted.Add(size) + } else { + d.bytesSent.Add(size) + } + } + + if !spmd.disableRTPStats { + packetTime := spmd.arrival + if packetTime.IsZero() { + packetTime = sendTime + } + if spmd.isPadding { + d.rtpStats.Update(hdr, 0, payloadSize, packetTime) + } else { + d.rtpStats.Update(hdr, payloadSize, 0, packetTime) + } + } + + if spmd.isKeyFrame { + d.isNACKThrottled.Store(false) + d.rtpStats.UpdateKeyFrame(1) + d.logger.Debugw( + "forwarded key frame", + "layer", spmd.layer, + "rtpsn", hdr.SequenceNumber, + "rtpts", hdr.Timestamp, + ) + } + + if spmd.tp != nil { + if spmd.tp.isSwitching { + d.postMaxLayerNotifierEvent() + } + + if spmd.tp.isResuming { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnResume(d) + } + } + } +} + +// ------------------------------------------------------------------------------- diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 04463eb8d..dd7ac59c6 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -34,7 +48,11 @@ func (d *DownTrackSpreader) GetDownTracks() []TrackSender { d.downTrackMu.RLock() defer d.downTrackMu.RUnlock() - return d.downTracksShadow + downTracks := make([]TrackSender, 0, len(d.downTracksShadow)) + for _, dt := range d.downTracksShadow { + downTracks = append(downTracks, dt) + } + return downTracks } func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender { diff --git a/pkg/sfu/errors.go b/pkg/sfu/errors.go index 22831db4c..10743808c 100644 --- a/pkg/sfu/errors.go +++ b/pkg/sfu/errors.go @@ -1 +1,15 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1f2277374..9839fd76c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1,26 +1,49 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( "fmt" "math" + "math/rand" "strings" "sync" "time" + "github.com/pion/rtp" "github.com/pion/webrtc/v3" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/codecmunger" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" ) // Forwarder const ( - FlagPauseOnDowngrade = true - FlagFilterRTX = true - TransitionCostSpatial = 10 - ParkedLayersWaitDuration = 2 * time.Second + FlagPauseOnDowngrade = true + FlagFilterRTX = true + TransitionCostSpatial = 10 + ParkedLayerWaitDuration = 2 * time.Second + + ResumeBehindThresholdSeconds = float64(0.1) // 100ms + LayerSwitchBehindThresholdSeconds = float64(0.05) // 50ms + SwitchAheadThresholdSeconds = float64(0.025) // 25ms ) // ------------------------------------------------------------------- @@ -55,116 +78,105 @@ func (v VideoPauseReason) String() string { // ------------------------------------------------------------------- type VideoAllocation struct { - pauseReason VideoPauseReason - isDeficient bool - bandwidthRequested int64 - bandwidthDelta int64 - bandwidthNeeded int64 - bitrates Bitrates - targetLayers VideoLayers - requestLayerSpatial int32 - maxLayers VideoLayers - distanceToDesired float64 + PauseReason VideoPauseReason + IsDeficient bool + BandwidthRequested int64 + BandwidthDelta int64 + BandwidthNeeded int64 + Bitrates Bitrates + TargetLayer buffer.VideoLayer + RequestLayerSpatial int32 + MaxLayer buffer.VideoLayer + DistanceToDesired float64 } func (v VideoAllocation) String() string { return fmt.Sprintf("VideoAllocation{pause: %s, def: %+v, bwr: %d, del: %d, bwn: %d, rates: %+v, target: %s, req: %d, max: %s, dist: %0.2f}", - v.pauseReason, - v.isDeficient, - v.bandwidthRequested, - v.bandwidthDelta, - v.bandwidthNeeded, - v.bitrates, - v.targetLayers, - v.requestLayerSpatial, - v.maxLayers, - v.distanceToDesired, + v.PauseReason, + v.IsDeficient, + v.BandwidthRequested, + v.BandwidthDelta, + v.BandwidthNeeded, + v.Bitrates, + v.TargetLayer, + v.RequestLayerSpatial, + v.MaxLayer, + v.DistanceToDesired, ) } var ( VideoAllocationDefault = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: InvalidLayers, + PauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.InvalidLayer, } ) // ------------------------------------------------------------------- type VideoAllocationProvisional struct { - muted bool - pubMuted bool - maxPublishedLayer int32 - maxTemporalLayerSeen int32 - bitrates Bitrates - maxLayers VideoLayers - currentLayers VideoLayers - parkedLayers VideoLayers - allocatedLayers VideoLayers + muted bool + pubMuted bool + maxSeenLayer buffer.VideoLayer + availableLayers []int32 + Bitrates Bitrates + maxLayer buffer.VideoLayer + currentLayer buffer.VideoLayer + parkedLayer buffer.VideoLayer + allocatedLayer buffer.VideoLayer } // ------------------------------------------------------------------- type VideoTransition struct { - from VideoLayers - to VideoLayers - bandwidthDelta int64 + From buffer.VideoLayer + To buffer.VideoLayer + BandwidthDelta int64 } func (v VideoTransition) String() string { - return fmt.Sprintf("VideoTransition{from: %s, to: %s, del: %d}", v.from, v.to, v.bandwidthDelta) + return fmt.Sprintf("VideoTransition{from: %s, to: %s, del: %d}", v.From, v.To, v.BandwidthDelta) } // ------------------------------------------------------------------- type TranslationParams struct { - shouldDrop bool - isDroppingRelevant bool - isSwitchingToMaxLayer bool - rtp *TranslationParamsRTP - vp8 *TranslationParamsVP8 - ddExtension *dd.DependencyDescriptorExtension - marker bool - - // indicates this frame has 'Switch' decode indication for target layer - // TODO : in theory, we need check frame chain is not broken for the target - // but we don't have frame queue now, so just use decode target indication - isSwitchingToTargetLayer bool + shouldDrop bool + isResuming bool + isSwitching bool + rtp *TranslationParamsRTP + codecBytes []byte + ddBytes []byte + marker bool } // ------------------------------------------------------------------- -type VideoLayers = buffer.VideoLayer - -const ( - InvalidLayerSpatial = buffer.InvalidLayerSpatial - InvalidLayerTemporal = buffer.InvalidLayerTemporal - - DefaultMaxLayerSpatial = buffer.DefaultMaxLayerSpatial - DefaultMaxLayerTemporal = buffer.DefaultMaxLayerTemporal -) - -var ( - InvalidLayers = buffer.InvalidLayers - - DefaultMaxLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial, - Temporal: DefaultMaxLayerTemporal, - } -) - -// ------------------------------------------------------------------- - type ForwarderState struct { - Started bool - RTP RTPMungerState - VP8 VP8MungerState + Started bool + PreStartTime time.Time + FirstTS uint32 + RefTSOffset uint32 + RTP RTPMungerState + Codec interface{} } func (f ForwarderState) String() string { - return fmt.Sprintf("ForwarderState{started: %v, rtp: %s, vp8: %s}", f.Started, f.RTP.String(), f.VP8.String()) + codecString := "" + switch codecState := f.Codec.(type) { + case codecmunger.VP8State: + codecString = codecState.String() + } + return fmt.Sprintf("ForwarderState{started: %v, preStartTime: %s, firstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", + f.Started, + f.PreStartTime.String(), + f.FirstTS, + f.RefTSOffset, + f.RTP.String(), + codecString, + ) } // ------------------------------------------------------------------- @@ -175,112 +187,100 @@ type Forwarder struct { kind webrtc.RTPCodecType logger logger.Logger getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) + getExpectedRTPTimestamp func(at time.Time) (uint64, error) muted bool pubMuted bool - maxPublishedLayer int32 - maxTemporalLayerSeen int32 - started bool + preStartTime time.Time + firstTS uint32 lastSSRC uint32 referenceLayerSpatial int32 + refTSOffset uint32 - maxLayers VideoLayers - currentLayers VideoLayers - targetLayers VideoLayers - requestLayerSpatial int32 - parkedLayers VideoLayers // layers that can resume without key frame - parkedLayersTimer *time.Timer + parkedLayerTimer *time.Timer provisional *VideoAllocationProvisional lastAllocation VideoAllocation rtpMunger *RTPMunger - vp8Munger *VP8Munger - isTemporalSupported bool + vls videolayerselector.VideoLayerSelector - ddLayerSelector *DDVideoLayerSelector + codecMunger codecmunger.CodecMunger - onParkedLayersExpired func() + onParkedLayerExpired func() } func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), + getExpectedRTPTimestamp func(at time.Time) (uint64, error), ) *Forwarder { f := &Forwarder{ kind: kind, logger: logger, getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, - - maxPublishedLayer: InvalidLayerSpatial, - maxTemporalLayerSeen: InvalidLayerTemporal, - - referenceLayerSpatial: InvalidLayerSpatial, - - // start off with nothing, let streamallocator/opportunistic forwarder set the target - currentLayers: InvalidLayers, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - parkedLayers: InvalidLayers, - - lastAllocation: VideoAllocationDefault, - - rtpMunger: NewRTPMunger(logger), + getExpectedRTPTimestamp: getExpectedRTPTimestamp, + referenceLayerSpatial: buffer.InvalidLayerSpatial, + lastAllocation: VideoAllocationDefault, + rtpMunger: NewRTPMunger(logger), + vls: videolayerselector.NewNull(logger), + codecMunger: codecmunger.NewNull(logger), } if f.kind == webrtc.RTPCodecTypeVideo { - f.maxLayers = VideoLayers{Spatial: InvalidLayerSpatial, Temporal: DefaultMaxLayerTemporal} - } else { - f.maxLayers = InvalidLayers + f.vls.SetMaxTemporal(buffer.DefaultMaxLayerTemporal) } - return f } -func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) { +func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) bool { f.lock.Lock() defer f.lock.Unlock() - if maxPublishedLayer <= f.maxPublishedLayer { - return + existingMaxSeen := f.vls.GetMaxSeen() + if maxPublishedLayer <= existingMaxSeen.Spatial { + return false } - f.maxPublishedLayer = maxPublishedLayer - f.logger.Infow("setting max published layer", "maxPublishedLayer", f.maxPublishedLayer) + f.vls.SetMaxSeenSpatial(maxPublishedLayer) + f.logger.Debugw("setting max published layer", "maxPublishedLayer", maxPublishedLayer) + return true } -func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { +func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool { f.lock.Lock() defer f.lock.Unlock() - if maxTemporalLayerSeen <= f.maxTemporalLayerSeen { - return + existingMaxSeen := f.vls.GetMaxSeen() + if maxTemporalLayerSeen <= existingMaxSeen.Temporal { + return false } - f.maxTemporalLayerSeen = maxTemporalLayerSeen - f.logger.Infow("setting max temporal layer seen", "maxTemporalLayerSeen", f.maxTemporalLayerSeen) + f.vls.SetMaxSeenTemporal(maxTemporalLayerSeen) + f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", maxTemporalLayerSeen) + return true } -func (f *Forwarder) OnParkedLayersExpired(fn func()) { +func (f *Forwarder) OnParkedLayerExpired(fn func()) { f.lock.Lock() defer f.lock.Unlock() - f.onParkedLayersExpired = fn + f.onParkedLayerExpired = fn } -func (f *Forwarder) getOnParkedLayersExpired() func() { +func (f *Forwarder) getOnParkedLayerExpired() func() { f.lock.RLock() defer f.lock.RUnlock() - return f.onParkedLayersExpired + return f.onParkedLayerExpired } -func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { +func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) { f.lock.Lock() defer f.lock.Unlock() @@ -291,12 +291,49 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { switch strings.ToLower(codec.MimeType) { case "video/vp8": - f.isTemporalSupported = true - f.vp8Munger = NewVP8Munger(f.logger) + f.codecMunger = codecmunger.NewVP8FromNull(f.codecMunger, f.logger) + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromNull(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) + case "video/h264": + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromNull(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + case "video/vp9": + isDDAvailable := false + searchDone: + for _, ext := range extensions { + switch ext.URI { + case dd.ExtensionUrl: + isDDAvailable = true + break searchDone + } + } + if isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromNull(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } + } else { + if f.vls != nil { + f.vls = videolayerselector.NewVP9FromNull(f.vls) + } else { + f.vls = videolayerselector.NewVP9(f.logger) + } + } case "video/av1": - // TODO : we only enable dd layer selector for av1 now, at future we can - // enable it for vp8 too - f.ddLayerSelector = NewDDVideoLayerSelector(f.logger) + // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromNull(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } } } @@ -308,16 +345,14 @@ func (f *Forwarder) GetState() ForwarderState { return ForwarderState{} } - state := ForwarderState{ - Started: f.started, - RTP: f.rtpMunger.GetLast(), + return ForwarderState{ + Started: f.started, + PreStartTime: f.preStartTime, + FirstTS: f.firstTS, + RefTSOffset: f.refTSOffset, + RTP: f.rtpMunger.GetLast(), + Codec: f.codecMunger.GetState(), } - - if f.vp8Munger != nil { - state.VP8 = f.vp8Munger.GetLast() - } - - return state } func (f *Forwarder) SeedState(state ForwarderState) { @@ -329,19 +364,39 @@ func (f *Forwarder) SeedState(state ForwarderState) { defer f.lock.Unlock() f.rtpMunger.SeedLast(state.RTP) - if f.vp8Munger != nil { - f.vp8Munger.SeedLast(state.VP8) - } + f.codecMunger.SeedState(state.Codec) f.started = true + f.preStartTime = state.PreStartTime + f.firstTS = state.FirstTS + f.refTSOffset = state.RefTSOffset } -func (f *Forwarder) Mute(muted bool) (bool, VideoLayers) { +func (f *Forwarder) Mute(muted bool) bool { f.lock.Lock() defer f.lock.Unlock() if f.muted == muted { - return false, f.maxLayers + return false + } + + // Do not mute when paused due to bandwidth limitation. + // There are two issues + // 1. Muting means probing cannot happen on this track. + // 2. Muting also triggers notification to publisher about layers this forwarder needs. + // If this forwarder does not need any layer, publisher could turn off all layers. + // So, muting could lead to not being able to restart the track. + // To avoid that, ignore mute when paused due to bandwidth limitations. + // + // NOTE: The above scenario refers to mute getting triggered due + // to video stream visibility changes. When a stream is paused, it is possible + // that the receiver hides the video tile triggering subscription mute. + // The work around here to ignore mute does ignore an intentional mute. + // It could result in some bandwidth consumed for stream without visibility in + // the case of intentional mute. + if muted && f.isDeficientLocked() && f.lastAllocation.PauseReason == VideoPauseReasonBandwidth { + f.logger.Infow("ignoring forwarder mute, paused due to congestion") + return false } f.logger.Debugw("setting forwarder mute", "muted", muted) @@ -352,7 +407,7 @@ func (f *Forwarder) Mute(muted bool) (bool, VideoLayers) { f.resyncLocked() } - return true, f.maxLayers + return true } func (f *Forwarder) IsMuted() bool { @@ -362,12 +417,12 @@ func (f *Forwarder) IsMuted() bool { return f.muted } -func (f *Forwarder) PubMute(pubMuted bool) (bool, VideoLayers) { +func (f *Forwarder) PubMute(pubMuted bool) bool { f.lock.Lock() defer f.lock.Unlock() if f.pubMuted == pubMuted { - return false, f.maxLayers + return false } f.logger.Debugw("setting forwarder pub mute", "pubMuted", pubMuted) @@ -380,15 +435,16 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, VideoLayers) { f.resyncLocked() } } else { - // Do not resync on publisher mute as forwarding can continue on unmute using same layers. + // Do not resync on publisher mute as forwarding can continue on unmute using same layer. // On unmute, park current layers as streaming can continue without a key frame when publisher starts the stream. - if !pubMuted && f.targetLayers.IsValid() && f.currentLayers.Spatial == f.targetLayers.Spatial { - f.setupParkedLayers(f.targetLayers) - f.currentLayers = InvalidLayers + targetLayer := f.vls.GetTarget() + if !pubMuted && targetLayer.IsValid() && f.vls.GetCurrent().Spatial == targetLayer.Spatial { + f.setupParkedLayer(targetLayer) + f.vls.SetCurrent(buffer.InvalidLayer) } } - return true, f.maxLayers + return true } func (f *Forwarder) IsPubMuted() bool { @@ -405,57 +461,86 @@ func (f *Forwarder) IsAnyMuted() bool { return f.muted || f.pubMuted } -func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, VideoLayers, VideoLayers) { +func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() - if f.kind == webrtc.RTPCodecTypeAudio || spatialLayer == f.maxLayers.Spatial { - return false, f.maxLayers, f.currentLayers + if f.kind == webrtc.RTPCodecTypeAudio { + return false, buffer.InvalidLayer } - f.logger.Infow("setting max spatial layer", "layer", spatialLayer) - f.maxLayers.Spatial = spatialLayer + existingMax := f.vls.GetMax() + if spatialLayer == existingMax.Spatial { + return false, existingMax + } - f.clearParkedLayers() + f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) + f.vls.SetMaxSpatial(spatialLayer) - return true, f.maxLayers, f.currentLayers + f.clearParkedLayer() + + return true, f.vls.GetMax() } -func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, VideoLayers, VideoLayers) { +func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() - if f.kind == webrtc.RTPCodecTypeAudio || temporalLayer == f.maxLayers.Temporal { - return false, f.maxLayers, f.currentLayers + if f.kind == webrtc.RTPCodecTypeAudio { + return false, buffer.InvalidLayer } - f.logger.Infow("setting max temporal layer", "layer", temporalLayer) - f.maxLayers.Temporal = temporalLayer + existingMax := f.vls.GetMax() + if temporalLayer == existingMax.Temporal { + return false, existingMax + } - f.clearParkedLayers() + f.logger.Debugw("setting max temporal layer", "layer", temporalLayer) + f.vls.SetMaxTemporal(temporalLayer) - return true, f.maxLayers, f.currentLayers + f.clearParkedLayer() + + return true, f.vls.GetMax() } -func (f *Forwarder) MaxLayers() VideoLayers { +func (f *Forwarder) MaxLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.maxLayers + return f.vls.GetMax() } -func (f *Forwarder) CurrentLayers() VideoLayers { +func (f *Forwarder) CurrentLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.currentLayers + return f.vls.GetCurrent() } -func (f *Forwarder) TargetLayers() VideoLayers { +func (f *Forwarder) TargetLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.targetLayers + return f.vls.GetTarget() +} + +func (f *Forwarder) GetMaxSubscribedSpatial() int32 { + f.lock.RLock() + defer f.lock.RUnlock() + + layer := buffer.InvalidLayerSpatial // covers muted case + if !f.muted { + layer = f.vls.GetMax().Spatial + + // If current is higher, mark the current layer as max subscribed layer + // to prevent the current layer from stopping before forwarder switches + // to the new and lower max layer, + if layer < f.vls.GetCurrent().Spatial { + layer = f.vls.GetCurrent().Spatial + } + } + + return layer } func (f *Forwarder) GetReferenceLayerSpatial() int32 { @@ -466,7 +551,7 @@ func (f *Forwarder) GetReferenceLayerSpatial() int32 { } func (f *Forwarder) isDeficientLocked() bool { - return f.lastAllocation.isDeficient + return f.lastAllocation.IsDeficient } func (f *Forwarder) IsDeficient() bool { @@ -476,39 +561,40 @@ func (f *Forwarder) IsDeficient() bool { return f.isDeficientLocked() } +func (f *Forwarder) PauseReason() VideoPauseReason { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.lastAllocation.PauseReason +} + func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() - if !f.targetLayers.IsValid() { - if f.targetLayers != InvalidLayers { - f.logger.Warnw( - "unexpected target layers", nil, - "target", f.targetLayers, - "current", f.currentLayers, - "parked", f.parkedLayers, - "max", f.maxLayers, - "lastAllocation", f.lastAllocation, - ) - } - return 0 - } - - return brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + return getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } -func (f *Forwarder) DistanceToDesired(brs Bitrates) float64 { +func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { f.lock.RLock() defer f.lock.RUnlock() - return getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, f.targetLayers, f.maxLayers) + return getDistanceToDesired( + f.muted, + f.pubMuted, + f.vls.GetMaxSeen(), + availableLayers, + brs, + f.vls.GetTarget(), + f.vls.GetMax(), + ) } func (f *Forwarder) GetOptimalBandwidthNeeded(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() - return getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + return getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.vls.GetMaxSeen().Spatial, brs, f.vls.GetMax()) } func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allowOvershoot bool) VideoAllocation { @@ -519,162 +605,207 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow return f.lastAllocation } + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + parkedLayer := f.vls.GetParked() + currentLayer := f.vls.GetCurrent() + requestSpatial := f.vls.GetRequestSpatial() alloc := VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bitrates: brs, - targetLayers: InvalidLayers, - requestLayerSpatial: f.requestLayerSpatial, - maxLayers: f.maxLayers, + PauseReason: VideoPauseReasonNone, + Bitrates: brs, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: requestSpatial, + MaxLayer: maxLayer, } - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) if optimalBandwidthNeeded == 0 { - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry + } + alloc.BandwidthNeeded = optimalBandwidthNeeded + + getMaxTemporal := func() int32 { + maxTemporal := maxLayer.Temporal + if maxSeenLayer.Temporal != buffer.InvalidLayerTemporal && maxSeenLayer.Temporal < maxTemporal { + maxTemporal = maxSeenLayer.Temporal + } + return maxTemporal } - alloc.bandwidthNeeded = optimalBandwidthNeeded opportunisticAlloc := func() { // opportunistically latch on to anything - maxSpatial := f.maxLayers.Spatial - if allowOvershoot && f.maxPublishedLayer > maxSpatial { - maxSpatial = f.maxPublishedLayer + maxSpatial := maxLayer.Spatial + if allowOvershoot && f.vls.IsOvershootOkay() && maxSeenLayer.Spatial > maxSpatial { + maxSpatial = maxSeenLayer.Spatial } - alloc.targetLayers = VideoLayers{ - Spatial: int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))), - Temporal: DefaultMaxLayerTemporal, + + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: int32(math.Min(float64(maxSeenLayer.Spatial), float64(maxSpatial))), + Temporal: getMaxTemporal(), } } switch { - case !f.maxLayers.IsValid() || f.maxPublishedLayer == InvalidLayerSpatial: - // nothing to do when max layers are not valid OR max publisher layer is invalid + case !maxLayer.IsValid() || maxSeenLayer.Spatial == buffer.InvalidLayerSpatial: + // nothing to do when max layers are not valid OR max published layer is invalid case f.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted // leave it at current layers for opportunistic resume - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayer = currentLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial - case f.parkedLayers.IsValid(): + case parkedLayer.IsValid(): // if parked on a layer, let it continue - alloc.targetLayers = f.parkedLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayer = parkedLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial - case len(availableLayers) == 0: - // feed may be dry - if f.currentLayers.IsValid() { - // let it continue at current layer if valid. - // Covers the cases of - // 1. mis-detection of layer stop - can continue streaming - // 2. current layer resuming - can latch on when it starts - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + default: + // lots of different events could end up here + // 1. Publisher side layer resuming/stopping + // 2. Bitrate becoming available + // 3. New max published spatial layer or max temporal layer seen + // 4. Subscriber layer changes + // + // to handle all of the above + // 1. Find highest that can be requested - takes into account available layers and overshoot. + // This should catch scenarios like layers resuming/stopping. + // 2. If current is a valid layer, check against currently available layers and continue at current + // if possible. Else, choose the highest available layer as the next target. + // 3. If current is not valid, set next target to be opportunistic. + maxLayerSpatialLimit := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) + highestAvailableLayer := buffer.InvalidLayerSpatial + requestLayerSpatial := buffer.InvalidLayerSpatial + for _, al := range availableLayers { + if al > requestLayerSpatial && al <= maxLayerSpatialLimit { + requestLayerSpatial = al + } + if al > highestAvailableLayer { + highestAvailableLayer = al + } + } + if requestLayerSpatial == buffer.InvalidLayerSpatial && highestAvailableLayer != buffer.InvalidLayerSpatial && allowOvershoot && f.vls.IsOvershootOkay() { + requestLayerSpatial = highestAvailableLayer + } + + if currentLayer.IsValid() { + if (requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial) || requestLayerSpatial == buffer.InvalidLayerSpatial { + // 1. current is locked to desired, stay there + // OR + // 2. feed may be dry, let it continue at current layer if valid. + // covers the cases of + // 1. mis-detection of layer stop - can continue streaming + // 2. current layer resuming - can latch on when it starts + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: currentLayer.Spatial, + Temporal: getMaxTemporal(), + } + } else { + // current layer has stopped, switch to highest available + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: requestLayerSpatial, + Temporal: getMaxTemporal(), + } + } + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { // opportunistically latch on to anything opportunisticAlloc() - alloc.requestLayerSpatial = int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) - } - - default: - isCurrentLayerAvailable := false - if f.currentLayers.IsValid() { - for _, l := range availableLayers { - if l == f.currentLayers.Spatial { - isCurrentLayerAvailable = true - break - } - } - } - - if !isCurrentLayerAvailable && f.currentLayers.IsValid() { - // current layer maybe stopped, move to highest available - for _, l := range availableLayers { - if l > alloc.targetLayers.Spatial { - alloc.targetLayers.Spatial = l - } - } - alloc.targetLayers.Temporal = DefaultMaxLayerTemporal - - alloc.requestLayerSpatial = alloc.targetLayers.Spatial - } else { - requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) - if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial { - // current is locked to desired, stay there - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = f.requestLayerSpatial + if requestLayerSpatial == buffer.InvalidLayerSpatial { + alloc.RequestLayerSpatial = maxLayerSpatialLimit } else { - // opportunistically latch on to anything - opportunisticAlloc() - alloc.requestLayerSpatial = requestLayerSpatial + alloc.RequestLayerSpatial = requestLayerSpatial } } } - if !alloc.targetLayers.IsValid() { - alloc.targetLayers = InvalidLayers - alloc.requestLayerSpatial = InvalidLayerSpatial + if !alloc.TargetLayer.IsValid() { + alloc.TargetLayer = buffer.InvalidLayer + alloc.RequestLayerSpatial = buffer.InvalidLayerSpatial } - if alloc.targetLayers.IsValid() { - alloc.bandwidthRequested = optimalBandwidthNeeded + if alloc.TargetLayer.IsValid() { + alloc.BandwidthRequested = optimalBandwidthNeeded } - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested - alloc.distanceToDesired = getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, alloc.targetLayers, f.maxLayers) + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) + alloc.DistanceToDesired = getDistanceToDesired( + f.muted, + f.pubMuted, + f.vls.GetMaxSeen(), + availableLayers, + brs, + alloc.TargetLayer, + f.vls.GetMax(), + ) return f.updateAllocation(alloc, "optimal") } -func (f *Forwarder) ProvisionalAllocatePrepare(bitrates Bitrates) { +func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, Bitrates Bitrates) { f.lock.Lock() defer f.lock.Unlock() f.provisional = &VideoAllocationProvisional{ - allocatedLayers: InvalidLayers, - muted: f.muted, - pubMuted: f.pubMuted, - maxPublishedLayer: f.maxPublishedLayer, - maxTemporalLayerSeen: f.maxTemporalLayerSeen, - bitrates: bitrates, - maxLayers: f.maxLayers, - currentLayers: f.currentLayers, - parkedLayers: f.parkedLayers, + allocatedLayer: buffer.InvalidLayer, + muted: f.muted, + pubMuted: f.pubMuted, + maxSeenLayer: f.vls.GetMaxSeen(), + Bitrates: Bitrates, + maxLayer: f.vls.GetMax(), + currentLayer: f.vls.GetCurrent(), + parkedLayer: f.vls.GetParked(), } + + f.provisional.availableLayers = make([]int32, len(availableLayers)) + copy(f.provisional.availableLayers, availableLayers) } -func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { +func (f *Forwarder) ProvisionalAllocateReset() { f.lock.Lock() defer f.lock.Unlock() - if f.provisional.muted || f.provisional.pubMuted || f.provisional.maxPublishedLayer == InvalidLayerSpatial || !f.provisional.maxLayers.IsValid() || (!allowOvershoot && layers.GreaterThan(f.provisional.maxLayers)) { + f.provisional.allocatedLayer = buffer.InvalidLayer +} + +func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { + f.lock.Lock() + defer f.lock.Unlock() + + if f.provisional.muted || + f.provisional.pubMuted || + f.provisional.maxSeenLayer.Spatial == buffer.InvalidLayerSpatial || + !f.provisional.maxLayer.IsValid() || + ((!allowOvershoot || !f.vls.IsOvershootOkay()) && layer.GreaterThan(f.provisional.maxLayer)) { return 0 } - requiredBitrate := f.provisional.bitrates[layers.Spatial][layers.Temporal] + requiredBitrate := f.provisional.Bitrates[layer.Spatial][layer.Temporal] if requiredBitrate == 0 { return 0 } alreadyAllocatedBitrate := int64(0) - if f.provisional.allocatedLayers.IsValid() { - alreadyAllocatedBitrate = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + if f.provisional.allocatedLayer.IsValid() { + alreadyAllocatedBitrate = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } // a layer under maximum fits, take it - if !layers.GreaterThan(f.provisional.maxLayers) && requiredBitrate <= (availableChannelCapacity+alreadyAllocatedBitrate) { - f.provisional.allocatedLayers = layers + if !layer.GreaterThan(f.provisional.maxLayer) && requiredBitrate <= (availableChannelCapacity+alreadyAllocatedBitrate) { + f.provisional.allocatedLayer = layer return requiredBitrate - alreadyAllocatedBitrate } // - // Given layer does not fit. But overshoot is allowed. + // Given layer does not fit. + // // Could be one of // 1. a layer below maximum that does not fit - // 2. a layer above maximum which may or may not fit. + // 2. a layer above maximum which may or may not fit, but overshoot is allowed. // In any of those cases, take the lowest possible layer if pause is not allowed // - if !allowPause && (!f.provisional.allocatedLayers.IsValid() || !layers.GreaterThan(f.provisional.allocatedLayers)) { - f.provisional.allocatedLayers = layers + if !allowPause && (!f.provisional.allocatedLayer.IsValid() || !layer.GreaterThan(f.provisional.allocatedLayer)) { + f.provisional.allocatedLayer = layer return requiredBitrate - alreadyAllocatedBitrate } @@ -704,29 +835,34 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b f.lock.Lock() defer f.lock.Unlock() + existingTargetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = InvalidLayers + bandwidthRequired := int64(0) + f.provisional.allocatedLayer = buffer.InvalidLayer if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayers = f.provisional.currentLayers + f.provisional.allocatedLayer = f.provisional.currentLayer + if f.provisional.allocatedLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] + } } return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: f.vls.GetTarget(), + To: f.provisional.allocatedLayer, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } // check if we should preserve current target - if f.targetLayers.IsValid() { + if existingTargetLayer.IsValid() { // what is the highest that is available - maximalLayers := InvalidLayers + maximalLayer := buffer.InvalidLayer maximalBandwidthRequired := int64(0) - for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { - for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { - if f.provisional.bitrates[s][t] != 0 { - maximalLayers = VideoLayers{Spatial: s, Temporal: t} - maximalBandwidthRequired = f.provisional.bitrates[s][t] + for s := f.provisional.maxLayer.Spatial; s >= 0; s-- { + for t := f.provisional.maxLayer.Temporal; t >= 0; t-- { + if f.provisional.Bitrates[s][t] != 0 { + maximalLayer = buffer.VideoLayer{Spatial: s, Temporal: t} + maximalBandwidthRequired = f.provisional.Bitrates[s][t] break } } @@ -736,25 +872,25 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } } - if maximalLayers.IsValid() { - if !f.targetLayers.GreaterThan(maximalLayers) && f.provisional.bitrates[f.targetLayers.Spatial][f.targetLayers.Temporal] != 0 { - // currently streaming and maybe wanting an upgrade (f.targetLayers <= maximalLayers), + if maximalLayer.IsValid() { + if !existingTargetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[existingTargetLayer.Spatial][existingTargetLayer.Temporal] != 0 { + // currently streaming and maybe wanting an upgrade (existingTargetLayer <= maximalLayer), // just preserve current target in the cooperative scheme of things - f.provisional.allocatedLayers = f.targetLayers + f.provisional.allocatedLayer = existingTargetLayer return VideoTransition{ - from: f.targetLayers, - to: f.targetLayers, - bandwidthDelta: 0, + From: existingTargetLayer, + To: existingTargetLayer, + BandwidthDelta: 0, } } - if f.targetLayers.GreaterThan(maximalLayers) { - // maximalLayers < f.targetLayers, make the down move - f.provisional.allocatedLayers = maximalLayers + if existingTargetLayer.GreaterThan(maximalLayer) { + // maximalLayer < existingTargetLayer, make the down move + f.provisional.allocatedLayer = maximalLayer return VideoTransition{ - from: f.targetLayers, - to: maximalLayers, - bandwidthDelta: maximalBandwidthRequired - f.lastAllocation.bandwidthRequested, + From: existingTargetLayer, + To: maximalLayer, + BandwidthDelta: maximalBandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } } @@ -763,14 +899,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b findNextLayer := func( minSpatial, maxSpatial int32, minTemporal, maxTemporal int32, - ) (VideoLayers, int64) { - layers := InvalidLayers + ) (buffer.VideoLayer, int64) { + layers := buffer.InvalidLayer bw := int64(0) for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - if f.provisional.bitrates[s][t] != 0 { - layers = VideoLayers{Spatial: s, Temporal: t} - bw = f.provisional.bitrates[s][t] + if f.provisional.Bitrates[s][t] != 0 { + layers = buffer.VideoLayer{Spatial: s, Temporal: t} + bw = f.provisional.Bitrates[s][t] break } } @@ -783,40 +919,44 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b return layers, bw } - targetLayers := f.targetLayers + targetLayer := buffer.InvalidLayer bandwidthRequired := int64(0) - if !targetLayers.IsValid() { + if !existingTargetLayer.IsValid() { // currently not streaming, find minimal // NOTE: a layer in feed could have paused and there could be other options than going back to minimal, // but the cooperative scheme knocks things back to minimal - targetLayers, bandwidthRequired = findNextLayer( - 0, f.provisional.maxLayers.Spatial, - 0, f.provisional.maxLayers.Temporal, + targetLayer, bandwidthRequired = findNextLayer( + 0, f.provisional.maxLayer.Spatial, + 0, f.provisional.maxLayer.Temporal, ) // could not find a minimal layer, overshoot if allowed - if bandwidthRequired == 0 && f.provisional.maxLayers.IsValid() && allowOvershoot { - targetLayers, bandwidthRequired = findNextLayer( - f.provisional.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + if bandwidthRequired == 0 && f.provisional.maxLayer.IsValid() && allowOvershoot && f.vls.IsOvershootOkay() { + targetLayer, bandwidthRequired = findNextLayer( + f.provisional.maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) } } // if nothing available, just leave target at current to enable opportunistic forwarding in case current resumes - if !targetLayers.IsValid() { - if f.provisional.parkedLayers.IsValid() { - targetLayers = f.provisional.parkedLayers + if !targetLayer.IsValid() { + if f.provisional.parkedLayer.IsValid() { + targetLayer = f.provisional.parkedLayer } else { - targetLayers = f.provisional.currentLayers + targetLayer = f.provisional.currentLayer + } + + if targetLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] } } - f.provisional.allocatedLayers = targetLayers + f.provisional.allocatedLayer = targetLayer return VideoTransition{ - from: f.targetLayers, - to: targetLayers, - bandwidthDelta: bandwidthRequired - f.lastAllocation.bandwidthRequested, + From: f.vls.GetTarget(), + To: targetLayer, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } @@ -839,67 +979,67 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.lock.Lock() defer f.lock.Unlock() + targetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = InvalidLayers - if f.provisional.pubMuted { - // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayers = f.provisional.currentLayers - } + // if publisher muted, give up opportunistic resume and give back the bandwidth + f.provisional.allocatedLayer = buffer.InvalidLayer return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: targetLayer, + To: f.provisional.allocatedLayer, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } - maxReachableLayerTemporal := InvalidLayerTemporal - for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { - for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { - if f.provisional.bitrates[s][t] != 0 { + maxReachableLayerTemporal := buffer.InvalidLayerTemporal + for t := f.provisional.maxLayer.Temporal; t >= 0; t-- { + for s := f.provisional.maxLayer.Spatial; s >= 0; s-- { + if f.provisional.Bitrates[s][t] != 0 { maxReachableLayerTemporal = t break } } - if maxReachableLayerTemporal != InvalidLayerTemporal { + if maxReachableLayerTemporal != buffer.InvalidLayerTemporal { break } } - if maxReachableLayerTemporal == InvalidLayerTemporal { + if maxReachableLayerTemporal == buffer.InvalidLayerTemporal { // feed has gone dry, just leave target at current to enable opportunistic forwarding in case current resumes. // Note that this is giving back bits and opportunistic forwarding resuming might trigger congestion again, // but that should be handled by stream allocator. - if f.provisional.parkedLayers.IsValid() { - f.provisional.allocatedLayers = f.provisional.parkedLayers + if f.provisional.parkedLayer.IsValid() { + f.provisional.allocatedLayer = f.provisional.parkedLayer } else { - f.provisional.allocatedLayers = f.provisional.currentLayers + f.provisional.allocatedLayer = f.provisional.currentLayer } return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: targetLayer, + To: f.provisional.allocatedLayer, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } // starting from minimum to target, find transition which gives the best // transition taking into account bits saved vs cost of such a transition - bestLayers := InvalidLayers + existingBandwidthNeeded := getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested) + bestLayer := buffer.InvalidLayer bestBandwidthDelta := int64(0) bestValue := float32(0) - for s := int32(0); s <= f.targetLayers.Spatial; s++ { - for t := int32(0); t <= f.targetLayers.Temporal; t++ { - if s == f.targetLayers.Spatial && t == f.targetLayers.Temporal { + for s := int32(0); s <= targetLayer.Spatial; s++ { + for t := int32(0); t <= targetLayer.Temporal; t++ { + if s == targetLayer.Spatial && t == targetLayer.Temporal { break } - bandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.bandwidthRequested-f.provisional.bitrates[s][t]))) + bandwidthDelta := int64(math.Max(float64(0), float64(existingBandwidthNeeded-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) - if f.targetLayers.Spatial != s { + // SVC-TODO: SVC will need a different cost transition + if targetLayer.Spatial != s { transitionCost = TransitionCostSpatial } - qualityCost := (maxReachableLayerTemporal+1)*(f.targetLayers.Spatial-s) + (f.targetLayers.Temporal - t) + qualityCost := (maxReachableLayerTemporal+1)*(targetLayer.Spatial-s) + (targetLayer.Temporal - t) value := float32(0) if (transitionCost + qualityCost) != 0 { @@ -908,16 +1048,16 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti if value > bestValue || (value == bestValue && bandwidthDelta > bestBandwidthDelta) { bestValue = value bestBandwidthDelta = bandwidthDelta - bestLayers = VideoLayers{Spatial: s, Temporal: t} + bestLayer = buffer.VideoLayer{Spatial: s, Temporal: t} } } } - f.provisional.allocatedLayers = bestLayers + f.provisional.allocatedLayer = bestLayer return VideoTransition{ - from: f.targetLayers, - to: bestLayers, - bandwidthDelta: bestBandwidthDelta, + From: targetLayer, + To: bestLayer, + BandwidthDelta: -bestBandwidthDelta, } } @@ -928,81 +1068,81 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { optimalBandwidthNeeded := getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, - f.provisional.bitrates, - f.provisional.maxLayers, + f.provisional.maxSeenLayer.Spatial, + f.provisional.Bitrates, + f.provisional.maxLayer, ) alloc := VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: -f.lastAllocation.bandwidthRequested, - bitrates: f.provisional.bitrates, - bandwidthNeeded: optimalBandwidthNeeded, - targetLayers: f.provisional.allocatedLayers, - requestLayerSpatial: f.provisional.allocatedLayers.Spatial, - maxLayers: f.provisional.maxLayers, - distanceToDesired: getDistanceToDesired( + BandwidthRequested: 0, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), + Bitrates: f.provisional.Bitrates, + BandwidthNeeded: optimalBandwidthNeeded, + TargetLayer: f.provisional.allocatedLayer, + RequestLayerSpatial: f.provisional.allocatedLayer.Spatial, + MaxLayer: f.provisional.maxLayer, + DistanceToDesired: getDistanceToDesired( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, - f.provisional.maxTemporalLayerSeen, - f.provisional.bitrates, - f.provisional.allocatedLayers, - f.provisional.maxLayers, + f.provisional.maxSeenLayer, + f.provisional.availableLayers, + f.provisional.Bitrates, + f.provisional.allocatedLayer, + f.provisional.maxLayer, ), } switch { case f.provisional.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.provisional.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted case optimalBandwidthNeeded == 0: - if f.provisional.allocatedLayers.IsValid() { + if f.provisional.allocatedLayer.IsValid() { // overshoot - alloc.bandwidthRequested = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } else { - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry // leave target at current for opportunistic forwarding - if f.provisional.currentLayers.IsValid() && f.provisional.currentLayers.Spatial <= f.provisional.maxLayers.Spatial { - f.provisional.allocatedLayers = f.provisional.currentLayers - alloc.targetLayers = f.provisional.allocatedLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + if f.provisional.currentLayer.IsValid() && f.provisional.currentLayer.Spatial <= f.provisional.maxLayer.Spatial { + f.provisional.allocatedLayer = f.provisional.currentLayer + alloc.TargetLayer = f.provisional.allocatedLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } } default: - if f.provisional.allocatedLayers.IsValid() { - alloc.bandwidthRequested = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + if f.provisional.allocatedLayer.IsValid() { + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) - if f.provisional.allocatedLayers.GreaterThan(f.provisional.maxLayers) || - alloc.bandwidthRequested >= getOptimalBandwidthNeeded( + if f.provisional.allocatedLayer.GreaterThan(f.provisional.maxLayer) || + alloc.BandwidthRequested >= getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, - f.provisional.bitrates, - f.provisional.maxLayers, + f.provisional.maxSeenLayer.Spatial, + f.provisional.Bitrates, + f.provisional.maxLayer, ) { // could be greater than optimal if overshooting - alloc.isDeficient = false + alloc.IsDeficient = false } else { - alloc.isDeficient = true - if !f.provisional.allocatedLayers.IsValid() { - alloc.pauseReason = VideoPauseReasonBandwidth + alloc.IsDeficient = true + if !f.provisional.allocatedLayer.IsValid() { + alloc.PauseReason = VideoPauseReasonBandwidth } } } - f.clearParkedLayers() + f.clearParkedLayer() return f.updateAllocation(alloc, "cooperative") } -func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { +func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, availableLayers []int32, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { f.lock.Lock() defer f.lock.Unlock() @@ -1016,15 +1156,18 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra } // if targets are still pending, don't increase - if f.targetLayers.IsValid() && f.targetLayers != f.currentLayers { + targetLayer := f.vls.GetTarget() + if targetLayer.IsValid() && targetLayer != f.vls.GetCurrent() { return f.lastAllocation, false } - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alreadyAllocated := int64(0) - if f.targetLayers.IsValid() { - alreadyAllocated = brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + if targetLayer.IsValid() { + alreadyAllocated = brs[targetLayer.Spatial][targetLayer.Temporal] } doAllocation := func( @@ -1038,25 +1181,33 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra continue } - if !allowOvershoot && bandwidthRequested-alreadyAllocated > availableChannelCapacity { + if (!allowOvershoot || !f.vls.IsOvershootOkay()) && bandwidthRequested-alreadyAllocated > availableChannelCapacity { // next higher available layer does not fit, return return true, f.lastAllocation, false } - targetLayers := VideoLayers{Spatial: s, Temporal: t} + newTargetLayer := buffer.VideoLayer{Spatial: s, Temporal: t} alloc := VideoAllocation{ - isDeficient: true, - bandwidthRequested: bandwidthRequested, - bandwidthDelta: bandwidthRequested - alreadyAllocated, - bandwidthNeeded: optimalBandwidthNeeded, - bitrates: brs, - targetLayers: targetLayers, - requestLayerSpatial: targetLayers.Spatial, - maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, targetLayers, f.maxLayers), + IsDeficient: true, + BandwidthRequested: bandwidthRequested, + BandwidthDelta: bandwidthRequested - alreadyAllocated, + BandwidthNeeded: optimalBandwidthNeeded, + Bitrates: brs, + TargetLayer: newTargetLayer, + RequestLayerSpatial: newTargetLayer.Spatial, + MaxLayer: maxLayer, + DistanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + maxSeenLayer, + availableLayers, + brs, + newTargetLayer, + maxLayer, + ), } - if targetLayers.GreaterThan(f.maxLayers) || bandwidthRequested >= optimalBandwidthNeeded { - alloc.isDeficient = false + if newTargetLayer.GreaterThan(maxLayer) || bandwidthRequested >= optimalBandwidthNeeded { + alloc.IsDeficient = false } return true, f.updateAllocation(alloc, "next-higher"), true @@ -1071,10 +1222,10 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra boosted := false // try moving temporal layer up in currently streaming spatial layer - if f.targetLayers.IsValid() { + if targetLayer.IsValid() { done, allocation, boosted = doAllocation( - f.targetLayers.Spatial, f.targetLayers.Spatial, - f.targetLayers.Temporal+1, f.maxLayers.Temporal, + targetLayer.Spatial, targetLayer.Spatial, + targetLayer.Temporal+1, maxLayer.Temporal, ) if done { return allocation, boosted @@ -1083,17 +1234,17 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra // try moving spatial layer up if temporal layer move up is not available done, allocation, boosted = doAllocation( - f.targetLayers.Spatial+1, f.maxLayers.Spatial, - 0, f.maxLayers.Temporal, + targetLayer.Spatial+1, maxLayer.Spatial, + 0, maxLayer.Temporal, ) if done { return allocation, boosted } - if allowOvershoot && f.maxLayers.IsValid() { + if allowOvershoot && f.vls.IsOvershootOkay() && maxLayer.IsValid() { done, allocation, boosted = doAllocation( - f.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) if done { return allocation, boosted @@ -1117,13 +1268,14 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( } // if targets are still pending, don't increase - if f.targetLayers.IsValid() && f.targetLayers != f.currentLayers { + targetLayer := f.vls.GetTarget() + if targetLayer.IsValid() && targetLayer != f.vls.GetCurrent() { return VideoTransition{}, false } alreadyAllocated := int64(0) - if f.targetLayers.IsValid() { - alreadyAllocated = brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + if targetLayer.IsValid() { + alreadyAllocated = brs[targetLayer.Spatial][targetLayer.Temporal] } findNextHigher := func( @@ -1138,9 +1290,9 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( } transition := VideoTransition{ - from: f.targetLayers, - to: VideoLayers{Spatial: s, Temporal: t}, - bandwidthDelta: bandwidthRequested - alreadyAllocated, + From: targetLayer, + To: buffer.VideoLayer{Spatial: s, Temporal: t}, + BandwidthDelta: bandwidthRequested - alreadyAllocated, } return true, transition, true @@ -1155,10 +1307,11 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( isAvailable := false // try moving temporal layer up in currently streaming spatial layer - if f.targetLayers.IsValid() { + maxLayer := f.vls.GetMax() + if targetLayer.IsValid() { done, transition, isAvailable = findNextHigher( - f.targetLayers.Spatial, f.targetLayers.Spatial, - f.targetLayers.Temporal+1, f.maxLayers.Temporal, + targetLayer.Spatial, targetLayer.Spatial, + targetLayer.Temporal+1, maxLayer.Temporal, ) if done { return transition, isAvailable @@ -1167,17 +1320,17 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( // try moving spatial layer up if temporal layer move up is not available done, transition, isAvailable = findNextHigher( - f.targetLayers.Spatial+1, f.maxLayers.Spatial, - 0, f.maxLayers.Temporal, + targetLayer.Spatial+1, maxLayer.Spatial, + 0, maxLayer.Temporal, ) if done { return transition, isAvailable } - if allowOvershoot && f.maxLayers.IsValid() { + if allowOvershoot && f.vls.IsOvershootOkay() && maxLayer.IsValid() { done, transition, isAvailable = findNextHigher( - f.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) if done { return transition, isAvailable @@ -1187,66 +1340,81 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( return VideoTransition{}, false } -func (f *Forwarder) Pause(brs Bitrates) VideoAllocation { +func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation { f.lock.Lock() defer f.lock.Unlock() - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alloc := VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, - bitrates: brs, - bandwidthNeeded: optimalBandwidthNeeded, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, InvalidLayers, f.maxLayers), + BandwidthRequested: 0, + BandwidthDelta: 0 - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), + Bitrates: brs, + BandwidthNeeded: optimalBandwidthNeeded, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: maxLayer, + DistanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + maxSeenLayer, + availableLayers, + brs, + buffer.InvalidLayer, + maxLayer, + ), } switch { case f.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted case optimalBandwidthNeeded == 0: - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry default: // pausing due to lack of bandwidth - alloc.isDeficient = true - alloc.pauseReason = VideoPauseReasonBandwidth + alloc.IsDeficient = true + alloc.PauseReason = VideoPauseReasonBandwidth } - f.clearParkedLayers() + f.clearParkedLayer() return f.updateAllocation(alloc, "pause") } func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) VideoAllocation { - if alloc.isDeficient != f.lastAllocation.isDeficient || - alloc.pauseReason != f.lastAllocation.pauseReason || - alloc.targetLayers != f.lastAllocation.targetLayers || - alloc.requestLayerSpatial != f.lastAllocation.requestLayerSpatial { - f.logger.Infow(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + // restrict target temporal to 0 if codec does not support temporal layers + if alloc.TargetLayer.IsValid() && strings.ToLower(f.codec.MimeType) == "video/h264" { + alloc.TargetLayer.Temporal = 0 + } + + if alloc.IsDeficient != f.lastAllocation.IsDeficient || + alloc.PauseReason != f.lastAllocation.PauseReason || + alloc.TargetLayer != f.lastAllocation.TargetLayer || + alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial { + if reason == "optimal" { + f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + } else { + f.logger.Infow(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + } } f.lastAllocation = alloc - f.setTargetLayers(f.lastAllocation.targetLayers, f.lastAllocation.requestLayerSpatial) - if !f.targetLayers.IsValid() { + f.setTargetLayer(f.lastAllocation.TargetLayer, f.lastAllocation.RequestLayerSpatial) + if !f.vls.GetTarget().IsValid() { f.resyncLocked() } return f.lastAllocation } -func (f *Forwarder) setTargetLayers(targetLayers VideoLayers, requestLayerSpatial int32) { - f.targetLayers = targetLayers - if f.ddLayerSelector != nil { - f.ddLayerSelector.SelectLayer(targetLayers) - } - - f.requestLayerSpatial = requestLayerSpatial +func (f *Forwarder) setTargetLayer(targetLayer buffer.VideoLayer, requestLayerSpatial int32) { + f.vls.SetTarget(targetLayer) + f.vls.SetRequestSpatial(requestLayerSpatial) } func (f *Forwarder) Resync() { @@ -1257,30 +1425,31 @@ func (f *Forwarder) Resync() { } func (f *Forwarder) resyncLocked() { - f.currentLayers = InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayer) f.lastSSRC = 0 - f.clearParkedLayers() + f.clearParkedLayer() } -func (f *Forwarder) clearParkedLayers() { - f.parkedLayers = InvalidLayers - if f.parkedLayersTimer != nil { - f.parkedLayersTimer.Stop() - f.parkedLayersTimer = nil +func (f *Forwarder) clearParkedLayer() { + f.vls.SetParked(buffer.InvalidLayer) + if f.parkedLayerTimer != nil { + f.parkedLayerTimer.Stop() + f.parkedLayerTimer = nil } } -func (f *Forwarder) setupParkedLayers(parkedLayers VideoLayers) { - f.clearParkedLayers() +func (f *Forwarder) setupParkedLayer(parkedLayer buffer.VideoLayer) { + f.clearParkedLayer() - f.parkedLayers = parkedLayers - f.parkedLayersTimer = time.AfterFunc(ParkedLayersWaitDuration, func() { + f.vls.SetParked(parkedLayer) + f.parkedLayerTimer = time.AfterFunc(ParkedLayerWaitDuration, func() { f.lock.Lock() - f.clearParkedLayers() + notify := f.vls.GetParked().IsValid() + f.clearParkedLayer() f.lock.Unlock() - if onParkedLayersExpired := f.getOnParkedLayersExpired(); onParkedLayersExpired != nil { - onParkedLayersExpired() + if onParkedLayerExpired := f.getOnParkedLayerExpired(); onParkedLayerExpired != nil && notify { + onParkedLayerExpired() } }) } @@ -1289,12 +1458,10 @@ func (f *Forwarder) CheckSync() (locked bool, layer int32) { f.lock.RLock() defer f.lock.RUnlock() - layer = f.requestLayerSpatial - locked = f.requestLayerSpatial == f.currentLayers.Spatial || f.parkedLayers.IsValid() - return + return f.vls.CheckSync() } -func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [DefaultMaxLayerSpatial + 1]bool) { +func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) { if !FlagFilterRTX { filtered = nacks return @@ -1313,8 +1480,10 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye // // Without the curb, when congestion hits, RTX rate could be so high that it further congests the channel. // - for layer := int32(0); layer < DefaultMaxLayerSpatial+1; layer++ { - if f.isDeficientLocked() && (f.targetLayers.Spatial < f.currentLayers.Spatial || layer > f.currentLayers.Spatial) { + currentLayer := f.vls.GetCurrent() + targetLayer := f.vls.GetTarget() + for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { + if f.isDeficientLocked() && (targetLayer.Spatial < currentLayer.Spatial || layer > currentLayer.Spatial) { disallowedLayers[layer] = true } } @@ -1350,63 +1519,182 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) return nil, ErrUnknownKind } -// should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (*TranslationParams, error) { - if f.lastSSRC != extPkt.Packet.SSRC { - if !f.started { - f.started = true - f.referenceLayerSpatial = layer - f.rtpMunger.SetLastSnTs(extPkt) - if f.vp8Munger != nil { - f.vp8Munger.SetLast(extPkt) +func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { + if !f.started { + f.started = true + f.referenceLayerSpatial = layer + f.rtpMunger.SetLastSnTs(extPkt) + f.codecMunger.SetLast(extPkt) + f.logger.Infow( + "starting forwarding", + "sequenceNumber", extPkt.Packet.SequenceNumber, + "timestamp", extPkt.Packet.Timestamp, + "layer", layer, + "referenceLayerSpatial", f.referenceLayerSpatial, + ) + return nil + } + + if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { + // on a resume, reference layer may not be set, so only set when it is invalid + f.referenceLayerSpatial = layer + } + + // Compute how much time passed between the previous forwarded packet + // and the current incoming (to be forwarded) packet and calculate + // timestamp offset on source change. + // + // There are three timestamps to consider here + // 1. lastTS -> timestamp of last sent packet + // 2. refTS -> timestamp of this packet (after munging) calculated using feed's RTCP sender report + // 3. expectedTS -> expected timestamp of this packet calculated based on elapsed time since first packet + // Ideally, refTS and expectedTS should be very close and lastTS should be before both of those. + // But, cases like muting/unmuting, clock vagaries, pacing, etc. make them not satisfy those conditions always. + lastTS := f.rtpMunger.GetLast().LastTS + refTS := lastTS + expectedTS := lastTS + switchingAt := time.Now() + if f.getReferenceLayerRTPTimestamp != nil { + ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + if err == nil { + refTS = ts + } + // AVSYNC-TODO: can error out here if refTS is not available. It can happen when there is no sender report + // for the layer being switched to. Can especially happen at the start of the track when layer switches are + // potentially happening very quickly. Erroring out and waiting for a layer for which a sender report has been + // received will calculate a better offset, but may result in initial adaptation to take a bit longer depending + // on how often publisher/remote side sends RTCP sender report. + } + + if f.getExpectedRTPTimestamp != nil { + tsExt, err := f.getExpectedRTPTimestamp(switchingAt) + if err == nil { + expectedTS = uint32(tsExt) + } else { + rtpDiff := uint32(0) + if !f.preStartTime.IsZero() && f.refTSOffset == 0 { + timeSinceFirst := time.Since(f.preStartTime) + rtpDiff = uint32(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) + f.refTSOffset = f.firstTS + rtpDiff - refTS + f.logger.Infow( + "calculating refTSOffset", + "preStartTime", f.preStartTime.String(), + "firstTS", f.firstTS, + "timeSinceFirst", timeSinceFirst, + "rtpDiff", rtpDiff, + "refTS", refTS, + "refTSOffset", f.refTSOffset, + ) + } + expectedTS += rtpDiff + } + } + refTS += f.refTSOffset + + var nextTS uint32 + if f.lastSSRC == 0 { + // If resuming (e. g. on unmute), keep next timestamp close to expected timestamp. + // + // Rationale: + // Case 1: If mute is implemented via something like stopping a track and resuming it on unmute, + // the RTP timestamp may not have jumped across mute valley. In this case, old timestamp + // should not be used. + // + // Case 2: OTOH, something like pacing may be adding latency in the publisher path (even if + // the timestamps incremented correctly across the mute valley). In this case, reference + // timestamp should be used as things will catch up to real time when channel capacity + // increases and pacer starts sending at faster rate. + // + // But, the challenege is distinguishing between the two cases. As a compromise, the difference + // between expectedTS and refTS is thresholded. Difference below the threshold is treated as Case 2 + // and above as Case 1. + // + // In the event of refTS > expectedTS, another threshold is used to pick the next timestamp. + // Ideally, refTS should not be ahead of expectedTS, but expectedTS uses the first packet's + // wall clock time. So, if the first packet experienced abmormal latency, it is possible + // for refTS > expectedTS + diffSeconds := float64(expectedTS-refTS) / float64(f.codec.ClockRate) + if diffSeconds >= 0.0 { + if diffSeconds > ResumeBehindThresholdSeconds { + f.logger.Infow("resume, reference too far behind", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", diffSeconds) + nextTS = expectedTS + } else { + nextTS = refTS } } else { - if f.referenceLayerSpatial == InvalidLayerSpatial { - // on a resume, reference layer may not be set, so only set when it is invalid - f.referenceLayerSpatial = layer - } - - // Compute how much time passed between the old RTP extPkt - // and the current packet, and fix timestamp on source change - td := uint32(1) - if f.getReferenceLayerRTPTimestamp != nil { - refTS, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) - if err == nil { - last := f.rtpMunger.GetLast() - td = refTS - last.LastTS - if td == 0 || td > (1<<31) { - f.logger.Infow("reference timestamp out-of-order, using default", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td)) - td = 1 - } - } else { - f.logger.Infow("reference timestamp get error, using default", "error", err) - } - } - - f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, td) - if f.vp8Munger != nil { - f.vp8Munger.UpdateOffsets(extPkt) + if math.Abs(diffSeconds) > SwitchAheadThresholdSeconds { + f.logger.Infow("resume, reference too far ahead", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", math.Abs(diffSeconds)) + nextTS = expectedTS + } else { + nextTS = refTS } } + } else { + // switching between layers, check if refTS is too far behind the last sent + diffSeconds := float64(refTS-lastTS) / float64(f.codec.ClockRate) + if diffSeconds < 0.0 { + if math.Abs(diffSeconds) > LayerSwitchBehindThresholdSeconds { + // AVSYNC-TODO: This could be due to pacer trickling out this layer. Should potentially return error here and wait for a more opportune time + // or some forcing function (like "have waited for too long for layer switch, nothing available, switch to whatever is available" kind of condition) + // to do the switch. Just logging it for now. + f.logger.Infow("layer switch, reference too far behind", "expectedTS", expectedTS, "refTS", refTS, "lastTS", lastTS, "diffSeconds", math.Abs(diffSeconds)) + } + // use a nominal increase to ensure that timestamp is always moving forward + nextTS = lastTS + 1 + } else { + diffSeconds = float64(expectedTS-refTS) / float64(f.codec.ClockRate) + if diffSeconds < 0.0 && math.Abs(diffSeconds) > SwitchAheadThresholdSeconds { + f.logger.Infow("layer switch, reference too far ahead", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", math.Abs(diffSeconds)) + nextTS = expectedTS + } else { + nextTS = refTS + } + } + } + if nextTS-lastTS == 0 || nextTS-lastTS > (1<<31) { + f.logger.Infow("next timestamp is before last, adjusting", "nextTS", nextTS, "lastTS", lastTS) + // nominal increase + nextTS = lastTS + 1 + } + f.logger.Infow( + "next timestamp on switch", + "switchingAt", switchingAt.String(), + "layer", layer, + "lastTS", lastTS, + "refTS", refTS, + "refTSOffset", f.refTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, + "expectedTS", expectedTS, + "nextTS", nextTS, + "jump", nextTS-lastTS, + ) + + f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, nextTS-lastTS) + f.codecMunger.UpdateOffsets(extPkt) + return nil +} + +// should be called with lock held +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (*TranslationParams, error) { + if tp == nil { + tp = &TranslationParams{} + } + if f.lastSSRC != extPkt.Packet.SSRC { + if err := f.processSourceSwitch(extPkt, layer); err != nil { + tp.shouldDrop = true + return tp, err + } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC } - if tp == nil { - tp = &TranslationParams{} - } tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt) if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - if err == ErrOutOfOrderSequenceNumberCacheMiss { - tp.isDroppingRelevant = true - } return tp, nil } - - tp.isDroppingRelevant = true return tp, err } @@ -1421,150 +1709,37 @@ func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer in // should be called with lock held func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { + maybeRollback := func(isSwitching bool) { + if isSwitching { + f.vls.Rollback() + } + } + tp := &TranslationParams{} - if !f.targetLayers.IsValid() { + if !f.vls.GetTarget().IsValid() { // stream is paused by streamallocator tp.shouldDrop = true return tp, nil } - if f.ddLayerSelector != nil { - if selected := f.ddLayerSelector.Select(extPkt, tp); !selected { - tp.shouldDrop = true - f.rtpMunger.PacketDropped(extPkt) - return tp, nil - } else if tp.isSwitchingToTargetLayer { - // lock to target layer - f.logger.Infow( - "locking to target layer", - "current", f.currentLayers, - "target", f.targetLayers, - "req", f.requestLayerSpatial, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers.Spatial = f.targetLayers.Spatial - if !f.isTemporalSupported { - f.currentLayers.Temporal = f.targetLayers.Temporal - } - // TODO : we switch to target layer immediately now since we assume all frame chain is integrity - // if we have frame chain check, should switch only if target chain is not broken and decodable - // if f.ddLayerSelector != nil { - // f.ddLayerSelector.SelectLayer(f.currentLayers) - // } - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - tp.isSwitchingToMaxLayer = true - } - } - } else { - if f.currentLayers.Spatial != f.targetLayers.Spatial { - // Three things to check when not locked to target - // 1. Resumable layer - don't need a key frame - // 2. Opportunistic layer upgrade - needs a key frame - // 3. Need to downgrade - needs a key frame - found := false - if f.parkedLayers.IsValid() { - if f.parkedLayers.Spatial == layer { - f.logger.Infow( - "resuming at parked layer", - "current", f.currentLayers, - "target", f.targetLayers, - "parked", f.parkedLayers, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers = f.parkedLayers - found = true - } - } else { - if extPkt.KeyFrame { - if layer > f.currentLayers.Spatial && layer <= f.targetLayers.Spatial { - f.logger.Infow( - "upgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if layer < f.currentLayers.Spatial && layer >= f.targetLayers.Spatial { - f.logger.Infow( - "downgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if found { - f.currentLayers.Spatial = layer - if !f.isTemporalSupported { - f.currentLayers.Temporal = extPkt.Temporal - } - } - } - } - - if found { - tp.isSwitchingToTargetLayer = true - f.clearParkedLayers() - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - tp.isSwitchingToMaxLayer = true - - // if maximum is attained, adjust target to enable fast path layer check in per-packet path - f.logger.Infow( - "reached max layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - f.targetLayers.Spatial = f.currentLayers.Spatial - } - } - } - - // if locked to higher than max layer due to overshoot, check if it can be dialed back - if f.currentLayers.Spatial > f.maxLayers.Spatial { - if layer <= f.maxLayers.Spatial && extPkt.KeyFrame { - f.logger.Infow( - "adjusting overshoot", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers.Spatial = layer - - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - tp.isSwitchingToMaxLayer = true - f.targetLayers.Spatial = layer - } - } - } - } - - if f.currentLayers.Spatial != layer { + result := f.vls.Select(extPkt, layer) + if !result.IsSelected { tp.shouldDrop = true + if f.started && result.IsRelevant { + // call to update highest incoming sequence number and other internal structures + if _, err := f.rtpMunger.UpdateAndGetSnTs(extPkt); err == nil { + f.rtpMunger.PacketDropped(extPkt) + } + } return tp, nil } + tp.isResuming = result.IsResuming + tp.isSwitching = result.IsSwitching + tp.ddBytes = result.DependencyDescriptorExtension + tp.marker = result.RTPMarker - if FlagPauseOnDowngrade && f.targetLayers.Spatial < f.currentLayers.Spatial && f.isDeficientLocked() { + if FlagPauseOnDowngrade && f.isDeficientLocked() && f.vls.GetTarget().Spatial < f.vls.GetCurrent().Spatial { // // If target layer is lower than both the current and // maximum subscribed layer, it is due to bandwidth @@ -1584,81 +1759,119 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // To differentiate between the two cases, drop only when in DEFICIENT state. // tp.shouldDrop = true - tp.isDroppingRelevant = true + maybeRollback(result.IsSwitching) return tp, nil } _, err := f.getTranslationParamsCommon(extPkt, layer, tp) - if tp.shouldDrop || f.vp8Munger == nil || len(extPkt.Packet.Payload) == 0 { + if tp.shouldDrop || len(extPkt.Packet.Payload) == 0 { + maybeRollback(result.IsSwitching) return tp, err } - // catch up temporal layer if necessary - if f.currentLayers.Temporal != f.targetLayers.Temporal { - incomingVP8, ok := extPkt.Payload.(buffer.VP8) - if ok { - if incomingVP8.TIDPresent == 0 || incomingVP8.TID <= uint8(f.targetLayers.Temporal) { - f.currentLayers.Temporal = f.targetLayers.Temporal - } - } - } - - tpVP8, err := f.vp8Munger.UpdateAndGet(extPkt, tp.rtp.snOrdering, f.currentLayers.Temporal) + // codec specific forwarding check and any needed packet munging + tl, isSwitching := f.vls.SelectTemporal(extPkt) + codecBytes, err := f.codecMunger.UpdateAndGet( + extPkt, + tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, + tp.rtp.snOrdering == SequenceNumberOrderingGap, + tl, + ) if err != nil { tp.rtp = nil tp.shouldDrop = true - if err == ErrFilteredVP8TemporalLayer || err == ErrOutOfOrderVP8PictureIdCacheMiss { - if err == ErrFilteredVP8TemporalLayer { + if err == codecmunger.ErrFilteredVP8TemporalLayer || err == codecmunger.ErrOutOfOrderVP8PictureIdCacheMiss { + if err == codecmunger.ErrFilteredVP8TemporalLayer { // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - if err == ErrOutOfOrderVP8PictureIdCacheMiss { - tp.isDroppingRelevant = true - } + maybeRollback(result.IsSwitching || isSwitching) return tp, nil } - tp.isDroppingRelevant = true + maybeRollback(result.IsSwitching || isSwitching) return tp, err } - tp.vp8 = tpVP8 + tp.codecBytes = codecBytes return tp, nil } -func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { +func (f *Forwarder) maybeStart() { + if f.started { + return + } + + f.started = true + f.preStartTime = time.Now() + + extPkt := &buffer.ExtPacket{ + Packet: &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: uint16(rand.Intn(1<<14)) + uint16(1<<15), // a random number in third quartile of sequence number space + Timestamp: uint32(rand.Intn(1<<30)) + uint32(1<<31), // a random number in third quartile of timestamp space + }, + }, + } + f.rtpMunger.SetLastSnTs(extPkt) + + f.firstTS = extPkt.Packet.Timestamp + f.logger.Infow( + "starting with dummy forwarding", + "sequenceNumber", extPkt.Packet.SequenceNumber, + "timestamp", extPkt.Packet.Timestamp, + "preStartTime", f.preStartTime, + ) +} + +func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) { f.lock.Lock() defer f.lock.Unlock() - // padding is used for probing. Padding packets should be - // at only the frame boundaries to ensure decoder sequencer does + f.maybeStart() + + // padding is used for probing. Padding packets should only + // be at frame boundaries to ensure decoder sequencer does // not get out-of-sync. But, when a stream is paused, // force a frame marker as a restart of the stream will // start with a key frame which will reset the decoder. - forceMarker := false - if !f.targetLayers.IsValid() { + if !f.vls.GetTarget().IsValid() { forceMarker = true } - return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker) + return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker, 0) } func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]SnTs, bool, error) { f.lock.Lock() defer f.lock.Unlock() + f.maybeStart() + frameEndNeeded := !f.rtpMunger.IsOnFrameBoundary() if frameEndNeeded { numPackets++ } - snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded) + + lastTS := f.rtpMunger.GetLast().LastTS + expectedTS := lastTS + if f.getExpectedRTPTimestamp != nil { + tsExt, err := f.getExpectedRTPTimestamp(time.Now()) + if err == nil { + expectedTS = uint32(tsExt) + } + } + if expectedTS-lastTS == 0 || expectedTS-lastTS > (1<<31) { + expectedTS = lastTS + 1 + } + snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded, expectedTS) return snts, frameEndNeeded, err } -func (f *Forwarder) GetPaddingVP8(frameEndNeeded bool) *buffer.VP8 { +func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { f.lock.Lock() defer f.lock.Unlock() - return f.vp8Munger.UpdateAndGetPadding(!frameEndNeeded) + return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) } func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { @@ -1670,13 +1883,13 @@ func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { // ----------------------------------------------------------------------------- -func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayers VideoLayers) int64 { - if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial { +func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayer buffer.VideoLayer) int64 { + if muted || pubMuted || maxPublishedLayer == buffer.InvalidLayerSpatial { return 0 } - for i := maxLayers.Spatial; i >= 0; i-- { - for j := maxLayers.Temporal; j >= 0; j-- { + for i := maxLayer.Spatial; i >= 0; i-- { + for j := maxLayer.Temporal; j >= 0; j-- { if brs[i][j] == 0 { continue } @@ -1693,52 +1906,98 @@ func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int3 return 0 } +func getBandwidthNeeded(brs Bitrates, layer buffer.VideoLayer, fallback int64) int64 { + if layer.IsValid() && brs[layer.Spatial][layer.Temporal] > 0 { + return brs[layer.Spatial][layer.Temporal] + } + + return fallback +} + func getDistanceToDesired( muted bool, pubMuted bool, - maxPublishedLayer int32, - maxTemporalLayerSeen int32, + maxSeenLayer buffer.VideoLayer, + availableLayers []int32, brs Bitrates, - targetLayers VideoLayers, - maxLayers VideoLayers, + targetLayer buffer.VideoLayer, + maxLayer buffer.VideoLayer, ) float64 { - if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial || !maxLayers.IsValid() { + if muted || pubMuted || !maxSeenLayer.IsValid() || !maxLayer.IsValid() { return 0.0 } - found := false - distance := float64(0.0) + adjustedMaxLayer := maxLayer + + maxAvailableSpatial := buffer.InvalidLayerSpatial + maxAvailableTemporal := buffer.InvalidLayerTemporal + + // max available spatial is min(subscribedMax, publishedMax, availableMax) + // subscribedMax = subscriber requested max spatial layer + // publishedMax = max spatial layer ever published + // availableMax = based on bit rate measurement, available max spatial layer done: - for s := maxLayers.Spatial; s >= 0; s-- { - for t := maxLayers.Temporal; t >= 0; t-- { - if brs[s][t] == 0 { - continue - } - if s == targetLayers.Spatial && t == targetLayers.Temporal { - found = true + for s := int32(len(brs)) - 1; s >= 0; s-- { + for t := int32(len(brs[0])) - 1; t >= 0; t-- { + if brs[s][t] != 0 { + maxAvailableSpatial = s break done } - - distance++ } } - // maybe overshooting - if !found && targetLayers.IsValid() { - distance = 0.0 - for s := targetLayers.Spatial; s > maxLayers.Spatial; s-- { - for t := maxLayers.Temporal; t >= 0; t-- { - if targetLayers.Temporal < t || brs[s][t] == 0 { - continue - } - distance-- + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range availableLayers { + if layer > maxAvailableSpatial { + maxAvailableSpatial = layer + maxAvailableTemporal = maxSeenLayer.Temporal // till bit rate measurement is available, assume max seen as temporal + } + } + + if maxAvailableSpatial < adjustedMaxLayer.Spatial { + adjustedMaxLayer.Spatial = maxAvailableSpatial + } + + if maxSeenLayer.Spatial < adjustedMaxLayer.Spatial { + adjustedMaxLayer.Spatial = maxSeenLayer.Spatial + } + + // max available temporal is min(subscribedMax, temporalLayerSeenMax, availableMax) + // subscribedMax = subscriber requested max temporal layer + // temporalLayerSeenMax = max temporal layer ever published/seen + // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer + if adjustedMaxLayer.Spatial != buffer.InvalidLayerSpatial { + for t := int32(len(brs[0])) - 1; t >= 0; t-- { + if brs[adjustedMaxLayer.Spatial][t] != 0 { + maxAvailableTemporal = t + break } } } - - if maxTemporalLayerSeen < 0 { - maxTemporalLayerSeen = 0 + if maxAvailableTemporal < adjustedMaxLayer.Temporal { + adjustedMaxLayer.Temporal = maxAvailableTemporal } - return distance / float64(maxTemporalLayerSeen+1) + if maxSeenLayer.Temporal < adjustedMaxLayer.Temporal { + adjustedMaxLayer.Temporal = maxSeenLayer.Temporal + } + + if !adjustedMaxLayer.IsValid() { + adjustedMaxLayer = buffer.VideoLayer{Spatial: 0, Temporal: 0} + } + + // adjust target layers if they are invalid, i. e. not streaming + adjustedTargetLayer := targetLayer + if !targetLayer.IsValid() { + adjustedTargetLayer = buffer.VideoLayer{Spatial: 0, Temporal: 0} + } + + distance := + ((adjustedMaxLayer.Spatial - adjustedTargetLayer.Spatial) * (maxSeenLayer.Temporal + 1)) + + (adjustedMaxLayer.Temporal - adjustedTargetLayer.Temporal) + if !targetLayer.IsValid() { + distance += (maxSeenLayer.Temporal + 1) + } + + return float64(distance) / float64(maxSeenLayer.Temporal+1) } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 2765bb0ce..935009792 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -13,26 +27,26 @@ import ( ) func disable(f *Forwarder) { - f.currentLayers = InvalidLayers - f.targetLayers = InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayer) + f.vls.SetTarget(buffer.InvalidLayer) } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { - f := NewForwarder(kind, logger.GetLogger(), nil) - f.DetermineCodec(codec) + f := NewForwarder(kind, logger.GetLogger(), nil, nil) + f.DetermineCodec(codec, nil) return f } func TestForwarderMute(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) require.False(t, f.IsMuted()) - muted, _ := f.Mute(false) + muted := f.Mute(false) require.False(t, muted) // no change in mute state require.False(t, f.IsMuted()) - muted, _ = f.Mute(true) + muted = f.Mute(true) require.True(t, muted) require.True(t, f.IsMuted()) - muted, _ = f.Mute(false) + muted = f.Mute(false) require.True(t, muted) require.False(t, f.IsMuted()) } @@ -40,74 +54,67 @@ func TestForwarderMute(t *testing.T) { func TestForwarderLayersAudio(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) - require.Equal(t, InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayer, f.MaxLayer()) - require.Equal(t, InvalidLayers, f.CurrentLayers()) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) - changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(1) + changed, maxLayer := f.SetMaxSpatialLayer(1) require.False(t, changed) - require.Equal(t, InvalidLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayer, maxLayer) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(1) + changed, maxLayer = f.SetMaxTemporalLayer(1) require.False(t, changed) - require.Equal(t, InvalidLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayer, maxLayer) - require.Equal(t, InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayer, f.MaxLayer()) } func TestForwarderLayersVideo(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - maxLayers := f.MaxLayers() - expectedLayers := VideoLayers{Spatial: InvalidLayerSpatial, Temporal: DefaultMaxLayerTemporal} - require.Equal(t, expectedLayers, maxLayers) + maxLayer := f.MaxLayer() + expectedLayers := buffer.VideoLayer{Spatial: buffer.InvalidLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal} + require.Equal(t, expectedLayers, maxLayer) - require.Equal(t, InvalidLayers, f.CurrentLayers()) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial, - Temporal: DefaultMaxLayerTemporal, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial, + Temporal: buffer.DefaultMaxLayerTemporal, } - changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) + changed, maxLayer := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) require.True(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, expectedLayers, maxLayer) - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(DefaultMaxLayerSpatial - 1) + changed, maxLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.True(t, changed) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial - 1, - Temporal: DefaultMaxLayerTemporal, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial - 1, + Temporal: buffer.DefaultMaxLayerTemporal, } - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(DefaultMaxLayerSpatial - 1) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + changed, maxLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.False(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + changed, maxLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) require.False(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(DefaultMaxLayerTemporal - 1) + changed, maxLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) require.True(t, changed) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial - 1, - Temporal: DefaultMaxLayerTemporal - 1, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial - 1, + Temporal: buffer.DefaultMaxLayerTemporal - 1, } - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) } func TestForwarderAllocateOptimal(t *testing.T) { @@ -121,53 +128,53 @@ func TestForwarderAllocateOptimal(t *testing.T) { } // invalid max layers - f.maxLayers = InvalidLayers + f.vls.SetMax(buffer.InvalidLayer) expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: InvalidLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.InvalidLayer, + DistanceToDesired: 0, } result := f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) - // should still have target at InvalidLayers until max publisher layer is available + // should still have target at buffer.InvalidLayer until max publisher layer is available expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) // muted should not consume any bandwidth f.Mute(true) disable(f) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) @@ -179,14 +186,14 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.PubMute(true) disable(f) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonPubMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonPubMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) @@ -195,219 +202,216 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.PubMute(false) // when parked layers valid, should stay there - f.parkedLayers = VideoLayers{ + f.vls.SetParked(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } + }) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: emptyBitrates, - targetLayers: f.parkedLayers, - requestLayerSpatial: f.parkedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: emptyBitrates, + TargetLayer: f.vls.GetParked(), + RequestLayerSpatial: f.vls.GetParked().Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, f.parkedLayers, f.TargetLayers()) - f.parkedLayers = InvalidLayers + require.Equal(t, f.vls.GetParked(), f.TargetLayer()) + f.vls.SetParked(buffer.InvalidLayer) // when max layers changes, target is opportunistic, but requested spatial layer should be at max - f.maxLayers = VideoLayers{Spatial: 1, Temporal: 3} + f.SetMaxTemporalLayerSeen(3) + f.vls.SetMax(buffer.VideoLayer{Spatial: 1, Temporal: 3}) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[1][3], - bandwidthDelta: bitrates[1][3], - bandwidthNeeded: bitrates[1][3], - bitrates: bitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: f.maxLayers.Spatial, - maxLayers: f.maxLayers, - distanceToDesired: -1, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[1][3], + BandwidthDelta: bitrates[1][3] - bitrates[0][1], + BandwidthNeeded: bitrates[1][3], + Bitrates: bitrates, + TargetLayer: buffer.DefaultMaxLayer, + RequestLayerSpatial: f.vls.GetMax().Spatial, + MaxLayer: f.vls.GetMax(), + DistanceToDesired: -1, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) // reset max layers for rest of the tests below - f.maxLayers = DefaultMaxLayers + f.vls.SetMax(buffer.DefaultMaxLayer) // when feed is dry and current is not valid, should set up for opportunistic forwarding // NOTE: feed is dry due to availableLayers = nil, some valid bitrates may be passed in here for testing purposes only disable(f) - expectedTargetLayers := VideoLayers{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 2, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1] - bitrates[1][3], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1] - bitrates[1][3], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.5, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) - f.targetLayers = VideoLayers{Spatial: 0, Temporal: 0} // set to valid to trigger paths in tests below - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 3} // set to valid to trigger paths in tests below + f.vls.SetTarget(buffer.VideoLayer{Spatial: 0, Temporal: 0}) // set to valid to trigger paths in tests below + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 3}) // set to valid to trigger paths in tests below // when feed is dry and current is valid, should stay at current - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 3, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.75, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) - f.currentLayers = InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayer) // opportunistic target if feed is not dry and current is not valid, i. e. not forwarding expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: buffer.DefaultMaxLayer, + RequestLayerSpatial: 1, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) + + // opportunistic target if feed is dry and current is not valid, i. e. not forwarding + expectedResult = VideoAllocation{ + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: 0, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: buffer.DefaultMaxLayer, + RequestLayerSpatial: 2, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.5, + } + result = f.AllocateOptimal(nil, bitrates, true) + require.Equal(t, expectedResult, result) + require.Equal(t, expectedResult, f.lastAllocation) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) // if feed is not dry and current is not locked, should be opportunistic (with and without overshoot) - f.targetLayers = InvalidLayers + f.vls.SetTarget(buffer.InvalidLayer) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayer: buffer.DefaultMaxLayer, + RequestLayerSpatial: 1, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -1.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.targetLayers = InvalidLayers - expectedTargetLayers = VideoLayers{ + f.vls.SetTarget(buffer.InvalidLayer) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 2, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: 1, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - // switches to highest available if feed is not dry and current is valid and current is not available - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - expectedTargetLayers = VideoLayers{ + // switches request layer to highest available if feed is not dry and current is valid and current is not available + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: 0, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 1, - maxLayers: DefaultMaxLayers, - distanceToDesired: 1, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: 0, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: 1, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) // stays the same if feed is not dry and current is valid, available and locked - f.maxLayers = VideoLayers{Spatial: 0, Temporal: 1} - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - f.requestLayerSpatial = 0 - expectedTargetLayers = VideoLayers{ + f.vls.SetMax(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + f.vls.SetRequestSpatial(0) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 0, - maxLayers: f.maxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: 0, + MaxLayer: f.vls.GetMax(), + DistanceToDesired: 0.0, } - result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) - require.Equal(t, expectedResult, result) - require.Equal(t, expectedResult, f.lastAllocation) - - // opportunistic if feed is not dry and current is valid, but request layer has changed - f.maxLayers = VideoLayers{Spatial: 2, Temporal: 1} - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - f.requestLayerSpatial = 0 - expectedTargetLayers = VideoLayers{ - Spatial: 2, - Temporal: 3, - } - expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 2, - maxLayers: f.maxLayers, - distanceToDesired: 0, - } - result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) + result = f.AllocateOptimal([]int32{0}, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) } func TestForwarderProvisionalAllocate(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -415,71 +419,71 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + usedBitrate := f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, bitrates[0][0], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, true, false) require.Equal(t, bitrates[2][3]-bitrates[0][0], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 3}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 3}, true, false) require.Equal(t, bitrates[0][3]-bitrates[2][3], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 2}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 2}, true, false) require.Equal(t, bitrates[1][2]-bitrates[0][3], usedBitrate) // available not enough to reach (2, 2), allocating at (2, 2) should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][2]-bitrates[1][2]-1, VideoLayers{Spatial: 2, Temporal: 2}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][2]-bitrates[1][2]-1, buffer.VideoLayer{Spatial: 2, Temporal: 2}, true, false) require.Equal(t, int64(0), usedBitrate) // committing should set target to (1, 2) - expectedTargetLayers := VideoLayers{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 1, Temporal: 2, } expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: bitrates[1][2], - bandwidthDelta: bitrates[1][2], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 5, + IsDeficient: true, + BandwidthRequested: bitrates[1][2], + BandwidthDelta: bitrates[1][2], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 1.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // when nothing fits and pausing disallowed, should allocate (0, 0) - f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) - usedBitrate = f.ProvisionalAllocate(0, VideoLayers{Spatial: 0, Temporal: 0}, false, false) + f.vls.SetTarget(buffer.InvalidLayer) + f.ProvisionalAllocatePrepare(nil, bitrates) + usedBitrate = f.ProvisionalAllocate(0, buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) // committing should set target to (0, 0) - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: bitrates[0][0], - bandwidthDelta: bitrates[0][0] - bitrates[1][2], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 11, + IsDeficient: true, + BandwidthRequested: bitrates[0][0], + BandwidthDelta: bitrates[0][0] - bitrates[1][2], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 2.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Test allowOvershoot. @@ -492,41 +496,41 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, bitrates[2][3], usedBitrate) // overshoot should succeed - this should win as this is lesser overshoot - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, bitrates[1][3]-bitrates[2][3], usedBitrate) // committing should set target to (1, 3) - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } - expectedMaxLayers := VideoLayers{ + expectedMaxLayer := buffer.VideoLayer{ Spatial: 0, Temporal: 3, } expectedResult = VideoAllocation{ - bandwidthRequested: bitrates[1][3], - bandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwith requested - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -4, + BandwidthRequested: bitrates[1][3], + BandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwidth requested + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: -1.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Even if overshoot is allowed, but if higher layers do not have bit rates, should continue with current layer. @@ -537,80 +541,80 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 2}) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations should not succeed because the feed is dry - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // committing should set target to (0, 2), i. e. leave it at current for opportunistic forwarding - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 2, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: bitrates[0][2], - bandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwith requested - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: bitrates[0][2], + BandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwidth requested + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: 1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Same case as above, but current is above max, so target should go to invalid // - f.currentLayers = VideoLayers{Spatial: 1, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 1, Temporal: 2}) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations below should not succeed because the feed is dry - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: 1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) - require.Equal(t, InvalidLayers, f.CurrentLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) } func TestForwarderProvisionalAllocateMute(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -619,36 +623,37 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { } f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + usedBitrate := f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, int64(0), usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 2}, true, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 2}, true, true) require.Equal(t, int64(0), usedBitrate) - // committing should set target to InvalidLayers as track is muted + // committing should set target to buffer.InvalidLayer as track is muted expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -656,71 +661,71 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should give back layer (0, 0) + // from scratch (buffer.InvalidLayer) should give back layer (0, 0) expectedTransition := VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 0, Temporal: 0}, - bandwidthDelta: 1, + From: buffer.InvalidLayer, + To: buffer.VideoLayer{Spatial: 0, Temporal: 0}, + BandwidthDelta: 1, } transition := f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) // committing should set target to (0, 0) - expectedLayers := VideoLayers{Spatial: 0, Temporal: 0} + expectedLayers := buffer.VideoLayer{Spatial: 0, Temporal: 0} expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: 1, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 9, + IsDeficient: true, + BandwidthRequested: 1, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 2.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // a higher target that is already streaming, just maintain it - targetLayers := VideoLayers{Spatial: 2, Temporal: 1} - f.targetLayers = targetLayers - f.lastAllocation.bandwidthRequested = 10 + targetLayer := buffer.VideoLayer{Spatial: 2, Temporal: 1} + f.vls.SetTarget(targetLayer) + f.lastAllocation.BandwidthRequested = 10 expectedTransition = VideoTransition{ - from: targetLayers, - to: targetLayers, - bandwidthDelta: 0, + From: targetLayer, + To: targetLayer, + BandwidthDelta: 0, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) // committing should set target to (2, 1) - expectedLayers = VideoLayers{Spatial: 2, Temporal: 1} + expectedLayers = buffer.VideoLayer{Spatial: 2, Temporal: 1} expectedResult = VideoAllocation{ - bandwidthRequested: 10, - bandwidthDelta: 0, - bitrates: bitrates, - bandwidthNeeded: bitrates[2][1], - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + BandwidthRequested: 10, + BandwidthDelta: 0, + Bitrates: bitrates, + BandwidthNeeded: bitrates[2][1], + TargetLayer: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // from a target that has become unavailable, should switch to lower available layer - targetLayers = VideoLayers{Spatial: 2, Temporal: 2} - f.targetLayers = targetLayers + targetLayer = buffer.VideoLayer{Spatial: 2, Temporal: 2} + f.vls.SetTarget(targetLayer) expectedTransition = VideoTransition{ - from: targetLayers, - to: VideoLayers{Spatial: 2, Temporal: 1}, - bandwidthDelta: 0, + From: targetLayer, + To: buffer.VideoLayer{Spatial: 2, Temporal: 1}, + BandwidthDelta: 0, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) @@ -729,13 +734,13 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // mute f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - // mute should send target to InvalidLayers + // mute should send target to buffer.InvalidLayer expectedTransition = VideoTransition{ - from: VideoLayers{Spatial: 2, Temporal: 1}, - to: InvalidLayers, - bandwidthDelta: -10, + From: buffer.VideoLayer{Spatial: 2, Temporal: 1}, + To: buffer.InvalidLayer, + BandwidthDelta: -10, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) @@ -754,37 +759,37 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.vls.SetTarget(buffer.InvalidLayer) + f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should go to a layer past maximum as overshoot is allowed + // from scratch (buffer.InvalidLayer) should go to a layer past maximum as overshoot is allowed expectedTransition = VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 1, Temporal: 0}, - bandwidthDelta: 5, + From: buffer.InvalidLayer, + To: buffer.VideoLayer{Spatial: 1, Temporal: 0}, + BandwidthDelta: 5, } transition = f.ProvisionalAllocateGetCooperativeTransition(true) require.Equal(t, expectedTransition, transition) // committing should set target to (1, 0) - expectedLayers = VideoLayers{Spatial: 1, Temporal: 0} - expectedMaxLayers := VideoLayers{Spatial: 0, Temporal: DefaultMaxLayerTemporal} + expectedLayers = buffer.VideoLayer{Spatial: 1, Temporal: 0} + expectedMaxLayer := buffer.VideoLayer{Spatial: 0, Temporal: buffer.DefaultMaxLayerTemporal} expectedResult = VideoAllocation{ - bandwidthRequested: 5, - bandwidthDelta: 5, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -1, + BandwidthRequested: 5, + BandwidthDelta: 5, + Bitrates: bitrates, + TargetLayer: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: -1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // - // Test continuting at current layers when feed is dry + // Test continuing at current layers when feed is dry // bitrates = Bitrates{ {0, 0, 0, 0}, @@ -792,56 +797,56 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} - f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 2}) + f.vls.SetTarget(buffer.InvalidLayer) + f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should go to current layer - // NOTE: targetLayer is set to InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen + // from scratch (buffer.InvalidLayer) should go to current layer + // NOTE: targetLayer is set to buffer.InvalidLayer for testing, but in practice current layers valid and target layers invalid should not happen expectedTransition = VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 0, Temporal: 2}, - bandwidthDelta: -5, // 5 was the bandwidth needed for the last allocation + From: buffer.InvalidLayer, + To: buffer.VideoLayer{Spatial: 0, Temporal: 2}, + BandwidthDelta: -5, // 5 was the bandwidth needed for the last allocation } transition = f.ProvisionalAllocateGetCooperativeTransition(true) require.Equal(t, expectedTransition, transition) // committing should set target to (0, 2) - expectedLayers = VideoLayers{Spatial: 0, Temporal: 2} + expectedLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} expectedResult = VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: -5, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0, + BandwidthRequested: 0, + BandwidthDelta: -5, + Bitrates: bitrates, + TargetLayer: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // committing should set target to current layers to enable opportunistic forwarding expectedResult = VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayer: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) } func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -849,14 +854,14 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) - f.targetLayers = VideoLayers{Spatial: 2, Temporal: 2} - f.lastAllocation.bandwidthRequested = bitrates[2][2] + f.vls.SetTarget(buffer.VideoLayer{Spatial: 2, Temporal: 2}) + f.lastAllocation.BandwidthRequested = bitrates[2][2] expectedTransition := VideoTransition{ - from: f.targetLayers, - to: VideoLayers{Spatial: 2, Temporal: 0}, - bandwidthDelta: 2, + From: f.TargetLayer(), + To: buffer.VideoLayer{Spatial: 2, Temporal: 0}, + BandwidthDelta: -2, } transition := f.ProvisionalAllocateGetBestWeightedTransition() require.Equal(t, expectedTransition, transition) @@ -864,9 +869,9 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { func TestForwarderAllocateNextHigher(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) emptyBitrates := Bitrates{} bitrates := Bitrates{ @@ -875,182 +880,181 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {0, 7, 0, 0}, } - result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted := f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) // no layer for audio require.False(t, boosted) f = newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) // when not in deficient state, does not boost - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) // if layers have not caught up, should not allocate next layer even if deficient - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 0, - } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + }) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) - f.lastAllocation.isDeficient = true - f.currentLayers = VideoLayers{ + f.lastAllocation.IsDeficient = true + f.vls.SetCurrent(buffer.VideoLayer{ Spatial: 0, Temporal: 0, - } + }) // move from (0, 0) -> (0, 1), i.e. a higher temporal layer is available in the same spatial layer - expectedTargetLayers := VideoLayers{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 0, Temporal: 1, } expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: 3, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 3, + IsDeficient: true, + BandwidthRequested: 3, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 2.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // empty bitrates cannot increase layer, i. e. last allocation is left unchanged - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, emptyBitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, emptyBitrates, false) require.Equal(t, expectedResult, result) require.False(t, boosted) // move from (0, 1) -> (1, 0), i.e. a higher spatial layer is available - f.currentLayers.Temporal = 1 - expectedTargetLayers = VideoLayers{ + f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 1}) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 4, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2, + IsDeficient: true, + BandwidthRequested: 4, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 1.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // next higher, move from (1, 0) -> (1, 3), still deficient though - f.currentLayers.Spatial = 1 - f.currentLayers.Temporal = 0 - expectedTargetLayers = VideoLayers{ + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 1, Temporal: 0}) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 5, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 1, + IsDeficient: true, + BandwidthRequested: 5, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0.5, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // next higher, move from (1, 3) -> (2, 1), optimal allocation - f.currentLayers.Temporal = 3 - expectedTargetLayers = VideoLayers{ + f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 3}) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 2, Temporal: 1, } expectedResult = VideoAllocation{ - bandwidthRequested: 7, - bandwidthDelta: 2, - bitrates: bitrates, - bandwidthNeeded: bitrates[2][1], - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + BandwidthRequested: 7, + BandwidthDelta: 2, + Bitrates: bitrates, + BandwidthNeeded: bitrates[2][1], + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // ask again, should return not boosted as there is no room to go higher - f.currentLayers.Spatial = 2 - f.currentLayers.Temporal = 1 - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 2, Temporal: 1}) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.False(t, boosted) // turn off everything, allocating next layer should result in streaming lowest layers disable(f) - f.lastAllocation.isDeficient = true - f.lastAllocation.bandwidthRequested = 0 + f.lastAllocation.IsDeficient = true + f.lastAllocation.BandwidthRequested = 0 - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 2, - bandwidthDelta: 2, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 4, + IsDeficient: true, + BandwidthRequested: 2, + BandwidthDelta: 2, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // no new available capacity cannot bump up layer expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 2, - bandwidthDelta: 2, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 4, + IsDeficient: true, + BandwidthRequested: 2, + BandwidthDelta: 2, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(0, bitrates, false) + result, boosted = f.AllocateNextHigher(0, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.False(t, boosted) // test allowOvershoot @@ -1062,39 +1066,39 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {9, 10, 11, 12}, } - f.currentLayers = f.targetLayers + f.vls.SetCurrent(f.vls.GetTarget()) - expectedTargetLayers = VideoLayers{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } - expectedMaxLayers := VideoLayers{ + expectedMaxLayer := buffer.VideoLayer{ Spatial: 0, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - bandwidthRequested: bitrates[1][0], - bandwidthDelta: bitrates[1][0], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -1, + BandwidthRequested: bitrates[1][0], + BandwidthDelta: bitrates[1][0], + Bitrates: bitrates, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, + DistanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity - result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, bitrates, true) + result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) } func TestForwarderPause(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -1102,34 +1106,34 @@ func TestForwarderPause(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) - f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + f.ProvisionalAllocatePrepare(nil, bitrates) + f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) // should have set target at (0, 0) f.ProvisionalAllocateCommit() expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonBandwidth, - isDeficient: true, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[0][0], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 3, + PauseReason: VideoPauseReasonBandwidth, + IsDeficient: true, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[0][0], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 3.75, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderPauseMute(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -1137,26 +1141,26 @@ func TestForwarderPauseMute(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) - f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, true) + f.ProvisionalAllocatePrepare(nil, bitrates) + f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, true) // should have set target at (0, 0) f.ProvisionalAllocateCommit() f.Mute(true) expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[0][0], - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[0][0], + Bitrates: bitrates, + TargetLayer: buffer.InvalidLayer, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: 0, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderGetTranslationParamsMuted(t *testing.T) { @@ -1223,8 +1227,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - shouldDrop: true, - isDroppingRelevant: true, + shouldDrop: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1334,21 +1337,22 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, + SetMarker: true, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) @@ -1361,10 +1365,10 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.Equal(t, expectedTP, *actualTP) // although target layer matches, not a key frame, so should drop - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } + }) expectedTP = TranslationParams{ shouldDrop: true, } @@ -1372,48 +1376,50 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedTP, *actualTP) - // should lock onto packet (target layer and key frame) + // should lock onto packet (key frame) vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, - isSwitchingToTargetLayer: true, + isSwitching: true, + isResuming: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23333, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, + marker: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1424,6 +1430,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // send a duplicate, should be dropped expectedTP = TranslationParams{ shouldDrop: true, + marker: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1438,8 +1445,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ - shouldDrop: true, - isDroppingRelevant: true, + shouldDrop: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1467,35 +1473,36 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23334, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) require.Equal(t, expectedTP, *actualTP) - // temporal layer higher than target, should be dropped + // temporal layer matching target, should be forwarded params = &testutils.TestExtPacketParams{ SequenceNumber: 23336, Timestamp: 0xabcdef, @@ -1503,19 +1510,72 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + S: true, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + expectedTP = TranslationParams{ + rtp: &TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + sequenceNumber: 23335, + timestamp: 0xabcdef, + }, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, *actualTP) + + // temporal layer higher than target, should be dropped + params = &testutils.TestExtPacketParams{ + SequenceNumber: 23337, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + } + vp8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 2, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ @@ -1527,50 +1587,51 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // RTP sequence number and VP8 picture id should be contiguous after dropping higher temporal layer picture params = &testutils.TestExtPacketParams{ - SequenceNumber: 23337, + SequenceNumber: 23338, Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, - sequenceNumber: 23335, + sequenceNumber: 23336, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1578,7 +1639,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // padding only packet after a gap should be forwarded params = &testutils.TestExtPacketParams{ - SequenceNumber: 23339, + SequenceNumber: 23340, Timestamp: 0xabcdef, SSRC: 0x12345678, } @@ -1587,7 +1648,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingGap, - sequenceNumber: 23337, + sequenceNumber: 23338, timestamp: 0xabcdef, }, } @@ -1597,7 +1658,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // out-of-order should be forwarded using cache, even if it is padding only params = &testutils.TestExtPacketParams{ - SequenceNumber: 23338, + SequenceNumber: 23339, Timestamp: 0xabcdef, SSRC: 0x12345678, } @@ -1606,7 +1667,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, - sequenceNumber: 23336, + sequenceNumber: 23337, timestamp: 0xabcdef, }, } @@ -1616,10 +1677,10 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // switching SSRC (happens for new layer or new track source) // should lock onto the new source, but sequence number should be contiguous - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 1, Temporal: 1, - } + }) params = &testutils.TestExtPacketParams{ SequenceNumber: 123, @@ -1628,47 +1689,47 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 45, - MBit: false, - TL0PICIDXPresent: 1, - TL0PICIDX: 12, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 30, - HeaderSize: 5, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: false, + PictureID: 45, + L: true, + TL0PICIDX: 12, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 30, + HeaderSize: 5, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13470, + L: true, + TL0PICIDX: 235, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, - isSwitchingToTargetLayer: true, + isSwitching: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, - sequenceNumber: 23338, + sequenceNumber: 23339, timestamp: 0xabcdf0, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 235, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 1) require.NoError(t, err) @@ -1686,27 +1747,27 @@ func TestForwardGetSnTsForPadding(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1715,7 +1776,7 @@ func TestForwardGetSnTsForPadding(t *testing.T) { disable(f) // should get back frame end needed as the last packet did not have RTP marker set - snts, err := f.GetSnTsForPadding(5) + snts, err := f.GetSnTsForPadding(5, false) require.NoError(t, err) numPadding := 5 @@ -1731,7 +1792,7 @@ func TestForwardGetSnTsForPadding(t *testing.T) { require.Equal(t, sntsExpected, snts) // now that there is a marker, timestamp should jump on first padding when asked again - snts, err = f.GetSnTsForPadding(numPadding) + snts, err = f.GetSnTsForPadding(numPadding, false) require.NoError(t, err) for i := 0; i < numPadding; i++ { @@ -1753,27 +1814,27 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1790,9 +1851,15 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { frameRate := uint32(30) var sntsExpected = make([]SnTs, numPadding) for i := 0; i < numPadding; i++ { + // first blank frame should have same timestamp as last frame as end frame is synthesized + ts := params.Timestamp + if i != 0 { + // +1 here due to expected time stamp bumpint by at least one so that time stamp is always moving ahead + ts = params.Timestamp + 1 + ((uint32(i)*clockRate)+frameRate-1)/frameRate + } sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(i) + 1, - timestamp: params.Timestamp + (uint32(i)*clockRate)/frameRate, + timestamp: ts, } } require.Equal(t, sntsExpected, snts) @@ -1804,7 +1871,8 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(len(snts)) + uint16(i) + 1, - timestamp: snts[len(snts)-1].timestamp + (uint32(i+1)*clockRate)/frameRate, + // +1 here due to expected time stamp bumpint by at least one so that time stamp is always moving ahead + timestamp: snts[len(snts)-1].timestamp + 1 + ((uint32(i+1)*clockRate)+frameRate-1)/frameRate, } } snts, frameEndNeeded, err = f.GetSnTsForBlankFrames(30, numBlankFrames) @@ -1823,66 +1891,72 @@ func TestForwardGetPaddingVP8(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) // getting padding with frame end needed, should repeat the last picture id expectedVP8 := buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - blankVP8 := f.GetPaddingVP8(true) - require.Equal(t, expectedVP8, *blankVP8) + blankVP8, err := f.GetPadding(true) + require.NoError(t, err) + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankVP8) // getting padding with no frame end needed, should get next picture id expectedVP8 = buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, } - blankVP8 = f.GetPaddingVP8(false) - require.Equal(t, expectedVP8, *blankVP8) + blankVP8, err = f.GetPadding(false) + require.NoError(t, err) + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankVP8) } diff --git a/pkg/sfu/helpers.go b/pkg/sfu/helpers.go index b6dcafd9f..1f4101910 100644 --- a/pkg/sfu/helpers.go +++ b/pkg/sfu/helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/pacer/base.go b/pkg/sfu/pacer/base.go new file mode 100644 index 000000000..83e0efc43 --- /dev/null +++ b/pkg/sfu/pacer/base.go @@ -0,0 +1,104 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "errors" + "io" + "time" + + "github.com/livekit/protocol/logger" + "github.com/pion/rtp" +) + +type Base struct { + logger logger.Logger + + packetTime *PacketTime +} + +func NewBase(logger logger.Logger) *Base { + return &Base{ + logger: logger, + packetTime: NewPacketTime(), + } +} + +func (b *Base) SetInterval(_interval time.Duration) { +} + +func (b *Base) SetBitrate(_bitrate int) { +} + +func (b *Base) SendPacket(p *Packet) (int, error) { + var sendingAt time.Time + var err error + defer func() { + if p.OnSent != nil { + p.OnSent(p.Metadata, p.Header, len(p.Payload), sendingAt, err) + } + }() + + sendingAt, err = b.writeRTPHeaderExtensions(p) + if err != nil { + b.logger.Errorw("writing rtp header extensions err", err) + return 0, err + } + + var written int + written, err = p.WriteStream.WriteRTP(p.Header, p.Payload) + if err != nil { + if !errors.Is(err, io.ErrClosedPipe) { + b.logger.Errorw("write rtp packet failed", err) + } + return 0, err + } + + return written, nil +} + +// writes RTP header extensions of track +func (b *Base) writeRTPHeaderExtensions(p *Packet) (time.Time, error) { + // clear out extensions that may have been in the forwarded header + p.Header.Extension = false + p.Header.ExtensionProfile = 0 + p.Header.Extensions = []rtp.Extension{} + + for _, ext := range p.Extensions { + if ext.ID == 0 || len(ext.Payload) == 0 { + continue + } + + p.Header.SetExtension(ext.ID, ext.Payload) + } + + sendingAt := b.packetTime.Get() + if p.AbsSendTimeExtID != 0 { + sendTime := rtp.NewAbsSendTimeExtension(sendingAt) + b, err := sendTime.Marshal() + if err != nil { + return time.Time{}, err + } + + err = p.Header.SetExtension(p.AbsSendTimeExtID, b) + if err != nil { + return time.Time{}, err + } + } + + return sendingAt, nil +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/leaky_bucket.go b/pkg/sfu/pacer/leaky_bucket.go new file mode 100644 index 000000000..9ac2a1350 --- /dev/null +++ b/pkg/sfu/pacer/leaky_bucket.go @@ -0,0 +1,151 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "sync" + "time" + + "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" +) + +const ( + maxOvershootFactor = 2.0 +) + +type LeakyBucket struct { + *Base + + logger logger.Logger + + lock sync.RWMutex + packets deque.Deque[Packet] + interval time.Duration + bitrate int + isStopped bool +} + +func NewLeakyBucket(logger logger.Logger, interval time.Duration, bitrate int) *LeakyBucket { + l := &LeakyBucket{ + Base: NewBase(logger), + logger: logger, + interval: interval, + bitrate: bitrate, + } + l.packets.SetMinCapacity(9) + + go l.sendWorker() + return l +} + +func (l *LeakyBucket) SetInterval(interval time.Duration) { + l.lock.Lock() + defer l.lock.Unlock() + + l.interval = interval +} + +func (l *LeakyBucket) SetBitrate(bitrate int) { + l.lock.Lock() + defer l.lock.Unlock() + + l.bitrate = bitrate +} + +func (l *LeakyBucket) Stop() { + l.lock.Lock() + if l.isStopped { + l.lock.Unlock() + return + } + + l.isStopped = true + l.lock.Unlock() +} + +func (l *LeakyBucket) Enqueue(p Packet) { + l.lock.Lock() + defer l.lock.Unlock() + + if !l.isStopped { + l.packets.PushBack(p) + } +} + +func (l *LeakyBucket) sendWorker() { + l.lock.RLock() + interval := l.interval + bitrate := l.bitrate + l.lock.RUnlock() + + timer := time.NewTimer(interval) + overage := 0 + + for { + <-timer.C + + l.lock.RLock() + interval = l.interval + bitrate = l.bitrate + l.lock.RUnlock() + + // calculate number of bytes that can be sent in this interval + // adjusting for overage. + intervalBytes := int(interval.Seconds() * float64(bitrate) / 8.0) + maxOvershootBytes := int(float64(intervalBytes) * maxOvershootFactor) + toSendBytes := intervalBytes - overage + if toSendBytes < 0 { + // too much overage, wait for next interval + overage = -toSendBytes + timer.Reset(interval) + continue + } + + // do not allow too much overshoot in an interval + if toSendBytes > maxOvershootBytes { + toSendBytes = maxOvershootBytes + } + + for { + l.lock.Lock() + if l.isStopped { + l.lock.Unlock() + return + } + + if l.packets.Len() == 0 { + l.lock.Unlock() + // allow overshoot in next interval with shortage in this interval + overage = -toSendBytes + timer.Reset(interval) + break + } + p := l.packets.PopFront() + l.lock.Unlock() + + written, _ := l.Base.SendPacket(&p) + toSendBytes -= written + if toSendBytes < 0 { + // overage, wait for next interval + overage = -toSendBytes + timer.Reset(interval) + break + } + } + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/no_queue.go b/pkg/sfu/pacer/no_queue.go new file mode 100644 index 000000000..927236394 --- /dev/null +++ b/pkg/sfu/pacer/no_queue.go @@ -0,0 +1,94 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "sync" + + "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" +) + +type NoQueue struct { + *Base + + logger logger.Logger + + lock sync.RWMutex + packets deque.Deque[Packet] + wake chan struct{} + isStopped bool +} + +func NewNoQueue(logger logger.Logger) *NoQueue { + n := &NoQueue{ + Base: NewBase(logger), + logger: logger, + wake: make(chan struct{}, 1), + } + n.packets.SetMinCapacity(9) + + go n.sendWorker() + return n +} + +func (n *NoQueue) Stop() { + n.lock.Lock() + if n.isStopped { + n.lock.Unlock() + return + } + + close(n.wake) + n.isStopped = true + n.lock.Unlock() +} + +func (n *NoQueue) Enqueue(p Packet) { + n.lock.Lock() + defer n.lock.Unlock() + + n.packets.PushBack(p) + if n.packets.Len() == 1 && !n.isStopped { + select { + case n.wake <- struct{}{}: + default: + } + } +} + +func (n *NoQueue) sendWorker() { + for { + <-n.wake + for { + n.lock.Lock() + if n.isStopped { + n.lock.Unlock() + return + } + + if n.packets.Len() == 0 { + n.lock.Unlock() + break + } + p := n.packets.PopFront() + n.lock.Unlock() + + n.Base.SendPacket(&p) + } + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/pacer.go b/pkg/sfu/pacer/pacer.go new file mode 100644 index 000000000..48b20efea --- /dev/null +++ b/pkg/sfu/pacer/pacer.go @@ -0,0 +1,48 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3" +) + +type ExtensionData struct { + ID uint8 + Payload []byte +} + +type Packet struct { + Header *rtp.Header + Extensions []ExtensionData + Payload []byte + AbsSendTimeExtID uint8 + TransportWideExtID uint8 + WriteStream webrtc.TrackLocalWriter + Metadata interface{} + OnSent func(md interface{}, sentHeader *rtp.Header, payloadSize int, sentTime time.Time, sendError error) +} + +type Pacer interface { + Enqueue(p Packet) + Stop() + + SetInterval(interval time.Duration) + SetBitrate(bitrate int) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/packet_time.go b/pkg/sfu/pacer/packet_time.go new file mode 100644 index 000000000..eac3866b4 --- /dev/null +++ b/pkg/sfu/pacer/packet_time.go @@ -0,0 +1,36 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "time" +) + +type PacketTime struct { + baseTime time.Time +} + +func NewPacketTime() *PacketTime { + return &PacketTime{ + baseTime: time.Now(), + } +} + +func (p *PacketTime) Get() time.Time { + // construct current time based on monotonic clock + return p.baseTime.Add(time.Since(p.baseTime)) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/pass_through.go b/pkg/sfu/pacer/pass_through.go new file mode 100644 index 000000000..8c33d808f --- /dev/null +++ b/pkg/sfu/pacer/pass_through.go @@ -0,0 +1,38 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pacer + +import ( + "github.com/livekit/protocol/logger" +) + +type PassThrough struct { + *Base +} + +func NewPassThrough(logger logger.Logger) *PassThrough { + return &PassThrough{ + Base: NewBase(logger), + } +} + +func (p *PassThrough) Stop() { +} + +func (p *PassThrough) Enqueue(pkt Packet) { + p.Base.SendPacket(&pkt) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index ce99a6ef8..675df3e4c 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -20,6 +34,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" ) var ( @@ -30,7 +45,7 @@ var ( type AudioLevelHandle func(level uint8, duration uint32) -type Bitrates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]int64 +type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 // TrackReceiver defines an interface receive media from remote peer type TrackReceiver interface { @@ -66,7 +81,7 @@ type TrackReceiver interface { GetFrameRates() [][]float32 GetTemporalLayerFpsForSpatial(layer int32) (bool, []float32) - GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt + GetCalculatedClockRate(layer int32) uint32 GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) } @@ -95,11 +110,11 @@ type WebRTCReceiver struct { twcc *twcc.Responder bufferMu sync.RWMutex - buffers [DefaultMaxLayerSpatial + 1]*buffer.Buffer + buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer rtt uint32 upTrackMu sync.RWMutex - upTracks [DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote + upTracks [buffer.DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote lbThreshold int @@ -208,14 +223,21 @@ func NewWebRTCReceiver( MimeType: w.codec.MimeType, IsFECEnabled: strings.EqualFold(w.codec.MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "fec"), GetDeltaStats: w.getDeltaStats, - Logger: w.logger, + Logger: w.logger.WithValues("direction", "up"), }) w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { if w.onStatsUpdate != nil { w.onStatsUpdate(w, stat) } }) - w.connectionStats.Start(w.trackInfo, time.Now()) + w.connectionStats.Start(w.trackInfo) + + for _, ext := range receiver.GetParameters().HeaderExtensions { + if ext.URI == dd.ExtensionUrl { + w.streamTrackerManager.AddDependencyDescriptorTrackers() + break + } + } return w } @@ -298,7 +320,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff } layer := int32(0) - if w.Kind() == webrtc.RTPCodecTypeVideo { + if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo) } buff.SetLogger(w.logger.WithValues("layer", layer)) @@ -311,7 +333,8 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff }) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func(srData *buffer.RTCPSenderReportData) { - w.streamTrackerManager.SetRTCPSenderReportDataExt(layer, buff.GetSenderReportDataExt()) + srFirst, srNewest := buff.GetSenderReportData() + w.streamTrackerManager.SetRTCPSenderReportData(layer, srFirst, srNewest) w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, layer, srData) @@ -367,7 +390,7 @@ func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) { } w.bufferMu.RUnlock() - w.connectionStats.UpdateMute(paused, time.Now()) + w.connectionStats.UpdateMute(paused) } func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { @@ -381,65 +404,55 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { track.TrackInfoAvailable() track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer()) + track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen()) w.downTrackSpreader.Store(track) return nil } -func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { - if w.Kind() == webrtc.RTPCodecTypeAudio || w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - // screen share tracks have highly variable bitrate, do not use bit rate based quality for those - return - } - - expectedBitrate := int64(0) - for _, vl := range w.trackInfo.Layers { - l := buffer.VideoQualityToSpatialLayer(vl.Quality, w.trackInfo) - if l <= layer { - expectedBitrate += int64(vl.Bitrate) - } - } - - w.connectionStats.AddBitrateTransition(expectedBitrate, time.Now()) -} - func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) - w.notifyMaxExpectedLayer(layer) + + if layer == buffer.InvalidLayerSpatial { + w.connectionStats.UpdateLayerMute(true) + } else { + w.connectionStats.UpdateLayerMute(false) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + } } // StreamTrackerManagerListener.OnAvailableLayersChanged func (w *WebRTCReceiver) OnAvailableLayersChanged() { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackLayersChange() - } + }) + + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackBitrateAvailabilityChange() - } + }) } // StreamTrackerManagerListener.OnMaxPublishedLayerChanged func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) - } + }) - w.notifyMaxExpectedLayer(maxPublishedLayer) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) - } + }) - if w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) - } + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxAvailableLayerChanged @@ -459,9 +472,7 @@ func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitra dt.UpTrackBitrateReport(availableLayers, bitrates) } - if w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) - } + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) { @@ -495,7 +506,7 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) { } func (w *WebRTCReceiver) SendPLI(layer int32, force bool) { - // TODO : should send LRR (Layer Refresh Request) instead of PLI + // SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI buff := w.getBuffer(layer) if buff == nil { return @@ -516,10 +527,10 @@ func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer { } func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer { - // for svc codecs, use layer full quality instead. - // we only have buffer for full quality + // for svc codecs, use layer = 0 always. + // spatial layers are in-built and handled by single buffer if w.isSVC { - layer = int32(len(w.buffers)) - 1 + layer = 0 } if int(layer) >= len(w.buffers) { @@ -637,10 +648,10 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { return } - // svc packet, dispatch to correct tracker spatialTracker := tracker spatialLayer := layer if pkt.Spatial >= 0 { + // svc packet, dispatch to correct tracker spatialLayer = pkt.Spatial spatialTracker = w.streamTrackerManager.GetTracker(pkt.Spatial) if spatialTracker == nil { @@ -648,16 +659,6 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { } } - if spatialTracker != nil { - spatialTracker.Observe( - pkt.Temporal, - len(pkt.RawPacket), - len(pkt.Packet.Payload), - pkt.Packet.Marker, - pkt.Packet.Timestamp, - ) - } - w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.WriteRTP(pkt, spatialLayer) }) @@ -665,6 +666,17 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { if redPktWriter != nil { redPktWriter(pkt, spatialLayer) } + + if spatialTracker != nil { + spatialTracker.Observe( + pkt.Temporal, + len(pkt.RawPacket), + len(pkt.Packet.Payload), + pkt.Packet.Marker, + pkt.Packet.Timestamp, + pkt.DependencyDescriptor, + ) + } } } @@ -673,9 +685,7 @@ func (w *WebRTCReceiver) closeTracks() { w.connectionStats.Close() w.streamTrackerManager.Close() - for _, dt := range w.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks()) if w.onCloseHandler != nil { w.onCloseHandler() @@ -748,9 +758,9 @@ func (w *WebRTCReceiver) GetFrameRates() [][]float32 { w.bufferMu.RLock() defer w.bufferMu.RUnlock() - fps := make([][]float32, DefaultMaxLayerSpatial+1) + fps := make([][]float32, buffer.DefaultMaxLayerSpatial+1) for i := 0; i < len(fps); i++ { - fps[i] = make([]float32, DefaultMaxLayerTemporal+1) + fps[i] = make([]float32, buffer.DefaultMaxLayerTemporal+1) } if w.isSVC { @@ -758,7 +768,7 @@ func (w *WebRTCReceiver) GetFrameRates() [][]float32 { return nil } - for layer := int32(0); layer < DefaultMaxLayerSpatial+1; layer++ { + for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { isAvailable, fr := w.buffers[0].GetTemporalLayerFpsForSpatial(layer) if !isAvailable { // even if one layer does not have frame rate, not ready yet @@ -803,10 +813,24 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) (bool, []flo return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { - return w.streamTrackerManager.GetRTCPSenderReportDataExt(layer) +func (w *WebRTCReceiver) GetCalculatedClockRate(layer int32) uint32 { + return w.streamTrackerManager.GetCalculatedClockRate(layer) } func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } + +// closes all track senders in parallel, returns when all are closed +func closeTrackSenders(senders []TrackSender) { + wg := sync.WaitGroup{} + for _, dt := range senders { + dt := dt + wg.Add(1) + go func() { + defer wg.Done() + dt.Close() + }() + } + wg.Wait() +} diff --git a/pkg/sfu/receiver_test.go b/pkg/sfu/receiver_test.go index e451c3fe3..a1f475e2a 100644 --- a/pkg/sfu/receiver_test.go +++ b/pkg/sfu/receiver_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index 9789bb32e..eb0965627 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -98,9 +112,7 @@ func (r *RedPrimaryReceiver) CanClose() bool { func (r *RedPrimaryReceiver) Close() { r.closed.Store(true) - for _, dt := range r.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks()) } func (r *RedPrimaryReceiver) ReadRTP(buf []byte, layer uint8, sn uint16) (int, error) { diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index 0d2876cf9..57cf49e8b 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -95,9 +109,7 @@ func (r *RedReceiver) IsClosed() bool { func (r *RedReceiver) Close() { r.closed.Store(true) - for _, dt := range r.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks()) } func (r *RedReceiver) ReadRTP(buf []byte, layer uint8, sn uint16) (int, error) { diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 72d8d9a0e..2aae30182 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index d42a4fe5f..52faa8124 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -122,14 +136,8 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { if r.highestIncomingSN != extPkt.Packet.SequenceNumber { return } - r.snOffset += 1 + r.snOffset++ r.lastSN = extPkt.Packet.SequenceNumber - r.snOffset - - r.snOffsetsWritePtr = (r.snOffsetsWritePtr - 1) & SnOffsetCacheMask - r.snOffsetsOccupancy-- - if r.snOffsetsOccupancy < 0 { - r.logger.Warnw("sequence number offset cache is invalid", nil, "occupancy", r.snOffsetsOccupancy) - } } func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) { @@ -177,7 +185,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara // sequence number offset. if len(extPkt.Packet.Payload) == 0 { r.highestIncomingSN = extPkt.Packet.SequenceNumber - r.snOffset += 1 + r.snOffset++ return &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, @@ -230,7 +238,8 @@ func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 { return filtered } -func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool) ([]SnTs, error) { +func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, rtpTimestamp uint32) ([]SnTs, error) { + useLastTSForFirst := false tsOffset := 0 if !r.lastMarker { if !forceMarker { @@ -238,14 +247,25 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate } // if forcing frame end, use timestamp of latest received frame for the first one + useLastTSForFirst = true tsOffset = 1 } + lastTS := r.lastTS vals := make([]SnTs, num) for i := 0; i < num; i++ { vals[i].sequenceNumber = r.lastSN + uint16(i) + 1 if frameRate != 0 { - vals[i].timestamp = r.lastTS + uint32(i+1-tsOffset)*(clockRate/frameRate) + if useLastTSForFirst && i == 0 { + vals[i].timestamp = r.lastTS + } else { + ts := rtpTimestamp + ((uint32(i+1-tsOffset)*clockRate)+frameRate-1)/frameRate + if (ts-lastTS) == 0 || (ts-lastTS) > (1<<31) { + ts = lastTS + 1 + lastTS = ts + } + vals[i].timestamp = ts + } } else { vals[i].timestamp = r.lastTS } diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index faa68efe9..63a611a1f 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -27,25 +41,9 @@ func TestSetLastSnTs(t *testing.T) { require.NotNil(t, extPkt) r.SetLastSnTs(extPkt) - require.True(t, r.highestIncomingSN == 23332) - require.True(t, r.lastSN == 23333) - require.True(t, r.lastTS == 0xabcdef) - require.Equal(t, uint16(0), r.snOffset) - require.Equal(t, uint32(0), r.tsOffset) - - params = &testutils.TestExtPacketParams{ - SequenceNumber: 0, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - extPkt, err = testutils.GetTestExtPacket(params) - require.NoError(t, err) - require.NotNil(t, extPkt) - - r.SetLastSnTs(extPkt) - require.True(t, r.highestIncomingSN == 65535) - require.True(t, r.lastSN == 0) - require.True(t, r.lastTS == 0xabcdef) + require.Equal(t, uint16(23332), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) } @@ -68,9 +66,9 @@ func TestUpdateSnTsOffsets(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) r.UpdateSnTsOffsets(extPkt, 1, 1) - require.True(t, r.highestIncomingSN == 33332) - require.True(t, r.lastSN == 23333) - require.True(t, r.lastTS == 0xabcdef) + require.Equal(t, uint16(33332), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(9999), r.snOffset) require.Equal(t, uint32(0xffffffff), r.tsOffset) } @@ -82,15 +80,19 @@ func TestPacketDropped(t *testing.T) { SequenceNumber: 23333, Timestamp: 0xabcdef, SSRC: 0x12345678, + PayloadSize: 10, } extPkt, _ := testutils.GetTestExtPacket(params) r.SetLastSnTs(extPkt) - require.Equal(t, r.highestIncomingSN, uint16(23332)) - require.Equal(t, r.lastSN, uint16(23333)) - require.Equal(t, r.lastTS, uint32(0xabcdef)) + require.Equal(t, uint16(23332), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + require.Equal(t, 1, r.snOffsetsWritePtr) + // drop a non-head packet, should cause no change in internals params = &testutils.TestExtPacketParams{ SequenceNumber: 33333, @@ -99,8 +101,8 @@ func TestPacketDropped(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) r.PacketDropped(extPkt) - require.Equal(t, r.highestIncomingSN, uint16(23332)) - require.Equal(t, r.lastSN, uint16(23333)) + require.Equal(t, uint16(23333), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint16(0), r.snOffset) // drop a head packet and check offset increases @@ -108,12 +110,33 @@ func TestPacketDropped(t *testing.T) { SequenceNumber: 44444, Timestamp: 0xabcdef, SSRC: 0x12345678, + PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacket(params) - r.highestIncomingSN = 44444 + + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + snOffsetWritePtr := (44444 - 23333 + 1) & SnOffsetCacheMask + require.Equal(t, snOffsetWritePtr, r.snOffsetsWritePtr) + require.Equal(t, SnOffsetCacheSize, r.snOffsetsOccupancy) + r.PacketDropped(extPkt) require.Equal(t, r.lastSN, uint16(44443)) require.Equal(t, uint16(1), r.snOffset) + + params = &testutils.TestExtPacketParams{ + SequenceNumber: 44445, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + } + extPkt, _ = testutils.GetTestExtPacket(params) + + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + require.Equal(t, uint16(1), r.snOffsets[snOffsetWritePtr]) + snOffsetWritePtr = (snOffsetWritePtr + 1) & SnOffsetCacheMask + require.Equal(t, snOffsetWritePtr, r.snOffsetsWritePtr) + require.Equal(t, uint16(44444), r.lastSN) + require.Equal(t, uint16(1), r.snOffset) } func TestOutOfOrderSequenceNumber(t *testing.T) { @@ -207,8 +230,8 @@ func TestPaddingOnlyPacket(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 23333) - require.True(t, r.lastSN == 23333) + require.Equal(t, uint16(23333), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // padding only packet with a gap should not report an error @@ -228,8 +251,8 @@ func TestPaddingOnlyPacket(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 23335) - require.True(t, r.lastSN == 23334) + require.Equal(t, uint16(23335), r.highestIncomingSN) + require.Equal(t, uint16(23334), r.lastSN) require.Equal(t, uint16(1), r.snOffset) } @@ -266,8 +289,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 1) - require.True(t, r.lastSN == 1) + require.Equal(t, uint16(1), r.highestIncomingSN) + require.Equal(t, uint16(1), r.lastSN) require.Equal(t, uint16(0), r.snOffset) // ensure missing sequence numbers got recorded in cache @@ -294,8 +317,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 2) - require.True(t, r.lastSN == 1) + require.Equal(t, uint16(2), r.highestIncomingSN) + require.Equal(t, uint16(1), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // a packet with a gap should be adding to missing cache @@ -316,8 +339,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 4) - require.True(t, r.lastSN == 3) + require.Equal(t, uint16(4), r.highestIncomingSN) + require.Equal(t, uint16(3), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // another contiguous padding only packet should be dropped @@ -335,8 +358,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 5) - require.True(t, r.lastSN == 3) + require.Equal(t, uint16(5), r.highestIncomingSN) + require.Equal(t, uint16(3), r.lastSN) require.Equal(t, uint16(2), r.snOffset) // a packet with a gap should be adding to missing cache @@ -357,8 +380,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) // check the missing packets @@ -378,8 +401,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) params = &testutils.TestExtPacketParams{ @@ -398,8 +421,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) } @@ -416,7 +439,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { r.SetLastSnTs(extPkt) // getting padding without forcing marker should fail - _, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false) + _, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary) @@ -429,10 +452,10 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(i) + 1, - timestamp: params.Timestamp + (uint32(i)*clockRate)/frameRate, + timestamp: params.Timestamp + ((uint32(i)*clockRate)+frameRate-1)/frameRate, } } - snts, err := r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, true) + snts, err := r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, true, params.Timestamp) require.NoError(t, err) require.Equal(t, sntsExpected, snts) @@ -440,10 +463,10 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(len(snts)) + uint16(i) + 1, - timestamp: snts[len(snts)-1].timestamp + (uint32(i+1)*clockRate)/frameRate, + timestamp: snts[len(snts)-1].timestamp + ((uint32(i+1)*clockRate)+frameRate-1)/frameRate, } } - snts, err = r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, false) + snts, err = r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, false, snts[len(snts)-1].timestamp) require.NoError(t, err) require.Equal(t, sntsExpected, snts) } diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 0fb7693af..e0f89d2c7 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -6,13 +20,12 @@ import ( "time" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( defaultRtt = 70 ignoreRetransmission = 100 // Ignore packet retransmission after ignoreRetransmission milliseconds + maxAck = 3 ) func btoi(b bool) int { @@ -40,6 +53,8 @@ type packetMeta struct { // Modified timestamp for current associated // down track. timestamp uint32 + // Modified marker + marker bool // The last time this packet was nack requested. // Sometimes clients request the same packet more than once, so keep // track of the requested packets helps to avoid writing multiple times @@ -51,45 +66,11 @@ type packetMeta struct { // Spatial layer of packet layer int8 // Information that differs depending on the codec - misc uint64 + codecBytes []byte // Dependency Descriptor of packet ddBytes []byte } -func (p *packetMeta) packVP8(vp8 *buffer.VP8) { - p.misc = uint64(vp8.FirstByte)<<56 | - uint64(vp8.PictureIDPresent&0x1)<<55 | - uint64(vp8.TL0PICIDXPresent&0x1)<<54 | - uint64(vp8.TIDPresent&0x1)<<53 | - uint64(vp8.KEYIDXPresent&0x1)<<52 | - uint64(btoi(vp8.MBit)&0x1)<<51 | - uint64(btoi(vp8.IsKeyFrame)&0x1)<<50 | - uint64(vp8.PictureID&0x7FFF)<<32 | - uint64(vp8.TL0PICIDX&0xFF)<<24 | - uint64(vp8.TID&0x3)<<22 | - uint64(vp8.Y&0x1)<<21 | - uint64(vp8.KEYIDX&0x1F)<<16 | - uint64(vp8.HeaderSize&0xFF)<<8 -} - -func (p *packetMeta) unpackVP8() *buffer.VP8 { - return &buffer.VP8{ - FirstByte: byte(p.misc >> 56), - PictureIDPresent: int((p.misc >> 55) & 0x1), - PictureID: uint16((p.misc >> 32) & 0x7FFF), - MBit: itob(int((p.misc >> 51) & 0x1)), - TL0PICIDXPresent: int((p.misc >> 54) & 0x1), - TL0PICIDX: uint8((p.misc >> 24) & 0xFF), - TIDPresent: int((p.misc >> 53) & 0x1), - TID: uint8((p.misc >> 22) & 0x3), - Y: uint8((p.misc >> 21) & 0x1), - KEYIDXPresent: int((p.misc >> 52) & 0x1), - KEYIDX: uint8((p.misc >> 16) & 0x1F), - HeaderSize: int((p.misc >> 8) & 0xFF), - IsKeyFrame: itob(int((p.misc >> 50) & 0x1)), - } -} - // Sequencer stores the packet sequence received by the down track type sequencer struct { sync.Mutex @@ -128,20 +109,30 @@ func (s *sequencer) setRTT(rtt uint32) { } } -func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8) *packetMeta { +func (s *sequencer) push( + sn, offSn uint16, + timeStamp uint32, + marker bool, + layer int8, + codecBytes []byte, + ddBytes []byte, +) { s.Lock() defer s.Unlock() slot, isValid := s.getSlot(offSn) if !isValid { - return nil + return } s.meta[s.metaWritePtr] = packetMeta{ sourceSeqNo: sn, targetSeqNo: offSn, timestamp: timeStamp, + marker: marker, layer: layer, + codecBytes: append([]byte{}, codecBytes...), + ddBytes: append([]byte{}, ddBytes...), } s.seq[slot] = &s.meta[s.metaWritePtr] @@ -150,8 +141,6 @@ func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8) *packet if s.metaWritePtr >= len(s.meta) { s.metaWritePtr -= len(s.meta) } - - return s.seq[slot] } func (s *sequencer) pushPadding(offSn uint16) { @@ -204,11 +193,11 @@ func (s *sequencer) getSlot(offSn uint16) (int, bool) { return s.wrap(slot), true } -func (s *sequencer) getPacketsMeta(seqNo []uint16) []*packetMeta { +func (s *sequencer) getPacketsMeta(seqNo []uint16) []packetMeta { s.Lock() defer s.Unlock() - meta := make([]*packetMeta, 0, len(seqNo)) + meta := make([]packetMeta, 0, len(seqNo)) refTime := uint32(time.Now().UnixNano()/1e6 - s.startTime) for _, sn := range seqNo { diff := s.headSN - sn @@ -223,10 +212,14 @@ func (s *sequencer) getPacketsMeta(seqNo []uint16) []*packetMeta { continue } - if seq.lastNack == 0 || refTime-seq.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt))) { + if (seq.lastNack == 0 || refTime-seq.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt)))) && seq.nacked < maxAck { seq.nacked++ seq.lastNack = refTime - meta = append(meta, seq) + + pm := *seq + pm.codecBytes = append([]byte{}, seq.codecBytes...) + pm.ddBytes = append([]byte{}, seq.ddBytes...) + meta = append(meta, pm) } } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 51cbc0eb4..a2303742d 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( @@ -8,8 +22,6 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) func Test_sequencer(t *testing.T) { @@ -17,11 +29,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint16(1); i < 518; i++ { - seq.push(i, i+off, 123, 2) + seq.push(i, i+off, 123, true, 2, nil, nil) } // send the last two out-of-order - seq.push(519, 519+off, 123, 2) - seq.push(518, 518+off, 123, 2) + seq.push(519, 519+off, 123, false, 2, nil, nil) + seq.push(518, 518+off, 123, true, 2, nil, nil) time.Sleep(60 * time.Millisecond) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} @@ -43,11 +55,11 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.layer, int8(2)) } - seq.push(521, 521+off, 123, 1) + seq.push(521, 521+off, 123, true, 1, nil, nil) m := seq.getPacketsMeta([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(505, 505+off, 123, 1) + seq.push(505, 505+off, 123, false, 1, nil, nil) m = seq.getPacketsMeta([]uint16{505 + off}) require.Equal(t, 1, len(m)) } @@ -57,9 +69,15 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { seqNo []uint16 } type fields struct { - input []uint16 - padding []uint16 - offset uint16 + input []uint16 + padding []uint16 + offset uint16 + markerOdd bool + markerEven bool + codecBytesOdd []byte + codecBytesEven []byte + ddBytesOdd []byte + ddBytesEven []byte } tests := []struct { @@ -71,9 +89,15 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { { name: "Should get correct seq numbers", fields: fields{ - input: []uint16{2, 3, 4, 7, 8, 11}, - padding: []uint16{9, 10}, - offset: 5, + input: []uint16{2, 3, 4, 7, 8, 11}, + padding: []uint16{9, 10}, + offset: 5, + markerOdd: true, + markerEven: false, + codecBytesOdd: []byte{1, 2, 3, 4}, + codecBytesEven: []byte{5, 6, 7}, + ddBytesOdd: []byte{8, 9, 10}, + ddBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5}, @@ -87,7 +111,11 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { n := newSequencer(5, 10, logger.GetLogger()) for _, i := range tt.fields.input { - n.push(i, i+tt.fields.offset, 123, 3) + if i%2 == 0 { + n.push(i, i+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) + } else { + n.push(i, i+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) + } } for _, i := range tt.fields.padding { n.pushPadding(i + tt.fields.offset) @@ -97,6 +125,15 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { var got []uint16 for _, sn := range g { got = append(got, sn.sourceSeqNo) + if sn.sourceSeqNo%2 == 0 { + require.Equal(t, tt.fields.markerEven, sn.marker) + require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes) + require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) + } else { + require.Equal(t, tt.fields.markerOdd, sn.marker) + require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes) + require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) + } } if !reflect.DeepEqual(got, tt.want) { t.Errorf("getPacketsMeta() = %v, want %v", got, tt.want) @@ -104,82 +141,3 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { }) } } - -func Test_packetMeta_VP8(t *testing.T) { - p := &packetMeta{} - - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 55467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - - p.packVP8(vp8) - - // booleans are not packed, so they will be `false` in unpacked. - // Also, TID is only two bits, so it should be modulo 3. - expectedVP8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 55467 % 32768, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13 % 3, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - unpackedVP8 := p.unpackVP8() - require.Equal(t, expectedVP8, unpackedVP8) - - // short picture id and no TL0PICIDX - vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 63, - MBit: false, - TL0PICIDXPresent: 0, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 0, - KEYIDX: 23, - HeaderSize: 23, - IsKeyFrame: true, - } - - p.packVP8(vp8) - - expectedVP8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 63, - MBit: false, - TL0PICIDXPresent: 0, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 0, - KEYIDX: 23, - HeaderSize: 23, - IsKeyFrame: true, - } - unpackedVP8 = p.unpackVP8() - require.Equal(t, expectedVP8, unpackedVP8) -} diff --git a/pkg/sfu/sfu.go b/pkg/sfu/sfu.go index cd15dc01f..648c1e563 100644 --- a/pkg/sfu/sfu.go +++ b/pkg/sfu/sfu.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/streamallocator.go b/pkg/sfu/streamallocator.go deleted file mode 100644 index bf5ba725f..000000000 --- a/pkg/sfu/streamallocator.go +++ /dev/null @@ -1,1877 +0,0 @@ -package sfu - -import ( - "fmt" - "math" - "sort" - "sync" - "time" - - "github.com/pion/interceptor/pkg/cc" - "github.com/pion/rtcp" - "github.com/pion/webrtc/v3" - "go.uber.org/atomic" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/config" -) - -const ( - ChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps - - NackRatioAttenuator = 0.4 // how much to attenuate NACK ratio while calculating loss adjusted estimate - - ProbeWaitBase = 5 * time.Second - ProbeBackoffFactor = 1.5 - ProbeWaitMax = 30 * time.Second - ProbeSettleWait = 250 - ProbeTrendWait = 2 * time.Second - - ProbePct = 120 - ProbeMinBps = 200 * 1000 // 200 kbps - ProbeMinDuration = 20 * time.Second - ProbeMaxDuration = 21 * time.Second - - PriorityMin = uint8(1) - PriorityMax = uint8(255) - PriorityDefaultScreenshare = PriorityMax - PriorityDefaultVideo = PriorityMin - - FlagAllowOvershootWhileOptimal = true - FlagAllowOvershootWhileDeficient = false - FlagAllowOvershootExemptTrackWhileDeficient = true - FlagAllowOvershootInProbe = true - FlagAllowOvershootInCatchup = true -) - -var ( - ChannelObserverParamsProbe = ChannelObserverParams{ - Name: "probe", - EstimateRequiredSamples: 3, - EstimateDownwardTrendThreshold: 0.0, - EstimateCollapseValues: false, - NackWindowMinDuration: 500 * time.Millisecond, - NackWindowMaxDuration: 1 * time.Second, - NackRatioThreshold: 0.04, - } - - ChannelObserverParamsNonProbe = ChannelObserverParams{ - Name: "non-probe", - EstimateRequiredSamples: 8, - EstimateDownwardTrendThreshold: -0.5, - EstimateCollapseValues: true, - NackWindowMinDuration: 1 * time.Second, - NackWindowMaxDuration: 2 * time.Second, - NackRatioThreshold: 0.08, - } -) - -type streamAllocatorState int - -const ( - streamAllocatorStateStable streamAllocatorState = iota - streamAllocatorStateDeficient -) - -func (s streamAllocatorState) String() string { - switch s { - case streamAllocatorStateStable: - return "STABLE" - case streamAllocatorStateDeficient: - return "DEFICIENT" - default: - return fmt.Sprintf("%d", int(s)) - } -} - -type streamAllocatorSignal int - -const ( - streamAllocatorSignalAllocateTrack streamAllocatorSignal = iota - streamAllocatorSignalAllocateAllTracks - streamAllocatorSignalAdjustState - streamAllocatorSignalEstimate - streamAllocatorSignalPeriodicPing - streamAllocatorSignalSendProbe - streamAllocatorSignalProbeClusterDone - streamAllocatorSignalTargetLayerFound -) - -func (s streamAllocatorSignal) String() string { - switch s { - case streamAllocatorSignalAllocateTrack: - return "ALLOCATE_TRACK" - case streamAllocatorSignalAllocateAllTracks: - return "ALLOCATE_ALL_TRACKS" - case streamAllocatorSignalAdjustState: - return "ADJUST_STATE" - case streamAllocatorSignalEstimate: - return "ESTIMATE" - case streamAllocatorSignalPeriodicPing: - return "PERIODIC_PING" - case streamAllocatorSignalSendProbe: - return "SEND_PROBE" - case streamAllocatorSignalProbeClusterDone: - return "PROBE_CLUSTER_DONE" - case streamAllocatorSignalTargetLayerFound: - return "TARGET_LAYER_FOUND" - default: - return fmt.Sprintf("%d", int(s)) - } -} - -type Event struct { - Signal streamAllocatorSignal - TrackID livekit.TrackID - Data interface{} -} - -func (e Event) String() string { - return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data) -} - -type StreamAllocatorParams struct { - Config config.CongestionControlConfig - Logger logger.Logger -} - -type StreamAllocator struct { - params StreamAllocatorParams - - onStreamStateChange func(update *StreamStateUpdate) error - - bwe cc.BandwidthEstimator - - lastReceivedEstimate int64 - committedChannelCapacity int64 - - probeInterval time.Duration - lastProbeStartTime time.Time - probeGoalBps int64 - probeClusterId ProbeClusterId - abortedProbeClusterId ProbeClusterId - probeTrendObserved bool - probeEndTime time.Time - - prober *Prober - - channelObserver *ChannelObserver - - videoTracksMu sync.RWMutex - videoTracks map[livekit.TrackID]*Track - isAllocateAllPending bool - rembTrackingSSRC uint32 - - state streamAllocatorState - - eventChMu sync.RWMutex - eventCh chan Event - - isStopped atomic.Bool -} - -func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { - s := &StreamAllocator{ - params: params, - prober: NewProber(ProberParams{ - Logger: params.Logger, - }), - videoTracks: make(map[livekit.TrackID]*Track), - eventCh: make(chan Event, 200), - } - - s.resetState() - - s.prober.SetProberListener(s) - - return s -} - -func (s *StreamAllocator) Start() { - go s.processEvents() - go s.ping() -} - -func (s *StreamAllocator) Stop() { - s.eventChMu.Lock() - if s.isStopped.Swap(true) { - s.eventChMu.Unlock() - return - } - - close(s.eventCh) - s.eventChMu.Unlock() -} - -func (s *StreamAllocator) OnStreamStateChange(f func(update *StreamStateUpdate) error) { - s.onStreamStateChange = f -} - -func (s *StreamAllocator) SetBandwidthEstimator(bwe cc.BandwidthEstimator) { - if bwe != nil { - bwe.OnTargetBitrateChange(s.onTargetBitrateChange) - } - s.bwe = bwe -} - -type AddTrackParams struct { - Source livekit.TrackSource - Priority uint8 - IsSimulcast bool - PublisherID livekit.ParticipantID -} - -func (s *StreamAllocator) AddTrack(downTrack *DownTrack, params AddTrackParams) { - if downTrack.Kind() != webrtc.RTPCodecTypeVideo { - return - } - - track := newTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) - track.SetPriority(params.Priority) - - s.videoTracksMu.Lock() - s.videoTracks[livekit.TrackID(downTrack.ID())] = track - s.videoTracksMu.Unlock() - - downTrack.SetStreamAllocatorListener(s) - if s.prober.IsRunning() { - // LK-TODO: this can be changed to adapt to probe rate - downTrack.SetStreamAllocatorReportInterval(50 * time.Millisecond) - } - - s.maybePostEventAllocateTrack(downTrack) -} - -func (s *StreamAllocator) RemoveTrack(downTrack *DownTrack) { - s.videoTracksMu.Lock() - if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack { - delete(s.videoTracks, livekit.TrackID(downTrack.ID())) - } - s.videoTracksMu.Unlock() - - // LK-TODO: use any saved bandwidth to re-distribute - s.postEvent(Event{ - Signal: streamAllocatorSignalAdjustState, - }) -} - -func (s *StreamAllocator) SetTrackPriority(downTrack *DownTrack, priority uint8) { - s.videoTracksMu.Lock() - if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { - changed := track.SetPriority(priority) - if changed && !s.isAllocateAllPending { - // do a full allocation on a track priority change to keep it simple - s.isAllocateAllPending = true - s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateAllTracks, - }) - } - } - s.videoTracksMu.Unlock() -} - -func (s *StreamAllocator) resetState() { - s.channelObserver = s.newChannelObserverNonProbe() - s.resetProbe() - - s.state = streamAllocatorStateStable -} - -// called when a new REMB is received (receive side bandwidth estimation) -func (s *StreamAllocator) OnREMB(downTrack *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) { - // - // Channel capacity is estimated at a peer connection level. All down tracks - // in the peer connection will end up calling this for a REMB report with - // the same estimated channel capacity. Use a tracking SSRC to lock onto to - // one report. As SSRCs can be dropped over time, update tracking SSRC as needed - // - // A couple of things to keep in mind - // - REMB reports could be sent gratuitously as a way of providing - // periodic feedback, i.e. even if the estimated capacity does not - // change, there could be REMB packets on the wire. Those gratuitous - // REMBs should not trigger anything bad. - // - As each down track will issue this callback for the same REMB packet - // from the wire, theoretically it is possible that one down track's - // callback from previous REMB comes after another down track's callback - // from the new REMB. REMBs could fire very quickly especially when - // the network is entering congestion. - // LK-TODO-START - // Need to check if the same SSRC reports can somehow race, i.e. does pion send - // RTCP dispatch for same SSRC on different threads? If not, the tracking SSRC - // should prevent racing - // LK-TODO-END - // - - // if there are no video tracks, ignore any straggler REMB - s.videoTracksMu.Lock() - if len(s.videoTracks) == 0 { - s.videoTracksMu.Unlock() - return - } - - track := s.videoTracks[livekit.TrackID(downTrack.ID())] - downTrackSSRC := uint32(0) - if track != nil { - downTrackSSRC = track.DownTrack().SSRC() - } - - found := false - for _, ssrc := range remb.SSRCs { - if ssrc == s.rembTrackingSSRC { - found = true - break - } - } - if !found { - if len(remb.SSRCs) == 0 { - s.params.Logger.Warnw("stream allocator: no SSRC to track REMB", nil) - s.videoTracksMu.Unlock() - return - } - - // try to lock to track which is sending this update - if downTrackSSRC != 0 { - for _, ssrc := range remb.SSRCs { - if ssrc == downTrackSSRC { - s.rembTrackingSSRC = downTrackSSRC - found = true - break - } - } - } - - if !found { - s.rembTrackingSSRC = remb.SSRCs[0] - } - } - - if s.rembTrackingSSRC == 0 || s.rembTrackingSSRC != downTrackSSRC { - s.videoTracksMu.Unlock() - return - } - s.videoTracksMu.Unlock() - - s.postEvent(Event{ - Signal: streamAllocatorSignalEstimate, - Data: int64(remb.Bitrate), - }) -} - -// called when a new transport-cc feedback is received -func (s *StreamAllocator) OnTransportCCFeedback(downTrack *DownTrack, fb *rtcp.TransportLayerCC) { - if s.bwe != nil { - s.bwe.WriteRTCP([]rtcp.Packet{fb}, nil) - } -} - -// called when target bitrate changes (send side bandwidth estimation) -func (s *StreamAllocator) onTargetBitrateChange(bitrate int) { - s.postEvent(Event{ - Signal: streamAllocatorSignalEstimate, - Data: int64(bitrate), - }) -} - -// called when feeding track's layer availability changes -func (s *StreamAllocator) OnAvailableLayersChanged(downTrack *DownTrack) { - s.maybePostEventAllocateTrack(downTrack) -} - -// called when feeding track's bitrate measurement of any layer is available -func (s *StreamAllocator) OnBitrateAvailabilityChanged(downTrack *DownTrack) { - s.maybePostEventAllocateTrack(downTrack) -} - -// called when feeding track's max publisher layer changes -func (s *StreamAllocator) OnMaxPublishedLayerChanged(downTrack *DownTrack) { - s.maybePostEventAllocateTrack(downTrack) -} - -// called when subscription settings changes (muting/unmuting of track) -func (s *StreamAllocator) OnSubscriptionChanged(downTrack *DownTrack) { - s.maybePostEventAllocateTrack(downTrack) -} - -// called when subscribed layers changes (limiting max layers) -func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *DownTrack, layers VideoLayers) { - shouldPost := false - s.videoTracksMu.Lock() - if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { - if track.SetMaxLayers(layers) && track.SetDirty(true) { - shouldPost = true - } - } - s.videoTracksMu.Unlock() - - if shouldPost { - s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateTrack, - TrackID: livekit.TrackID(downTrack.ID()), - }) - } -} - -// called when forwarder finds a target layer -func (s *StreamAllocator) OnTargetLayerReached(downTrack *DownTrack) { - s.postEvent(Event{ - Signal: streamAllocatorSignalTargetLayerFound, - TrackID: livekit.TrackID(downTrack.ID()), - }) -} - -// called when a video DownTrack sends a packet -func (s *StreamAllocator) OnPacketsSent(downTrack *DownTrack, size int) { - s.prober.PacketsSent(size) -} - -// called when prober wants to send packet(s) -func (s *StreamAllocator) OnSendProbe(bytesToSend int) { - s.postEvent(Event{ - Signal: streamAllocatorSignalSendProbe, - Data: bytesToSend, - }) -} - -// called when prober wants to send packet(s) -func (s *StreamAllocator) OnProbeClusterDone(info ProbeClusterInfo) { - s.postEvent(Event{ - Signal: streamAllocatorSignalProbeClusterDone, - Data: info, - }) -} - -// called when prober active state changes -func (s *StreamAllocator) OnActiveChanged(isActive bool) { - for _, t := range s.getTracks() { - if isActive { - // LK-TODO: this can be changed to adapt to probe rate - t.DownTrack().SetStreamAllocatorReportInterval(50 * time.Millisecond) - } else { - t.DownTrack().ClearStreamAllocatorReportInterval() - } - } -} - -func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *DownTrack) { - shouldPost := false - s.videoTracksMu.Lock() - if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { - if track.SetDirty(true) { - shouldPost = true - } - } - s.videoTracksMu.Unlock() - - if shouldPost { - s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateTrack, - TrackID: livekit.TrackID(downTrack.ID()), - }) - } -} - -func (s *StreamAllocator) postEvent(event Event) { - s.eventChMu.RLock() - if s.isStopped.Load() { - s.eventChMu.RUnlock() - return - } - - select { - case s.eventCh <- event: - default: - s.params.Logger.Warnw("stream allocator: event queue full", nil) - } - s.eventChMu.RUnlock() -} - -func (s *StreamAllocator) processEvents() { - for event := range s.eventCh { - s.handleEvent(&event) - } - - s.stopProbe() -} - -func (s *StreamAllocator) ping() { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for { - <-ticker.C - if s.isStopped.Load() { - return - } - - s.postEvent(Event{ - Signal: streamAllocatorSignalPeriodicPing, - }) - } -} - -func (s *StreamAllocator) handleEvent(event *Event) { - switch event.Signal { - case streamAllocatorSignalAllocateTrack: - s.handleSignalAllocateTrack(event) - case streamAllocatorSignalAllocateAllTracks: - s.handleSignalAllocateAllTracks(event) - case streamAllocatorSignalAdjustState: - s.handleSignalAdjustState(event) - case streamAllocatorSignalEstimate: - s.handleSignalEstimate(event) - case streamAllocatorSignalPeriodicPing: - s.handleSignalPeriodicPing(event) - case streamAllocatorSignalSendProbe: - s.handleSignalSendProbe(event) - case streamAllocatorSignalProbeClusterDone: - s.handleSignalProbeClusterDone(event) - case streamAllocatorSignalTargetLayerFound: - s.handleSignalTargetLayerFound(event) - } -} - -func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { - s.videoTracksMu.Lock() - track := s.videoTracks[event.TrackID] - if track != nil { - track.SetDirty(false) - } - s.videoTracksMu.Unlock() - - if track != nil { - s.allocateTrack(track) - } -} - -func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { - s.videoTracksMu.Lock() - s.isAllocateAllPending = false - s.videoTracksMu.Unlock() - - if s.state == streamAllocatorStateDeficient { - s.allocateAllTracks() - } -} - -func (s *StreamAllocator) handleSignalAdjustState(event *Event) { - s.adjustState() -} - -func (s *StreamAllocator) handleSignalEstimate(event *Event) { - receivedEstimate, _ := event.Data.(int64) - s.lastReceivedEstimate = receivedEstimate - - // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails - if s.isInProbe() { - s.handleNewEstimateInProbe() - } else { - s.handleNewEstimateInNonProbe() - } -} - -func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { - // finalize probe if necessary - if s.isInProbe() && !s.probeEndTime.IsZero() && time.Now().After(s.probeEndTime) { - s.finalizeProbe() - } - - // probe if necessary and timing is right - if s.state == streamAllocatorStateDeficient { - s.maybeProbe() - } -} - -func (s *StreamAllocator) handleSignalSendProbe(event *Event) { - bytesToSend := event.Data.(int) - if bytesToSend <= 0 { - return - } - - bytesSent := 0 - for _, track := range s.getTracks() { - sent := track.WritePaddingRTP(bytesToSend) - bytesSent += sent - bytesToSend -= sent - if bytesToSend <= 0 { - break - } - } - - if bytesSent != 0 { - s.prober.ProbeSent(bytesSent) - } -} - -func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { - info, _ := event.Data.(ProbeClusterInfo) - if s.probeClusterId != info.Id { - return - } - - if s.abortedProbeClusterId == ProbeClusterIdInvalid { - // successful probe, finalize - s.finalizeProbe() - return - } - - // ensure probe queue is flushed - // LK-TODO: ProbeSettleWait should actually be a certain number of RTTs. - lowestEstimate := int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate()))) - expectedDuration := float64(info.BytesSent*8*1000) / float64(lowestEstimate) - queueTime := expectedDuration - float64(info.Duration.Milliseconds()) - if queueTime < 0.0 { - queueTime = 0.0 - } - queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond - s.probeEndTime = s.lastProbeStartTime.Add(queueWait) -} - -func (s *StreamAllocator) handleSignalTargetLayerFound(event *Event) { - s.videoTracksMu.Lock() - track := s.videoTracks[event.TrackID] - s.videoTracksMu.Unlock() - - if track != nil { - update := NewStreamStateUpdate() - if track.SetPaused(false) { - update.HandleStreamingChange(false, track) - } - s.maybeSendUpdate(update) - } -} - -func (s *StreamAllocator) setState(state streamAllocatorState) { - if s.state == state { - return - } - - s.params.Logger.Infow("stream allocator: state change", "from", s.state, "to", state) - s.state = state - - // reset probe to enforce a delay after state change before probing - s.lastProbeStartTime = time.Now() -} - -func (s *StreamAllocator) adjustState() { - for _, track := range s.getTracks() { - if track.IsDeficient() { - s.setState(streamAllocatorStateDeficient) - return - } - } - - s.setState(streamAllocatorStateStable) -} - -func (s *StreamAllocator) handleNewEstimateInProbe() { - // always update NACKs, even if aborted - packetDelta, repeatedNackDelta := s.getNackDelta() - - if s.abortedProbeClusterId != ProbeClusterIdInvalid { - // waiting for aborted probe to finalize - return - } - - s.channelObserver.AddEstimate(s.lastReceivedEstimate) - s.channelObserver.AddNack(packetDelta, repeatedNackDelta) - - trend, _ := s.channelObserver.GetTrend() - if trend != ChannelTrendNeutral { - s.probeTrendObserved = true - } - - switch { - case !s.probeTrendObserved && time.Since(s.lastProbeStartTime) > ProbeTrendWait: - // - // More of a safety net. - // In rare cases, the estimate gets stuck. Prevent from probe running amok - // LK-TODO: Need more testing here to ensure that probe does not cause a lot of damage - // - s.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", s.probeClusterId) - s.abortProbe() - - case trend == ChannelTrendCongesting: - // stop immediately if the probe is congesting channel more - s.params.Logger.Infow("stream allocator: probe: aborting, channel is congesting", "cluster", s.probeClusterId) - s.abortProbe() - - case s.channelObserver.GetHighestEstimate() > s.probeGoalBps: - // reached goal, stop probing - s.params.Logger.Infow( - "stream allocator: probe: stopping, goal reached", - "cluster", s.probeClusterId, - "goal", s.probeGoalBps, - "highest", s.channelObserver.GetHighestEstimate(), - ) - s.stopProbe() - } -} - -func (s *StreamAllocator) handleNewEstimateInNonProbe() { - s.channelObserver.AddEstimate(s.lastReceivedEstimate) - - packetDelta, repeatedNackDelta := s.getNackDelta() - s.channelObserver.AddNack(packetDelta, repeatedNackDelta) - - trend, reason := s.channelObserver.GetTrend() - if trend != ChannelTrendCongesting { - return - } - - var estimateToCommit int64 - var packets, repeatedNacks uint32 - var nackRatio float64 - expectedBandwidthUsage := s.getExpectedBandwidthUsage() - switch reason { - case ChannelCongestionReasonLoss: - packets, repeatedNacks, nackRatio = s.channelObserver.GetNackRatio() - estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*nackRatio)) - default: - estimateToCommit = s.lastReceivedEstimate - } - - s.params.Logger.Infow( - "stream allocator: channel congestion detected, updating channel capacity", - "reason", reason, - "old(bps)", s.committedChannelCapacity, - "new(bps)", estimateToCommit, - "lastReceived(bps)", s.lastReceivedEstimate, - "expectedUsage(bps)", expectedBandwidthUsage, - "packets", packets, - "repeatedNacks", repeatedNacks, - "nackRatio", nackRatio, - ) - s.committedChannelCapacity = estimateToCommit - - // reset to get new set of samples for next trend - s.channelObserver = s.newChannelObserverNonProbe() - - // reset probe to ensure it does not start too soon after a downward trend - s.resetProbe() - - s.allocateAllTracks() -} - -func (s *StreamAllocator) allocateTrack(track *Track) { - // abort any probe that may be running when a track specific change needs allocation - s.abortProbe() - - // if not deficient, free pass allocate track - if !s.params.Config.Enabled || s.state == streamAllocatorStateStable || !track.IsManaged() { - update := NewStreamStateUpdate() - allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal) - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - s.maybeSendUpdate(update) - return - } - - // - // In DEFICIENT state, - // 1. Find cooperative transition from track that needs allocation. - // 2. If track is currently streaming at minimum, do not do anything. - // 3. If that track is giving back bits, apply the transition. - // 4. If this track needs more, ask for best offer from others and try to use it. - // - track.ProvisionalAllocatePrepare() - transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) - - // track is currently streaming at minimum - if transition.bandwidthDelta == 0 { - return - } - - // downgrade, giving back bits - if transition.from.GreaterThan(transition.to) { - allocation := track.ProvisionalAllocateCommit() - - update := NewStreamStateUpdate() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - s.maybeSendUpdate(update) - - s.adjustState() - return - // LK-TODO-START - // Should use the bits given back to start any paused track. - // Note layer downgrade may actually have positive delta (i.e. consume more bits) - // because of when the measurement is done. Watch for that. - // LK-TODO-END - } - - // - // This track is currently not streaming and needs bits to start. - // Try to redistribute starting with tracks that are closest to their desired. - // - bandwidthAcquired := int64(0) - var contributingTracks []*Track - - minDistanceSorted := s.getMinDistanceSorted(track) - for _, t := range minDistanceSorted { - t.ProvisionalAllocatePrepare() - } - - for _, t := range minDistanceSorted { - tx := t.ProvisionalAllocateGetBestWeightedTransition() - if tx.bandwidthDelta < 0 { - contributingTracks = append(contributingTracks, t) - - bandwidthAcquired += -tx.bandwidthDelta - if bandwidthAcquired >= transition.bandwidthDelta { - break - } - } - } - - update := NewStreamStateUpdate() - if bandwidthAcquired >= transition.bandwidthDelta { - // commit the tracks that contributed - for _, t := range contributingTracks { - allocation := t.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, t) - } - } - - // LK-TODO if got too much extra, can potentially give it to some deficient track - } - - // commit the track that needs change if enough could be acquired or pause not allowed - if !s.params.Config.AllowPause || bandwidthAcquired >= transition.bandwidthDelta { - allocation := track.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - } - - s.maybeSendUpdate(update) - - s.adjustState() -} - -func (s *StreamAllocator) finalizeProbe() { - aborted := s.probeClusterId == s.abortedProbeClusterId - highestEstimateInProbe := s.channelObserver.GetHighestEstimate() - - s.clearProbe() - - // - // Reset estimator at the end of a probe irrespective of probe result to get fresh readings. - // With a failed probe, the latest estimate would be lower than committed estimate. - // As bandwidth estimator (remote in REMB case, local in TWCC case) holds state, - // subsequent estimates could start from the lower point. That should not trigger a - // downward trend and get latched to committed estimate as that would trigger a re-allocation. - // With fresh readings, as long as the trend is not going downward, it will not get latched. - // - // NOTE: With TWCC, it is possible to reset bandwidth estimation to clean state as - // the send side is in full control of bandwidth estimation. - // - s.channelObserver = s.newChannelObserverNonProbe() - - if aborted { - // failed probe, backoff - s.backoffProbeInterval() - return - } - - // reset probe interval on a successful probe - s.resetProbeInterval() - - // probe estimate is same or higher, commit it and try to allocate deficient tracks - s.params.Logger.Infow( - "successful probe, updating channel capacity", - "old(bps)", s.committedChannelCapacity, - "new(bps)", highestEstimateInProbe, - ) - s.committedChannelCapacity = highestEstimateInProbe - - s.maybeBoostDeficientTracks() -} - -func (s *StreamAllocator) maybeBoostDeficientTracks() { - committedChannelCapacity := s.committedChannelCapacity - if s.params.Config.MinChannelCapacity > committedChannelCapacity { - committedChannelCapacity = s.params.Config.MinChannelCapacity - s.params.Logger.Debugw( - "stream allocator: overriding channel capacity", - "actual", s.committedChannelCapacity, - "override", committedChannelCapacity, - ) - } - availableChannelCapacity := committedChannelCapacity - s.getExpectedBandwidthUsage() - if availableChannelCapacity <= 0 { - return - } - - update := NewStreamStateUpdate() - - for _, track := range s.getMaxDistanceSortedDeficient() { - allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, FlagAllowOvershootInCatchup) - if !boosted { - continue - } - - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - - availableChannelCapacity -= allocation.bandwidthDelta - if availableChannelCapacity <= 0 { - break - } - } - - s.maybeSendUpdate(update) - - s.adjustState() -} - -func (s *StreamAllocator) allocateAllTracks() { - if !s.params.Config.Enabled { - // nothing else to do when disabled - return - } - - // - // Goals: - // 1. Stream as many tracks as possible, i.e. no pauses. - // 2. Try to give fair allocation to all track. - // - // Start with the lowest layers and give each track a chance at that layer and keep going up. - // As long as there is enough bandwidth for tracks to stream at the lowest layers, the first goal is achieved. - // - // Tracks that have higher subscribed layers can use any additional available bandwidth. This tried to achieve the second goal. - // - // If there is not enough bandwidth even for the lowest layers, tracks at lower priorities will be paused. - // - update := NewStreamStateUpdate() - - availableChannelCapacity := s.committedChannelCapacity - if s.params.Config.MinChannelCapacity > availableChannelCapacity { - availableChannelCapacity = s.params.Config.MinChannelCapacity - s.params.Logger.Debugw( - "stream allocator: overriding channel capacity", - "actual", s.committedChannelCapacity, - "override", availableChannelCapacity, - ) - } - - // - // This pass is to find out if there is any leftover channel capacity after allocating exempt tracks. - // Exempt tracks are given optimal allocation (i. e. no bandwidth constraint) so that they do not fail allocation. - // - videoTracks := s.getTracks() - for _, track := range videoTracks { - if track.IsManaged() { - continue - } - - allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient) - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - - // LK-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? - availableChannelCapacity -= allocation.bandwidthRequested - } - - if availableChannelCapacity < 0 { - availableChannelCapacity = 0 - } - if availableChannelCapacity == 0 && s.params.Config.AllowPause { - // nothing left for managed tracks, pause them all - for _, track := range videoTracks { - if !track.IsManaged() { - continue - } - - allocation := track.Pause() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - } - } else { - sorted := s.getSorted() - for _, track := range sorted { - track.ProvisionalAllocatePrepare() - } - - for spatial := int32(0); spatial <= DefaultMaxLayerSpatial; spatial++ { - for temporal := int32(0); temporal <= DefaultMaxLayerTemporal; temporal++ { - layers := VideoLayers{ - Spatial: spatial, - Temporal: temporal, - } - - for _, track := range sorted { - usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layers, s.params.Config.AllowPause, FlagAllowOvershootWhileDeficient) - availableChannelCapacity -= usedChannelCapacity - if availableChannelCapacity < 0 { - availableChannelCapacity = 0 - } - } - } - } - - for _, track := range sorted { - allocation := track.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - } - } - - s.maybeSendUpdate(update) - - s.adjustState() -} - -func (s *StreamAllocator) maybeSendUpdate(update *StreamStateUpdate) { - if update.Empty() { - return - } - - // logging individual changes to make it easier for logging systems - for _, streamState := range update.StreamStates { - s.params.Logger.Debugw("streamed tracks changed", - "trackID", streamState.TrackID, - "state", streamState.State, - ) - } - if s.onStreamStateChange != nil { - err := s.onStreamStateChange(update) - if err != nil { - s.params.Logger.Errorw("could not send streamed tracks update", err) - } - } -} - -func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { - expected := int64(0) - for _, track := range s.getTracks() { - expected += track.BandwidthRequested() - } - - return expected -} - -func (s *StreamAllocator) getNackDelta() (uint32, uint32) { - aggPacketDelta := uint32(0) - aggRepeatedNackDelta := uint32(0) - for _, track := range s.getTracks() { - packetDelta, nackDelta := track.GetNackDelta() - aggPacketDelta += packetDelta - aggRepeatedNackDelta += nackDelta - } - - return aggPacketDelta, aggRepeatedNackDelta -} - -func (s *StreamAllocator) newChannelObserverProbe() *ChannelObserver { - return NewChannelObserver(ChannelObserverParamsProbe, s.params.Logger) -} - -func (s *StreamAllocator) newChannelObserverNonProbe() *ChannelObserver { - return NewChannelObserver(ChannelObserverParamsNonProbe, s.params.Logger) -} - -func (s *StreamAllocator) initProbe(probeRateBps int64) { - s.lastProbeStartTime = time.Now() - - expectedBandwidthUsage := s.getExpectedBandwidthUsage() - s.probeGoalBps = expectedBandwidthUsage + probeRateBps - - s.abortedProbeClusterId = ProbeClusterIdInvalid - - s.probeTrendObserved = false - - s.probeEndTime = time.Time{} - - s.channelObserver = s.newChannelObserverProbe() - s.channelObserver.SeedEstimate(s.lastReceivedEstimate) - - desiredRateBps := int(probeRateBps) + int(math.Max(float64(s.committedChannelCapacity), float64(expectedBandwidthUsage))) - s.probeClusterId = s.prober.AddCluster( - desiredRateBps, - int(expectedBandwidthUsage), - ProbeMinDuration, - ProbeMaxDuration, - ) - s.params.Logger.Infow( - "stream allocator: starting probe", - "probeClusterId", s.probeClusterId, - "current usage", expectedBandwidthUsage, - "committed", s.committedChannelCapacity, - "lastReceived", s.lastReceivedEstimate, - "probeRateBps", probeRateBps, - "goalBps", expectedBandwidthUsage+probeRateBps, - ) -} - -func (s *StreamAllocator) resetProbe() { - s.lastProbeStartTime = time.Now() - - s.resetProbeInterval() - - s.clearProbe() -} - -func (s *StreamAllocator) clearProbe() { - s.probeClusterId = ProbeClusterIdInvalid - s.abortedProbeClusterId = ProbeClusterIdInvalid -} - -func (s *StreamAllocator) backoffProbeInterval() { - s.probeInterval = time.Duration(s.probeInterval.Seconds()*ProbeBackoffFactor) * time.Second - if s.probeInterval > ProbeWaitMax { - s.probeInterval = ProbeWaitMax - } -} - -func (s *StreamAllocator) resetProbeInterval() { - s.probeInterval = ProbeWaitBase -} - -func (s *StreamAllocator) stopProbe() { - s.prober.Reset() -} - -func (s *StreamAllocator) abortProbe() { - s.abortedProbeClusterId = s.probeClusterId - s.stopProbe() -} - -func (s *StreamAllocator) isInProbe() bool { - return s.probeClusterId != ProbeClusterIdInvalid -} - -func (s *StreamAllocator) maybeProbe() { - if time.Since(s.lastProbeStartTime) < s.probeInterval || s.probeClusterId != ProbeClusterIdInvalid { - return - } - - switch s.params.Config.ProbeMode { - case config.CongestionControlProbeModeMedia: - s.maybeProbeWithMedia() - s.adjustState() - case config.CongestionControlProbeModePadding: - s.maybeProbeWithPadding() - } -} - -func (s *StreamAllocator) maybeProbeWithMedia() { - // boost deficient track farthest from desired layers - for _, track := range s.getMaxDistanceSortedDeficient() { - allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInCatchup) - if !boosted { - continue - } - - update := NewStreamStateUpdate() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } - s.maybeSendUpdate(update) - - s.lastProbeStartTime = time.Now() - break - } -} - -func (s *StreamAllocator) maybeProbeWithPadding() { - // use deficient track farthest from desired layers to find how much to probe - for _, track := range s.getMaxDistanceSortedDeficient() { - transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe) - if !available || transition.bandwidthDelta < 0 { - continue - } - - probeRateBps := (transition.bandwidthDelta * ProbePct) / 100 - if probeRateBps < ProbeMinBps { - probeRateBps = ProbeMinBps - } - - s.initProbe(probeRateBps) - break - } -} - -func (s *StreamAllocator) getTracks() []*Track { - s.videoTracksMu.RLock() - var tracks []*Track - for _, track := range s.videoTracks { - tracks = append(tracks, track) - } - s.videoTracksMu.RUnlock() - - return tracks -} - -func (s *StreamAllocator) getSorted() TrackSorter { - s.videoTracksMu.RLock() - var trackSorter TrackSorter - for _, track := range s.videoTracks { - if !track.IsManaged() { - continue - } - - trackSorter = append(trackSorter, track) - } - s.videoTracksMu.RUnlock() - - sort.Sort(trackSorter) - - return trackSorter -} - -func (s *StreamAllocator) getMinDistanceSorted(exclude *Track) MinDistanceSorter { - s.videoTracksMu.RLock() - var minDistanceSorter MinDistanceSorter - for _, track := range s.videoTracks { - if !track.IsManaged() || track == exclude { - continue - } - - minDistanceSorter = append(minDistanceSorter, track) - } - s.videoTracksMu.RUnlock() - - sort.Sort(minDistanceSorter) - - return minDistanceSorter -} - -func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { - s.videoTracksMu.RLock() - var maxDistanceSorter MaxDistanceSorter - for _, track := range s.videoTracks { - if !track.IsManaged() || !track.IsDeficient() { - continue - } - - maxDistanceSorter = append(maxDistanceSorter, track) - } - s.videoTracksMu.RUnlock() - - sort.Sort(maxDistanceSorter) - - return maxDistanceSorter -} - -// ------------------------------------------------ - -type StreamState int - -const ( - StreamStateActive StreamState = iota - StreamStatePaused -) - -func (s StreamState) String() string { - switch s { - case StreamStateActive: - return "active" - case StreamStatePaused: - return "paused" - default: - return "unknown" - } -} - -type StreamStateInfo struct { - ParticipantID livekit.ParticipantID - TrackID livekit.TrackID - State StreamState -} - -type StreamStateUpdate struct { - StreamStates []*StreamStateInfo -} - -func NewStreamStateUpdate() *StreamStateUpdate { - return &StreamStateUpdate{} -} - -func (s *StreamStateUpdate) HandleStreamingChange(isPaused bool, track *Track) { - if isPaused { - s.StreamStates = append(s.StreamStates, &StreamStateInfo{ - ParticipantID: track.PublisherID(), - TrackID: track.ID(), - State: StreamStatePaused, - }) - } else { - s.StreamStates = append(s.StreamStates, &StreamStateInfo{ - ParticipantID: track.PublisherID(), - TrackID: track.ID(), - State: StreamStateActive, - }) - } -} - -func (s *StreamStateUpdate) Empty() bool { - return len(s.StreamStates) == 0 -} - -// ------------------------------------------------ - -type Track struct { - downTrack *DownTrack - source livekit.TrackSource - isSimulcast bool - priority uint8 - publisherID livekit.ParticipantID - logger logger.Logger - - maxLayers VideoLayers - - totalPackets uint32 - totalRepeatedNacks uint32 - - isDirty bool - - isPaused bool -} - -func newTrack( - downTrack *DownTrack, - source livekit.TrackSource, - isSimulcast bool, - publisherID livekit.ParticipantID, - logger logger.Logger, -) *Track { - t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, - isPaused: true, - } - t.SetPriority(0) - t.SetMaxLayers(downTrack.MaxLayers()) - - return t -} - -func (t *Track) SetDirty(isDirty bool) bool { - if t.isDirty == isDirty { - return false - } - - t.isDirty = isDirty - return true -} - -func (t *Track) SetPaused(isPaused bool) bool { - if t.isPaused == isPaused { - return false - } - - t.isPaused = isPaused - return true -} - -func (t *Track) SetPriority(priority uint8) bool { - if priority == 0 { - switch t.source { - case livekit.TrackSource_SCREEN_SHARE: - priority = PriorityDefaultScreenshare - default: - priority = PriorityDefaultVideo - } - } - - if t.priority == priority { - return false - } - - t.priority = priority - return true -} - -func (t *Track) Priority() uint8 { - return t.priority -} - -func (t *Track) DownTrack() *DownTrack { - return t.downTrack -} - -func (t *Track) IsManaged() bool { - return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast -} - -func (t *Track) ID() livekit.TrackID { - return livekit.TrackID(t.downTrack.ID()) -} - -func (t *Track) PublisherID() livekit.ParticipantID { - return t.publisherID -} - -func (t *Track) SetMaxLayers(layers VideoLayers) bool { - if t.maxLayers == layers { - return false - } - - t.maxLayers = layers - return true -} - -func (t *Track) WritePaddingRTP(bytesToSend int) int { - return t.downTrack.WritePaddingRTP(bytesToSend, false) -} - -func (t *Track) AllocateOptimal(allowOvershoot bool) VideoAllocation { - return t.downTrack.AllocateOptimal(allowOvershoot) -} - -func (t *Track) ProvisionalAllocatePrepare() { - t.downTrack.ProvisionalAllocatePrepare() -} - -func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { - return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) -} - -func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) VideoTransition { - return t.downTrack.ProvisionalAllocateGetCooperativeTransition(allowOvershoot) -} - -func (t *Track) ProvisionalAllocateGetBestWeightedTransition() VideoTransition { - return t.downTrack.ProvisionalAllocateGetBestWeightedTransition() -} - -func (t *Track) ProvisionalAllocateCommit() VideoAllocation { - return t.downTrack.ProvisionalAllocateCommit() -} - -func (t *Track) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (VideoAllocation, bool) { - return t.downTrack.AllocateNextHigher(availableChannelCapacity, allowOvershoot) -} - -func (t *Track) GetNextHigherTransition(allowOvershoot bool) (VideoTransition, bool) { - return t.downTrack.GetNextHigherTransition(allowOvershoot) -} - -func (t *Track) Pause() VideoAllocation { - return t.downTrack.Pause() -} - -func (t *Track) IsDeficient() bool { - return t.downTrack.IsDeficient() -} - -func (t *Track) BandwidthRequested() int64 { - return t.downTrack.BandwidthRequested() -} - -func (t *Track) DistanceToDesired() float64 { - return t.downTrack.DistanceToDesired() -} - -func (t *Track) GetNackDelta() (uint32, uint32) { - totalPackets, totalRepeatedNacks := t.downTrack.GetNackStats() - - packetDelta := totalPackets - t.totalPackets - t.totalPackets = totalPackets - - nackDelta := totalRepeatedNacks - t.totalRepeatedNacks - t.totalRepeatedNacks = totalRepeatedNacks - - return packetDelta, nackDelta -} - -// ------------------------------------------------ - -type TrackSorter []*Track - -func (t TrackSorter) Len() int { - return len(t) -} - -func (t TrackSorter) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} - -func (t TrackSorter) Less(i, j int) bool { - // - // TrackSorter is used to allocate layer-by-layer. - // So, higher priority track should come earlier so that it gets an earlier shot at each layer - // - if t[i].priority != t[j].priority { - return t[i].priority > t[j].priority - } - - if t[i].maxLayers.Spatial != t[j].maxLayers.Spatial { - return t[i].maxLayers.Spatial > t[j].maxLayers.Spatial - } - - return t[i].maxLayers.Temporal > t[j].maxLayers.Temporal -} - -// ------------------------------------------------ - -type MaxDistanceSorter []*Track - -func (m MaxDistanceSorter) Len() int { - return len(m) -} - -func (m MaxDistanceSorter) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func (m MaxDistanceSorter) Less(i, j int) bool { - // - // MaxDistanceSorter is used to find a deficient track to use for probing during recovery from congestion. - // So, higher priority track should come earlier so that they have a chance to recover sooner. - // - if m[i].priority != m[j].priority { - return m[i].priority > m[j].priority - } - - return m[i].DistanceToDesired() > m[j].DistanceToDesired() -} - -// ------------------------------------------------ - -type MinDistanceSorter []*Track - -func (m MinDistanceSorter) Len() int { - return len(m) -} - -func (m MinDistanceSorter) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func (m MinDistanceSorter) Less(i, j int) bool { - // - // MinDistanceSorter is used to find excess bandwidth in cooperative allocation. - // So, lower priority track should come earlier so that they contribute bandwidth to higher priority tracks. - // - if m[i].priority != m[j].priority { - return m[i].priority < m[j].priority - } - - return m[i].DistanceToDesired() < m[j].DistanceToDesired() -} - -// ------------------------------------------------ - -type ChannelTrend int - -const ( - ChannelTrendNeutral ChannelTrend = iota - ChannelTrendClearing - ChannelTrendCongesting -) - -func (c ChannelTrend) String() string { - switch c { - case ChannelTrendNeutral: - return "NEUTRAL" - case ChannelTrendClearing: - return "CLEARING" - case ChannelTrendCongesting: - return "CONGESTING" - default: - return fmt.Sprintf("%d", int(c)) - } -} - -type ChannelCongestionReason int - -const ( - ChannelCongestionReasonNone ChannelCongestionReason = iota - ChannelCongestionReasonEstimate - ChannelCongestionReasonLoss -) - -func (c ChannelCongestionReason) String() string { - switch c { - case ChannelCongestionReasonNone: - return "NONE" - case ChannelCongestionReasonEstimate: - return "ESTIMATE" - case ChannelCongestionReasonLoss: - return "LOSS" - default: - return fmt.Sprintf("%d", int(c)) - } -} - -type ChannelObserverParams struct { - Name string - EstimateRequiredSamples int - EstimateDownwardTrendThreshold float64 - EstimateCollapseValues bool - NackWindowMinDuration time.Duration - NackWindowMaxDuration time.Duration - NackRatioThreshold float64 -} - -type ChannelObserver struct { - params ChannelObserverParams - logger logger.Logger - - estimateTrend *TrendDetector - - nackWindowStartTime time.Time - packets uint32 - repeatedNacks uint32 -} - -func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver { - return &ChannelObserver{ - params: params, - logger: logger, - estimateTrend: NewTrendDetector(TrendDetectorParams{ - Name: params.Name + "-estimate", - Logger: logger, - RequiredSamples: params.EstimateRequiredSamples, - DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, - CollapseValues: params.EstimateCollapseValues, - }), - } -} - -func (c *ChannelObserver) SeedEstimate(estimate int64) { - c.estimateTrend.Seed(estimate) -} - -func (c *ChannelObserver) SeedNack(packets uint32, repeatedNacks uint32) { - c.packets = packets - c.repeatedNacks = repeatedNacks -} - -func (c *ChannelObserver) AddEstimate(estimate int64) { - c.estimateTrend.AddValue(estimate) -} - -func (c *ChannelObserver) AddNack(packets uint32, repeatedNacks uint32) { - if c.params.NackWindowMaxDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMaxDuration { - c.nackWindowStartTime = time.Time{} - c.packets = 0 - c.repeatedNacks = 0 - } - - // - // Start NACK monitoring window only when a repeated NACK happens. - // This allows locking tightly to when NACKs start happening and - // check if the NACKs keep adding up (potentially a sign of congestion) - // or isolated losses - // - if c.repeatedNacks == 0 && repeatedNacks != 0 { - c.nackWindowStartTime = time.Now() - } - - if !c.nackWindowStartTime.IsZero() { - c.packets += packets - c.repeatedNacks += repeatedNacks - } -} - -func (c *ChannelObserver) GetLowestEstimate() int64 { - return c.estimateTrend.GetLowest() -} - -func (c *ChannelObserver) GetHighestEstimate() int64 { - return c.estimateTrend.GetHighest() -} - -func (c *ChannelObserver) GetNackRatio() (uint32, uint32, float64) { - ratio := 0.0 - if c.packets != 0 { - ratio = float64(c.repeatedNacks) / float64(c.packets) - if ratio > 1.0 { - ratio = 1.0 - } - } - - return c.packets, c.repeatedNacks, ratio -} - -func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { - estimateDirection := c.estimateTrend.GetDirection() - packets, repeatedNacks, nackRatio := c.GetNackRatio() - - switch { - case estimateDirection == TrendDirectionDownward: - c.logger.Debugw( - "stream allocator: channel observer: estimate is trending downward", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) - return ChannelTrendCongesting, ChannelCongestionReasonEstimate - case c.params.NackWindowMinDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMinDuration && nackRatio > c.params.NackRatioThreshold: - c.logger.Debugw( - "stream allocator: channel observer: high rate of repeated NACKs", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) - return ChannelTrendCongesting, ChannelCongestionReasonLoss - case estimateDirection == TrendDirectionUpward: - return ChannelTrendClearing, ChannelCongestionReasonNone - } - - return ChannelTrendNeutral, ChannelCongestionReasonNone -} - -// ------------------------------------------------ - -type TrendDirection int - -const ( - TrendDirectionNeutral TrendDirection = iota - TrendDirectionUpward - TrendDirectionDownward -) - -func (t TrendDirection) String() string { - switch t { - case TrendDirectionNeutral: - return "NEUTRAL" - case TrendDirectionUpward: - return "UPWARD" - case TrendDirectionDownward: - return "DOWNWARD" - default: - return fmt.Sprintf("%d", int(t)) - } -} - -type TrendDetectorParams struct { - Name string - Logger logger.Logger - RequiredSamples int - DownwardTrendThreshold float64 - CollapseValues bool -} - -type TrendDetector struct { - params TrendDetectorParams - - startTime time.Time - numSamples int - values []int64 - lowestValue int64 - highestValue int64 - - direction TrendDirection -} - -func NewTrendDetector(params TrendDetectorParams) *TrendDetector { - return &TrendDetector{ - params: params, - startTime: time.Now(), - direction: TrendDirectionNeutral, - } -} - -func (t *TrendDetector) Seed(value int64) { - if len(t.values) != 0 { - return - } - - t.values = append(t.values, value) -} - -func (t *TrendDetector) AddValue(value int64) { - t.numSamples++ - if t.lowestValue == 0 || value < t.lowestValue { - t.lowestValue = value - } - if value > t.highestValue { - t.highestValue = value - } - - // ignore duplicate values - if t.params.CollapseValues && len(t.values) != 0 && t.values[len(t.values)-1] == value { - return - } - - if len(t.values) == t.params.RequiredSamples { - t.values = t.values[1:] - } - t.values = append(t.values, value) - - t.updateDirection() -} - -func (t *TrendDetector) GetLowest() int64 { - return t.lowestValue -} - -func (t *TrendDetector) GetHighest() int64 { - return t.highestValue -} - -func (t *TrendDetector) GetValues() []int64 { - return t.values -} - -func (t *TrendDetector) GetDirection() TrendDirection { - return t.direction -} - -func (t *TrendDetector) ToString() string { - now := time.Now() - elapsed := now.Sub(t.startTime).Seconds() - str := fmt.Sprintf("n: %s", t.params.Name) - str += fmt.Sprintf(", t: %+v|%+v|%.2fs", t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) - str += fmt.Sprintf(", v: %d|%d|%d|%+v|%.2f", t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) - return str -} - -func (t *TrendDetector) updateDirection() { - if len(t.values) < t.params.RequiredSamples { - t.direction = TrendDirectionNeutral - return - } - - // using Kendall's Tau to find trend - kt := kendallsTau(t.values) - - t.direction = TrendDirectionNeutral - switch { - case kt > 0: - t.direction = TrendDirectionUpward - case kt < t.params.DownwardTrendThreshold: - t.direction = TrendDirectionDownward - } -} - -// ------------------------------------------------ - -func kendallsTau(values []int64) float64 { - concordantPairs := 0 - discordantPairs := 0 - - for i := 0; i < len(values)-1; i++ { - for j := i + 1; j < len(values); j++ { - if values[i] < values[j] { - concordantPairs++ - } else if values[i] > values[j] { - discordantPairs++ - } - } - } - - if (concordantPairs + discordantPairs) == 0 { - return 0.0 - } - - return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) -} diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go new file mode 100644 index 000000000..e8c432dc8 --- /dev/null +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -0,0 +1,162 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type ChannelTrend int + +const ( + ChannelTrendNeutral ChannelTrend = iota + ChannelTrendClearing + ChannelTrendCongesting +) + +func (c ChannelTrend) String() string { + switch c { + case ChannelTrendNeutral: + return "NEUTRAL" + case ChannelTrendClearing: + return "CLEARING" + case ChannelTrendCongesting: + return "CONGESTING" + default: + return fmt.Sprintf("%d", int(c)) + } +} + +// ------------------------------------------------ + +type ChannelCongestionReason int + +const ( + ChannelCongestionReasonNone ChannelCongestionReason = iota + ChannelCongestionReasonEstimate + ChannelCongestionReasonLoss +) + +func (c ChannelCongestionReason) String() string { + switch c { + case ChannelCongestionReasonNone: + return "NONE" + case ChannelCongestionReasonEstimate: + return "ESTIMATE" + case ChannelCongestionReasonLoss: + return "LOSS" + default: + return fmt.Sprintf("%d", int(c)) + } +} + +// ------------------------------------------------ + +type ChannelObserverParams struct { + Name string + Config config.CongestionControlChannelObserverConfig +} + +type ChannelObserver struct { + params ChannelObserverParams + logger logger.Logger + + estimateTrend *TrendDetector + nackTracker *NackTracker +} + +func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver { + return &ChannelObserver{ + params: params, + logger: logger, + estimateTrend: NewTrendDetector(TrendDetectorParams{ + Name: params.Name + "-estimate", + Logger: logger, + RequiredSamples: params.Config.EstimateRequiredSamples, + DownwardTrendThreshold: params.Config.EstimateDownwardTrendThreshold, + CollapseThreshold: params.Config.EstimateCollapseThreshold, + ValidityWindow: params.Config.EstimateValidityWindow, + }), + nackTracker: NewNackTracker(NackTrackerParams{ + Name: params.Name + "-nack", + Logger: logger, + WindowMinDuration: params.Config.NackWindowMinDuration, + WindowMaxDuration: params.Config.NackWindowMaxDuration, + RatioThreshold: params.Config.NackRatioThreshold, + }), + } +} + +func (c *ChannelObserver) SeedEstimate(estimate int64) { + c.estimateTrend.Seed(estimate) +} + +func (c *ChannelObserver) AddEstimate(estimate int64) { + c.estimateTrend.AddValue(estimate) +} + +func (c *ChannelObserver) AddNack(packets uint32, repeatedNacks uint32) { + c.nackTracker.Add(packets, repeatedNacks) +} + +func (c *ChannelObserver) GetLowestEstimate() int64 { + return c.estimateTrend.GetLowest() +} + +func (c *ChannelObserver) GetHighestEstimate() int64 { + return c.estimateTrend.GetHighest() +} + +func (c *ChannelObserver) HasEnoughEstimateSamples() bool { + return c.estimateTrend.HasEnoughSamples() +} + +func (c *ChannelObserver) GetNackRatio() float64 { + return c.nackTracker.GetRatio() +} + +func (c *ChannelObserver) GetNackHistory() []string { + return c.nackTracker.GetHistory() +} + +func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { + estimateDirection := c.estimateTrend.GetDirection() + + switch { + case estimateDirection == TrendDirectionDownward: + c.logger.Debugw("stream allocator: channel observer: estimate is trending downward", "channel", c.ToString()) + return ChannelTrendCongesting, ChannelCongestionReasonEstimate + + case c.nackTracker.IsTriggered(): + c.logger.Debugw("stream allocator: channel observer: high rate of repeated NACKs", "channel", c.ToString()) + return ChannelTrendCongesting, ChannelCongestionReasonLoss + + case estimateDirection == TrendDirectionUpward: + return ChannelTrendClearing, ChannelCongestionReasonNone + } + + return ChannelTrendNeutral, ChannelCongestionReasonNone +} + +func (c *ChannelObserver) ToString() string { + return fmt.Sprintf("name: %s, estimate: {%s}, nack {%s}", c.params.Name, c.estimateTrend.ToString(), c.nackTracker.ToString()) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go new file mode 100644 index 000000000..b353781e5 --- /dev/null +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -0,0 +1,119 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type NackTrackerParams struct { + Name string + Logger logger.Logger + WindowMinDuration time.Duration + WindowMaxDuration time.Duration + RatioThreshold float64 +} + +type NackTracker struct { + params NackTrackerParams + + windowStartTime time.Time + packets uint32 + repeatedNacks uint32 + + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove when cleaning up experimental stuff + history []string +} + +func NewNackTracker(params NackTrackerParams) *NackTracker { + return &NackTracker{ + params: params, + history: make([]string, 0, 10), + } +} + +func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) { + if n.params.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMaxDuration { + n.updateHistory() + + n.windowStartTime = time.Time{} + n.packets = 0 + n.repeatedNacks = 0 + } + + // + // Start NACK monitoring window only when a repeated NACK happens. + // This allows locking tightly to when NACKs start happening and + // check if the NACKs keep adding up (potentially a sign of congestion) + // or isolated losses + // + if n.repeatedNacks == 0 && repeatedNacks != 0 { + n.windowStartTime = time.Now() + } + + if !n.windowStartTime.IsZero() { + n.packets += packets + n.repeatedNacks += repeatedNacks + } +} + +func (n *NackTracker) GetRatio() float64 { + ratio := 0.0 + if n.packets != 0 { + ratio = float64(n.repeatedNacks) / float64(n.packets) + if ratio > 1.0 { + ratio = 1.0 + } + } + + return ratio +} + +func (n *NackTracker) IsTriggered() bool { + if n.params.WindowMinDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMinDuration { + return n.GetRatio() > n.params.RatioThreshold + } + + return false +} + +func (n *NackTracker) ToString() string { + window := "" + if !n.windowStartTime.IsZero() { + now := time.Now() + elapsed := now.Sub(n.windowStartTime).Seconds() + window = fmt.Sprintf("t: %+v|%+v|%.2fs", n.windowStartTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) + } + return fmt.Sprintf("n: %s, %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) +} + +func (n *NackTracker) GetHistory() []string { + return n.history +} + +func (n *NackTracker) updateHistory() { + if len(n.history) >= 10 { + n.history = n.history[1:] + } + + n.history = append(n.history, n.ToString()) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go new file mode 100644 index 000000000..0d7bb52b7 --- /dev/null +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -0,0 +1,297 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "sync" + "time" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/protocol/logger" +) + +// --------------------------------------------------------------------------- + +type ProbeControllerParams struct { + Config config.CongestionControlProbeConfig + Prober *Prober + Logger logger.Logger +} + +type ProbeController struct { + params ProbeControllerParams + + lock sync.RWMutex + probeInterval time.Duration + lastProbeStartTime time.Time + probeGoalBps int64 + probeClusterId ProbeClusterId + doneProbeClusterInfo ProbeClusterInfo + abortedProbeClusterId ProbeClusterId + goalReachedProbeClusterId ProbeClusterId + probeTrendObserved bool + probeEndTime time.Time + probeDuration time.Duration +} + +func NewProbeController(params ProbeControllerParams) *ProbeController { + p := &ProbeController{ + params: params, + probeDuration: params.Config.MinDuration, + } + + p.Reset() + return p +} + +func (p *ProbeController) Reset() { + p.lock.Lock() + defer p.lock.Unlock() + + p.lastProbeStartTime = time.Now() + + p.resetProbeIntervalLocked() + p.resetProbeDurationLocked() + + p.clearProbeLocked() +} + +func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo) { + p.lock.Lock() + defer p.lock.Unlock() + + if p.probeClusterId != info.Id { + p.params.Logger.Infow("not expected probe cluster", "probeClusterId", p.probeClusterId, "resetProbeClusterId", info.Id) + return + } + + p.doneProbeClusterInfo = info +} + +func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) { + p.lock.Lock() + defer p.lock.Unlock() + + if p.probeClusterId == ProbeClusterIdInvalid { + return + } + + if trend != ChannelTrendNeutral { + p.probeTrendObserved = true + } + + switch { + case !p.probeTrendObserved && time.Since(p.lastProbeStartTime) > p.params.Config.TrendWait: + // + // More of a safety net. + // In rare cases, the estimate gets stuck. Prevent from probe running amok + // STREAM-ALLOCATOR-TODO: Need more testing here to ensure that probe does not cause a lot of damage + // + p.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", p.probeClusterId) + p.abortProbeLocked() + + case trend == ChannelTrendCongesting: + // stop immediately if the probe is congesting channel more + p.params.Logger.Infow("stream allocator: probe: aborting, channel is congesting", "cluster", p.probeClusterId) + p.abortProbeLocked() + + case highestEstimate > p.probeGoalBps: + // reached goal, stop probing + p.params.Logger.Infow( + "stream allocator: probe: stopping, goal reached", + "cluster", p.probeClusterId, + "goal", p.probeGoalBps, + "highest", highestEstimate, + ) + p.goalReachedProbeClusterId = p.probeClusterId + p.StopProbe() + } +} + +func (p *ProbeController) MaybeFinalizeProbe( + isComplete bool, + trend ChannelTrend, + lowestEstimate int64, +) (isHandled bool, isNotFailing bool, isGoalReached bool) { + p.lock.Lock() + defer p.lock.Unlock() + + if !p.isInProbeLocked() { + return false, false, false + } + + if p.goalReachedProbeClusterId != ProbeClusterIdInvalid { + // finalise goal reached probe cluster + p.finalizeProbeLocked(ChannelTrendNeutral) + return true, true, true + } + + if (isComplete || p.abortedProbeClusterId != ProbeClusterIdInvalid) && p.probeEndTime.IsZero() && p.doneProbeClusterInfo.Id != ProbeClusterIdInvalid && p.doneProbeClusterInfo.Id == p.probeClusterId { + // ensure any queueing due to probing is flushed + // STREAM-ALLOCATOR-TODO: CongestionControlProbeConfig.SettleWait should actually be a certain number of RTTs. + expectedDuration := float64(9.0) + if lowestEstimate != 0 { + expectedDuration = float64(p.doneProbeClusterInfo.BytesSent*8*1000) / float64(lowestEstimate) + } + queueTime := expectedDuration - float64(p.doneProbeClusterInfo.Duration.Milliseconds()) + if queueTime < 0.0 { + queueTime = 0.0 + } + queueWait := (time.Duration(queueTime) * time.Millisecond) + p.params.Config.SettleWait + if queueWait > p.params.Config.SettleWaitMax { + queueWait = p.params.Config.SettleWaitMax + } + p.probeEndTime = p.lastProbeStartTime.Add(queueWait + p.doneProbeClusterInfo.Duration) + p.params.Logger.Infow( + "setting probe end time", + "probeClusterId", p.probeClusterId, + "expectedDuration", expectedDuration, + "queueTime", queueTime, + "queueWait", queueWait, + "probeEndTime", p.probeEndTime, + ) + } + + if !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { + // finalisze aborted or non-failing but non-goal-reached probe cluster + return true, p.finalizeProbeLocked(trend), false + } + + return false, false, false +} + +func (p *ProbeController) DoesProbeNeedFinalize() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.abortedProbeClusterId != ProbeClusterIdInvalid || p.goalReachedProbeClusterId != ProbeClusterIdInvalid +} + +func (p *ProbeController) finalizeProbeLocked(trend ChannelTrend) (isNotFailing bool) { + aborted := p.probeClusterId == p.abortedProbeClusterId + + p.clearProbeLocked() + + if aborted || trend == ChannelTrendCongesting { + // failed probe, backoff + p.backoffProbeIntervalLocked() + p.resetProbeDurationLocked() + return false + } + + // reset probe interval and increase probe duration on a upward trending probe + p.resetProbeIntervalLocked() + if trend == ChannelTrendClearing { + p.increaseProbeDurationLocked() + } + return true +} + +func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUsage int64) (ProbeClusterId, int64) { + p.lock.Lock() + defer p.lock.Unlock() + + p.lastProbeStartTime = time.Now() + + // overshoot a bit to account for noise (in measurement/estimate etc) + desiredIncreaseBps := (probeGoalDeltaBps * p.params.Config.OveragePct) / 100 + if desiredIncreaseBps < p.params.Config.MinBps { + desiredIncreaseBps = p.params.Config.MinBps + } + p.probeGoalBps = expectedBandwidthUsage + desiredIncreaseBps + + p.doneProbeClusterInfo = ProbeClusterInfo{Id: ProbeClusterIdInvalid} + p.abortedProbeClusterId = ProbeClusterIdInvalid + p.goalReachedProbeClusterId = ProbeClusterIdInvalid + + p.probeTrendObserved = false + + p.probeEndTime = time.Time{} + + p.probeClusterId = p.params.Prober.AddCluster( + ProbeClusterModeUniform, + int(p.probeGoalBps), + int(expectedBandwidthUsage), + p.probeDuration, + time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationOverflowFactor)*time.Millisecond, + ) + + return p.probeClusterId, p.probeGoalBps +} + +func (p *ProbeController) clearProbeLocked() { + p.probeClusterId = ProbeClusterIdInvalid + p.doneProbeClusterInfo = ProbeClusterInfo{Id: ProbeClusterIdInvalid} + p.abortedProbeClusterId = ProbeClusterIdInvalid + p.goalReachedProbeClusterId = ProbeClusterIdInvalid +} + +func (p *ProbeController) backoffProbeIntervalLocked() { + p.probeInterval = time.Duration(p.probeInterval.Seconds()*p.params.Config.BackoffFactor) * time.Second + if p.probeInterval > p.params.Config.MaxInterval { + p.probeInterval = p.params.Config.MaxInterval + } +} + +func (p *ProbeController) resetProbeIntervalLocked() { + p.probeInterval = p.params.Config.BaseInterval +} + +func (p *ProbeController) resetProbeDurationLocked() { + p.probeDuration = p.params.Config.MinDuration +} + +func (p *ProbeController) increaseProbeDurationLocked() { + p.probeDuration = time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationIncreaseFactor) * time.Millisecond + if p.probeDuration > p.params.Config.MaxDuration { + p.probeDuration = p.params.Config.MaxDuration + } +} + +func (p *ProbeController) StopProbe() { + p.params.Prober.Reset() +} + +func (p *ProbeController) AbortProbe() { + p.lock.Lock() + defer p.lock.Unlock() + + p.abortProbeLocked() +} + +func (p *ProbeController) abortProbeLocked() { + p.abortedProbeClusterId = p.probeClusterId + p.StopProbe() +} + +func (p *ProbeController) IsInProbe() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.isInProbeLocked() +} + +func (p *ProbeController) isInProbeLocked() bool { + return p.probeClusterId != ProbeClusterIdInvalid +} + +func (p *ProbeController) CanProbe() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return time.Since(p.lastProbeStartTime) >= p.probeInterval && p.probeClusterId == ProbeClusterIdInvalid +} + +// ------------------------------------------------ diff --git a/pkg/sfu/prober.go b/pkg/sfu/streamallocator/prober.go similarity index 72% rename from pkg/sfu/prober.go rename to pkg/sfu/streamallocator/prober.go index 42dd61d1e..c30202454 100644 --- a/pkg/sfu/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Design of Prober // // Probing is used to check for existence of excess channel capacity. @@ -103,7 +117,7 @@ // window being long(ish). But, RTT should be much shorter especially if // the subscriber peer connection of the client is able to connect to // the nearest data center. -package sfu +package streamallocator import ( "fmt" @@ -132,7 +146,7 @@ type Prober struct { clusterId atomic.Uint32 clustersMu sync.RWMutex - clusters deque.Deque + clusters deque.Deque[*Cluster] activeCluster *Cluster activeStateQueue []bool activeStateQueueInProcess atomic.Bool @@ -175,7 +189,7 @@ func (p *Prober) Reset() { p.clustersMu.Lock() if p.activeCluster != nil { - p.logger.Debugw("resetting active cluster", "cluster", p.activeCluster.String()) + p.logger.Infow("prober: resetting active cluster", "cluster", p.activeCluster.String()) reset = true info = p.activeCluster.GetInfo() } @@ -195,13 +209,13 @@ func (p *Prober) Reset() { p.processActiveStateQueue() } -func (p *Prober) AddCluster(desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) ProbeClusterId { +func (p *Prober) AddCluster(mode ProbeClusterMode, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) ProbeClusterId { if desiredRateBps <= 0 { return ProbeClusterIdInvalid } clusterId := ProbeClusterId(p.clusterId.Inc()) - cluster := NewCluster(clusterId, desiredRateBps, expectedRateBps, minDuration, maxDuration) + cluster := NewCluster(clusterId, mode, desiredRateBps, expectedRateBps, minDuration, maxDuration) p.logger.Debugw("cluster added", "cluster", cluster.String()) p.pushBackClusterAndMaybeStart(cluster) @@ -238,7 +252,7 @@ func (p *Prober) getFrontCluster() *Cluster { if p.clusters.Len() == 0 { p.activeCluster = nil } else { - p.activeCluster = p.clusters.Front().(*Cluster) + p.activeCluster = p.clusters.Front() p.activeCluster.Start() } return p.activeCluster @@ -253,7 +267,7 @@ func (p *Prober) popFrontCluster(cluster *Cluster) { return } - if p.clusters.Front().(*Cluster) == cluster { + if p.clusters.Front() == cluster { p.clusters.PopFront() } @@ -353,47 +367,117 @@ type ProbeClusterId uint32 const ( ProbeClusterIdInvalid ProbeClusterId = 0 + + bucketDuration = 100 * time.Millisecond + bytesPerProbe = 1000 + minProbeRateBps = 10000 ) +// ----------------------------------- + +type ProbeClusterMode int + +const ( + ProbeClusterModeUniform ProbeClusterMode = iota + ProbeClusterModeLinearChirp +) + +func (p ProbeClusterMode) String() string { + switch p { + case ProbeClusterModeUniform: + return "UNIFORM" + case ProbeClusterModeLinearChirp: + return "LINEAR_CHIRP" + default: + return fmt.Sprintf("%d", int(p)) + } +} + +// --------------------------------------------------------------------------- + type ProbeClusterInfo struct { Id ProbeClusterId BytesSent int Duration time.Duration } +type clusterBucket struct { + desiredBytes int + desiredElapsedTime time.Duration + sleepDuration time.Duration +} + type Cluster struct { lock sync.RWMutex id ProbeClusterId + mode ProbeClusterMode desiredBytes int minDuration time.Duration maxDuration time.Duration - sleepDuration time.Duration + buckets []clusterBucket + bucketIdx int bytesSentProbe int bytesSentNonProbe int startTime time.Time } -func NewCluster(id ProbeClusterId, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) *Cluster { - minDurationMs := minDuration.Milliseconds() - desiredBytes := int((int64(desiredRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) - expectedBytes := int((int64(expectedRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) - - // pace based on sending approximately 1000 bytes per probe - numProbes := (desiredBytes - expectedBytes + 999) / 1000 - sleepDurationMicroSeconds := int(float64(minDurationMs*1000)/float64(numProbes) + 0.5) +func NewCluster(id ProbeClusterId, mode ProbeClusterMode, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) *Cluster { c := &Cluster{ - id: id, - desiredBytes: desiredBytes, - minDuration: minDuration, - maxDuration: maxDuration, - sleepDuration: time.Duration(sleepDurationMicroSeconds) * time.Microsecond, + id: id, + mode: mode, + minDuration: minDuration, + maxDuration: maxDuration, } + c.initBuckets(desiredRateBps, expectedRateBps, minDuration) + c.desiredBytes = c.buckets[len(c.buckets)-1].desiredBytes return c } +func (c *Cluster) initBuckets(desiredRateBps int, expectedRateBps int, minDuration time.Duration) { + // split into granular buckets + // NOTE: splitting even if mode is unitform + numBuckets := int((minDuration.Milliseconds() + bucketDuration.Milliseconds() - 1) / bucketDuration.Milliseconds()) + if numBuckets < 1 { + numBuckets = 1 + } + + expectedRateBytesPerSec := (expectedRateBps + 7) / 8 + baseProbeRateBps := (desiredRateBps - expectedRateBps + numBuckets - 1) / numBuckets + + runningDesiredBytes := 0 + runningDesiredElapsedTime := time.Duration(0) + + c.buckets = make([]clusterBucket, 0, numBuckets) + for bucketIdx := 0; bucketIdx < numBuckets; bucketIdx++ { + multiplier := numBuckets + if c.mode == ProbeClusterModeLinearChirp { + multiplier = bucketIdx + 1 + } + + bucketProbeRateBps := baseProbeRateBps * multiplier + if bucketProbeRateBps < minProbeRateBps { + bucketProbeRateBps = minProbeRateBps + } + bucketProbeRateBytesPerSec := (bucketProbeRateBps + 7) / 8 + + // pace based on bytes per probe + numProbesPerSec := (bucketProbeRateBytesPerSec + bytesPerProbe - 1) / bytesPerProbe + sleepDurationMicroSeconds := int(float64(1_000_000)/float64(numProbesPerSec) + 0.5) + + runningDesiredBytes += (((bucketProbeRateBytesPerSec + expectedRateBytesPerSec) * int(bucketDuration.Milliseconds())) + 999) / 1000 + runningDesiredElapsedTime += bucketDuration + + c.buckets = append(c.buckets, clusterBucket{ + desiredBytes: runningDesiredBytes, + desiredElapsedTime: runningDesiredElapsedTime, + sleepDuration: time.Duration(sleepDurationMicroSeconds) * time.Microsecond, + }) + } +} + func (c *Cluster) Start() { c.lock.Lock() defer c.lock.Unlock() @@ -407,7 +491,7 @@ func (c *Cluster) GetSleepDuration() time.Duration { c.lock.RLock() defer c.lock.RUnlock() - return c.sleepDuration + return c.buckets[c.bucketIdx].sleepDuration } func (c *Cluster) PacketsSent(size int) { @@ -456,7 +540,6 @@ func (c *Cluster) GetInfo() ProbeClusterInfo { func (c *Cluster) Process(pl ProberListener) { c.lock.RLock() - timeElapsed := time.Since(c.startTime) // Calculate number of probe bytes that should have been sent since start. @@ -464,31 +547,32 @@ func (c *Cluster) Process(pl ProberListener) { // However, it is possible that timeElapsed is more than minDuration due // to scheduling variance. When overshooting time budget, use a capped // short fall if there is a grace period given. - windowDone := float64(timeElapsed) / float64(c.minDuration) - if windowDone > 1.0 { - // cluster has been running for longer than minDuration - windowDone = 1.0 - } - - bytesShouldHaveBeenSent := int(windowDone * float64(c.desiredBytes)) - bytesShortFall := bytesShouldHaveBeenSent - c.bytesSentProbe - c.bytesSentNonProbe + bytesShortFall := c.buckets[c.bucketIdx].desiredBytes - c.bytesSentProbe - c.bytesSentNonProbe if bytesShortFall < 0 { bytesShortFall = 0 } - // cap short fall to limit to 8 packets in an iteration + // cap short fall to limit to 5 packets in an iteration // 275 bytes per packet (255 max RTP padding payload + 20 bytes RTP header) - if bytesShortFall > (275 * 8) { - bytesShortFall = 275 * 8 + if bytesShortFall > (275 * 5) { + bytesShortFall = 275 * 5 } // round up to packet size bytesShortFall = ((bytesShortFall + 274) / 275) * 275 + + // move to next bucket if necessary + if timeElapsed > c.buckets[c.bucketIdx].desiredElapsedTime { + c.bucketIdx++ + if c.bucketIdx >= len(c.buckets) { + c.bucketIdx = len(c.buckets) - 1 + } + } c.lock.RUnlock() if bytesShortFall > 0 && pl != nil { pl.OnSendProbe(bytesShortFall) } - // LK-TODO look at adapting sleep time based on how many bytes and how much time is left + // STREAM-ALLOCATOR-TODO look at adapting sleep time based on how many bytes and how much time is left } func (c *Cluster) String() string { @@ -497,8 +581,9 @@ func (c *Cluster) String() string { activeTimeMs = time.Since(c.startTime).Milliseconds() } - return fmt.Sprintf("id: %d, bytes: desired %d / probe %d / non-probe %d / remaining: %d, time(ms): active %d / min %d / max %d", + return fmt.Sprintf("id: %d, mode: %s, bytes: desired %d / probe %d / non-probe %d / remaining: %d, time(ms): active %d / min %d / max %d", c.id, + c.mode, c.desiredBytes, c.bytesSentProbe, c.bytesSentNonProbe, @@ -507,3 +592,5 @@ func (c *Cluster) String() string { c.minDuration.Milliseconds(), c.maxDuration.Milliseconds()) } + +// ---------------------------------------------------------------------- diff --git a/pkg/sfu/streamallocator/ratemonitor.go b/pkg/sfu/streamallocator/ratemonitor.go new file mode 100644 index 000000000..9c2ba243e --- /dev/null +++ b/pkg/sfu/streamallocator/ratemonitor.go @@ -0,0 +1,172 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/utils/timeseries" +) + +// ------------------------------------------------ + +const ( + rateMonitorWindow = 10 * time.Second + queueMonitorWindow = 2 * time.Second +) + +// ------------------------------------------------ + +type RateMonitor struct { + bitrateEstimate *timeseries.TimeSeries[int64] + managedBytesSent *timeseries.TimeSeries[uint32] + managedBytesRetransmitted *timeseries.TimeSeries[uint32] + unmanagedBytesSent *timeseries.TimeSeries[uint32] + unmanagedBytesRetransmitted *timeseries.TimeSeries[uint32] + + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + history []string +} + +func NewRateMonitor() *RateMonitor { + return &RateMonitor{ + bitrateEstimate: timeseries.NewTimeSeries[int64](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpLatest, + Window: rateMonitorWindow, + }), + managedBytesSent: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + managedBytesRetransmitted: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + unmanagedBytesSent: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + unmanagedBytesRetransmitted: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + } +} + +func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedBytesRetransmitted uint32, unmanagedBytesSent uint32, unmanagedBytesRetransmitted uint32) { + now := time.Now() + r.bitrateEstimate.AddSampleAt(estimate, now) + r.managedBytesSent.AddSampleAt(managedBytesSent, now) + r.managedBytesRetransmitted.AddSampleAt(managedBytesRetransmitted, now) + r.unmanagedBytesSent.AddSampleAt(unmanagedBytesSent, now) + r.unmanagedBytesRetransmitted.AddSampleAt(unmanagedBytesRetransmitted, now) + + r.updateHistory() +} + +// STREAM-ALLOCATOR-TODO: +// This should be updated periodically to flush any pending. +// Reason is that the estimate could be higher than the actual rate by a significant amount. +// So, updating periodically to flush out samples that will not contribute to queueing would be good. +func (r *RateMonitor) GetQueuingGuess() float64 { + _, _, _, _, _, qd := r.getRates(queueMonitorWindow) + return qd +} + +func (r *RateMonitor) getRates(monitorDuration time.Duration) (float64, float64, float64, float64, float64, float64) { + threshold := time.Now().Add(-monitorDuration) + bitrateEstimateSamples := r.bitrateEstimate.GetSamplesAfter(threshold) + managedBytesSentSamples := r.managedBytesSent.GetSamplesAfter(threshold) + managedBytesRetransmittedSamples := r.managedBytesRetransmitted.GetSamplesAfter(threshold) + unmanagedBytesSentSamples := r.unmanagedBytesSent.GetSamplesAfter(threshold) + unmanagedBytesRetransmittedSamples := r.unmanagedBytesRetransmitted.GetSamplesAfter(threshold) + + if len(bitrateEstimateSamples) == 0 || (len(managedBytesSentSamples)+len(managedBytesRetransmittedSamples)+len(unmanagedBytesSentSamples)+len(unmanagedBytesRetransmittedSamples)) == 0 { + return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + } + + totalBitrateEstimate := getTimeWeightedSum(bitrateEstimateSamples) + totalManagedSent := getRate(managedBytesSentSamples) * 8 + totalManagedRetransmitted := getRate(managedBytesRetransmittedSamples) * 8 + totalUnmanagedSent := getRate(unmanagedBytesSentSamples) * 8 + totalUnmanagedRetransmitted := getRate(unmanagedBytesRetransmittedSamples) * 8 + totalBits := totalManagedSent + totalManagedRetransmitted + totalUnmanagedSent + totalUnmanagedRetransmitted + + queuingDelay := float64(0.0) + if totalBits > totalBitrateEstimate { + latestBitrateEstimate := bitrateEstimateSamples[len(bitrateEstimateSamples)-1].Value + excessBits := totalBits - totalBitrateEstimate + queuingDelay = excessBits / float64(latestBitrateEstimate) + } + return totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay +} + +func (r *RateMonitor) updateHistory() { + if len(r.history) >= 10 { + r.history = r.history[1:] + } + + e, m, mr, um, umr, qd := r.getRates(time.Second) + if e == 0.0 { + return + } + + r.history = append( + r.history, + fmt.Sprintf("t: %+v, e: %.2f, m: %.2f/%.2f, um: %.2f/%.2f, qd: %.2f", time.Now().UnixMilli(), e, m, mr, um, umr, qd), + ) +} + +func (r *RateMonitor) GetHistory() []string { + return r.history +} + +// ------------------------------------------------ + +func getTimeWeightedSum[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { + if len(samples) < 2 { + return 0.0 + } + + sum := 0.0 + for i := 1; i < len(samples); i++ { + diff := samples[i].At.Sub(samples[i-1].At).Seconds() + sum += diff * float64(samples[i-1].Value) + } + + diff := time.Now().Sub(samples[len(samples)-1].At).Seconds() + sum += diff * float64(samples[len(samples)-1].Value) + return sum +} + +func getRate[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { + if len(samples) < 2 { + return 0.0 + } + + sum := 0.0 + // start at 1 as the first sample duration is not available + for i := 1; i < len(samples); i++ { + sum += float64(samples[i].Value) + } + + duration := samples[len(samples)-1].At.Sub(samples[0].At) + if duration == 0 { + return 0.0 + } + + return sum / duration.Seconds() +} diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go new file mode 100644 index 000000000..aa274be3e --- /dev/null +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -0,0 +1,1416 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + "sort" + "sync" + "time" + + "github.com/pion/interceptor/pkg/cc" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" + "go.uber.org/atomic" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +const ( + ChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps + + NackRatioAttenuator = 0.4 // how much to attenuate NACK ratio while calculating loss adjusted estimate + + PriorityMin = uint8(1) + PriorityMax = uint8(255) + PriorityDefaultScreenshare = PriorityMax + PriorityDefaultVideo = PriorityMin + + FlagAllowOvershootWhileOptimal = true + FlagAllowOvershootWhileDeficient = false + FlagAllowOvershootExemptTrackWhileDeficient = true + FlagAllowOvershootInProbe = true + FlagAllowOvershootInCatchup = false + FlagAllowOvershootInBoost = true +) + +// --------------------------------------------------------------------------- + +type streamAllocatorState int + +const ( + streamAllocatorStateStable streamAllocatorState = iota + streamAllocatorStateDeficient +) + +func (s streamAllocatorState) String() string { + switch s { + case streamAllocatorStateStable: + return "STABLE" + case streamAllocatorStateDeficient: + return "DEFICIENT" + default: + return fmt.Sprintf("UNKNOWN: %d", int(s)) + } +} + +// --------------------------------------------------------------------------- + +type streamAllocatorSignal int + +const ( + streamAllocatorSignalAllocateTrack streamAllocatorSignal = iota + streamAllocatorSignalAllocateAllTracks + streamAllocatorSignalAdjustState + streamAllocatorSignalEstimate + streamAllocatorSignalPeriodicPing + streamAllocatorSignalSendProbe + streamAllocatorSignalProbeClusterDone + streamAllocatorSignalResume + streamAllocatorSignalSetAllowPause + streamAllocatorSignalSetChannelCapacity + streamAllocatorSignalNACK + streamAllocatorSignalRTCPReceiverReport +) + +func (s streamAllocatorSignal) String() string { + switch s { + case streamAllocatorSignalAllocateTrack: + return "ALLOCATE_TRACK" + case streamAllocatorSignalAllocateAllTracks: + return "ALLOCATE_ALL_TRACKS" + case streamAllocatorSignalAdjustState: + return "ADJUST_STATE" + case streamAllocatorSignalEstimate: + return "ESTIMATE" + case streamAllocatorSignalPeriodicPing: + return "PERIODIC_PING" + case streamAllocatorSignalSendProbe: + return "SEND_PROBE" + case streamAllocatorSignalProbeClusterDone: + return "PROBE_CLUSTER_DONE" + case streamAllocatorSignalResume: + return "RESUME" + case streamAllocatorSignalSetAllowPause: + return "SET_ALLOW_PAUSE" + case streamAllocatorSignalSetChannelCapacity: + return "SET_CHANNEL_CAPACITY" + case streamAllocatorSignalNACK: + return "NACK" + case streamAllocatorSignalRTCPReceiverReport: + return "RTCP_RECEIVER_REPORT" + default: + return fmt.Sprintf("%d", int(s)) + } +} + +// --------------------------------------------------------------------------- + +type Event struct { + Signal streamAllocatorSignal + TrackID livekit.TrackID + Data interface{} +} + +func (e Event) String() string { + return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data) +} + +// --------------------------------------------------------------------------- + +type StreamAllocatorParams struct { + Config config.CongestionControlConfig + Logger logger.Logger +} + +type StreamAllocator struct { + params StreamAllocatorParams + + onStreamStateChange func(update *StreamStateUpdate) error + + bwe cc.BandwidthEstimator + + allowPause bool + + lastReceivedEstimate int64 + committedChannelCapacity int64 + overriddenChannelCapacity int64 + + probeController *ProbeController + + prober *Prober + + channelObserver *ChannelObserver + rateMonitor *RateMonitor + + videoTracksMu sync.RWMutex + videoTracks map[livekit.TrackID]*Track + isAllocateAllPending bool + rembTrackingSSRC uint32 + + state streamAllocatorState + + eventChMu sync.RWMutex + eventCh chan Event + + isStopped atomic.Bool +} + +func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { + s := &StreamAllocator{ + params: params, + allowPause: params.Config.AllowPause, + prober: NewProber(ProberParams{ + Logger: params.Logger, + }), + rateMonitor: NewRateMonitor(), + videoTracks: make(map[livekit.TrackID]*Track), + eventCh: make(chan Event, 1000), + } + + s.probeController = NewProbeController(ProbeControllerParams{ + Config: s.params.Config.ProbeConfig, + Prober: s.prober, + Logger: params.Logger, + }) + + s.resetState() + + s.prober.SetProberListener(s) + + return s +} + +func (s *StreamAllocator) Start() { + go s.processEvents() + go s.ping() +} + +func (s *StreamAllocator) Stop() { + s.eventChMu.Lock() + if s.isStopped.Swap(true) { + s.eventChMu.Unlock() + return + } + + close(s.eventCh) + s.eventChMu.Unlock() +} + +func (s *StreamAllocator) OnStreamStateChange(f func(update *StreamStateUpdate) error) { + s.onStreamStateChange = f +} + +func (s *StreamAllocator) SetBandwidthEstimator(bwe cc.BandwidthEstimator) { + if bwe != nil { + bwe.OnTargetBitrateChange(s.onTargetBitrateChange) + } + s.bwe = bwe +} + +type AddTrackParams struct { + Source livekit.TrackSource + Priority uint8 + IsSimulcast bool + PublisherID livekit.ParticipantID +} + +func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackParams) { + if downTrack.Kind() != webrtc.RTPCodecTypeVideo { + return + } + + track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) + track.SetPriority(params.Priority) + + s.videoTracksMu.Lock() + s.videoTracks[livekit.TrackID(downTrack.ID())] = track + s.videoTracksMu.Unlock() + + downTrack.SetStreamAllocatorListener(s) + if s.prober.IsRunning() { + // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate + downTrack.SetStreamAllocatorReportInterval(50 * time.Millisecond) + } + + s.maybePostEventAllocateTrack(downTrack) +} + +func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) { + s.videoTracksMu.Lock() + if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack { + delete(s.videoTracks, livekit.TrackID(downTrack.ID())) + } + s.videoTracksMu.Unlock() + + // STREAM-ALLOCATOR-TODO: use any saved bandwidth to re-distribute + s.postEvent(Event{ + Signal: streamAllocatorSignalAdjustState, + }) +} + +func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority uint8) { + s.videoTracksMu.Lock() + if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { + changed := track.SetPriority(priority) + if changed && !s.isAllocateAllPending { + // do a full allocation on a track priority change to keep it simple + s.isAllocateAllPending = true + s.postEvent(Event{ + Signal: streamAllocatorSignalAllocateAllTracks, + }) + } + } + s.videoTracksMu.Unlock() +} + +func (s *StreamAllocator) SetAllowPause(allowPause bool) { + s.postEvent(Event{ + Signal: streamAllocatorSignalSetAllowPause, + Data: allowPause, + }) +} + +func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) { + s.postEvent(Event{ + Signal: streamAllocatorSignalSetChannelCapacity, + Data: channelCapacity, + }) +} + +func (s *StreamAllocator) resetState() { + s.channelObserver = s.newChannelObserverNonProbe() + s.probeController.Reset() + + s.state = streamAllocatorStateStable +} + +// called when a new REMB is received (receive side bandwidth estimation) +func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) { + // + // Channel capacity is estimated at a peer connection level. All down tracks + // in the peer connection will end up calling this for a REMB report with + // the same estimated channel capacity. Use a tracking SSRC to lock onto to + // one report. As SSRCs can be dropped over time, update tracking SSRC as needed + // + // A couple of things to keep in mind + // - REMB reports could be sent gratuitously as a way of providing + // periodic feedback, i.e. even if the estimated capacity does not + // change, there could be REMB packets on the wire. Those gratuitous + // REMBs should not trigger anything bad. + // - As each down track will issue this callback for the same REMB packet + // from the wire, theoretically it is possible that one down track's + // callback from previous REMB comes after another down track's callback + // from the new REMB. REMBs could fire very quickly especially when + // the network is entering congestion. + // STREAM-ALLOCATOR-TODO-START + // Need to check if the same SSRC reports can somehow race, i.e. does pion send + // RTCP dispatch for same SSRC on different threads? If not, the tracking SSRC + // should prevent racing + // STREAM-ALLOCATOR-TODO-END + // + + // if there are no video tracks, ignore any straggler REMB + s.videoTracksMu.Lock() + if len(s.videoTracks) == 0 { + s.videoTracksMu.Unlock() + return + } + + track := s.videoTracks[livekit.TrackID(downTrack.ID())] + downTrackSSRC := uint32(0) + if track != nil { + downTrackSSRC = track.DownTrack().SSRC() + } + + found := false + for _, ssrc := range remb.SSRCs { + if ssrc == s.rembTrackingSSRC { + found = true + break + } + } + if !found { + if len(remb.SSRCs) == 0 { + s.params.Logger.Warnw("stream allocator: no SSRC to track REMB", nil) + s.videoTracksMu.Unlock() + return + } + + // try to lock to track which is sending this update + if downTrackSSRC != 0 { + for _, ssrc := range remb.SSRCs { + if ssrc == downTrackSSRC { + s.rembTrackingSSRC = downTrackSSRC + found = true + break + } + } + } + + if !found { + s.rembTrackingSSRC = remb.SSRCs[0] + } + } + + if s.rembTrackingSSRC == 0 || s.rembTrackingSSRC != downTrackSSRC { + s.videoTracksMu.Unlock() + return + } + s.videoTracksMu.Unlock() + + s.postEvent(Event{ + Signal: streamAllocatorSignalEstimate, + Data: int64(remb.Bitrate), + }) +} + +// called when a new transport-cc feedback is received +func (s *StreamAllocator) OnTransportCCFeedback(downTrack *sfu.DownTrack, fb *rtcp.TransportLayerCC) { + if s.bwe != nil { + s.bwe.WriteRTCP([]rtcp.Packet{fb}, nil) + } +} + +// called when target bitrate changes (send side bandwidth estimation) +func (s *StreamAllocator) onTargetBitrateChange(bitrate int) { + s.postEvent(Event{ + Signal: streamAllocatorSignalEstimate, + Data: int64(bitrate), + }) +} + +// called when feeding track's layer availability changes +func (s *StreamAllocator) OnAvailableLayersChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when feeding track's bitrate measurement of any layer is available +func (s *StreamAllocator) OnBitrateAvailabilityChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when feeding track's max published spatial layer changes +func (s *StreamAllocator) OnMaxPublishedSpatialChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when feeding track's max published temporal layer changes +func (s *StreamAllocator) OnMaxPublishedTemporalChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when subscription settings changes (muting/unmuting of track) +func (s *StreamAllocator) OnSubscriptionChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when subscribed layer changes (limiting max layer) +func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, layer buffer.VideoLayer) { + shouldPost := false + s.videoTracksMu.Lock() + if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { + if track.SetMaxLayer(layer) && track.SetDirty(true) { + shouldPost = true + } + } + s.videoTracksMu.Unlock() + + if shouldPost { + s.postEvent(Event{ + Signal: streamAllocatorSignalAllocateTrack, + TrackID: livekit.TrackID(downTrack.ID()), + }) + } +} + +// called when forwarder resumes a track +func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) { + s.postEvent(Event{ + Signal: streamAllocatorSignalResume, + TrackID: livekit.TrackID(downTrack.ID()), + }) +} + +// called by a video DownTrack to report packet send +func (s *StreamAllocator) OnPacketsSent(downTrack *sfu.DownTrack, size int) { + s.prober.PacketsSent(size) +} + +// called by a video DownTrack when it processes NACKs +func (s *StreamAllocator) OnNACK(downTrack *sfu.DownTrack, nackInfos []sfu.NackInfo) { + s.postEvent(Event{ + Signal: streamAllocatorSignalNACK, + TrackID: livekit.TrackID(downTrack.ID()), + Data: nackInfos, + }) +} + +// called by a video DownTrack when it receives an RTCP Receiver Report +// STREAM-ALLOCATOR-TODO: this should probably be done for audio tracks also +func (s *StreamAllocator) OnRTCPReceiverReport(downTrack *sfu.DownTrack, rr rtcp.ReceptionReport) { + s.postEvent(Event{ + Signal: streamAllocatorSignalRTCPReceiverReport, + TrackID: livekit.TrackID(downTrack.ID()), + Data: rr, + }) +} + +// called when prober wants to send packet(s) +func (s *StreamAllocator) OnSendProbe(bytesToSend int) { + s.postEvent(Event{ + Signal: streamAllocatorSignalSendProbe, + Data: bytesToSend, + }) +} + +// called when prober finishes a probe cluster, could be called when prober is reset which stops an active cluster +func (s *StreamAllocator) OnProbeClusterDone(info ProbeClusterInfo) { + s.postEvent(Event{ + Signal: streamAllocatorSignalProbeClusterDone, + Data: info, + }) +} + +// called when prober active state changes +func (s *StreamAllocator) OnActiveChanged(isActive bool) { + for _, t := range s.getTracks() { + if isActive { + // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate + t.DownTrack().SetStreamAllocatorReportInterval(50 * time.Millisecond) + } else { + t.DownTrack().ClearStreamAllocatorReportInterval() + } + } +} + +func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) { + shouldPost := false + s.videoTracksMu.Lock() + if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { + if track.SetDirty(true) { + shouldPost = true + } + } + s.videoTracksMu.Unlock() + + if shouldPost { + s.postEvent(Event{ + Signal: streamAllocatorSignalAllocateTrack, + TrackID: livekit.TrackID(downTrack.ID()), + }) + } +} + +func (s *StreamAllocator) postEvent(event Event) { + s.eventChMu.RLock() + if s.isStopped.Load() { + s.eventChMu.RUnlock() + return + } + + select { + case s.eventCh <- event: + default: + s.params.Logger.Warnw("stream allocator: event queue full", nil) + } + s.eventChMu.RUnlock() +} + +func (s *StreamAllocator) processEvents() { + for event := range s.eventCh { + if s.isStopped.Load() { + break + } + + s.handleEvent(&event) + } + + s.probeController.StopProbe() +} + +func (s *StreamAllocator) ping() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + <-ticker.C + if s.isStopped.Load() { + return + } + + s.postEvent(Event{ + Signal: streamAllocatorSignalPeriodicPing, + }) + } +} + +func (s *StreamAllocator) handleEvent(event *Event) { + switch event.Signal { + case streamAllocatorSignalAllocateTrack: + s.handleSignalAllocateTrack(event) + case streamAllocatorSignalAllocateAllTracks: + s.handleSignalAllocateAllTracks(event) + case streamAllocatorSignalAdjustState: + s.handleSignalAdjustState(event) + case streamAllocatorSignalEstimate: + s.handleSignalEstimate(event) + case streamAllocatorSignalPeriodicPing: + s.handleSignalPeriodicPing(event) + case streamAllocatorSignalSendProbe: + s.handleSignalSendProbe(event) + case streamAllocatorSignalProbeClusterDone: + s.handleSignalProbeClusterDone(event) + case streamAllocatorSignalResume: + s.handleSignalResume(event) + case streamAllocatorSignalSetAllowPause: + s.handleSignalSetAllowPause(event) + case streamAllocatorSignalSetChannelCapacity: + s.handleSignalSetChannelCapacity(event) + case streamAllocatorSignalNACK: + s.handleSignalNACK(event) + case streamAllocatorSignalRTCPReceiverReport: + s.handleSignalRTCPReceiverReport(event) + } +} + +func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + if track != nil { + track.SetDirty(false) + } + s.videoTracksMu.Unlock() + + if track != nil { + s.allocateTrack(track) + } +} + +func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { + s.videoTracksMu.Lock() + s.isAllocateAllPending = false + s.videoTracksMu.Unlock() + + if s.state == streamAllocatorStateDeficient { + s.allocateAllTracks() + } +} + +func (s *StreamAllocator) handleSignalAdjustState(event *Event) { + s.adjustState() +} + +func (s *StreamAllocator) handleSignalEstimate(event *Event) { + receivedEstimate, _ := event.Data.(int64) + s.lastReceivedEstimate = receivedEstimate + s.monitorRate(receivedEstimate) + + // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails + if s.probeController.IsInProbe() { + s.handleNewEstimateInProbe() + } else { + s.handleNewEstimateInNonProbe() + } +} + +func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { + // finalize probe if necessary + trend, _ := s.channelObserver.GetTrend() + isHandled, isNotFailing, isGoalReached := s.probeController.MaybeFinalizeProbe( + s.channelObserver.HasEnoughEstimateSamples(), + trend, + s.channelObserver.GetLowestEstimate(), + ) + if isHandled { + s.onProbeDone(isNotFailing, isGoalReached) + } + + // probe if necessary and timing is right + if s.state == streamAllocatorStateDeficient { + s.maybeProbe() + } + + s.updateTracksHistory() +} + +func (s *StreamAllocator) handleSignalSendProbe(event *Event) { + bytesToSend := event.Data.(int) + if bytesToSend <= 0 { + return + } + + bytesSent := 0 + for _, track := range s.getTracks() { + sent := track.WritePaddingRTP(bytesToSend) + bytesSent += sent + bytesToSend -= sent + if bytesToSend <= 0 { + break + } + } + + if bytesSent != 0 { + s.prober.ProbeSent(bytesSent) + } +} + +func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { + info, _ := event.Data.(ProbeClusterInfo) + s.probeController.ProbeClusterDone(info) +} + +func (s *StreamAllocator) handleSignalResume(event *Event) { + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + s.videoTracksMu.Unlock() + + if track != nil { + update := NewStreamStateUpdate() + if track.SetStreamState(StreamStateActive) { + update.HandleStreamingChange(track, StreamStateActive) + } + s.maybeSendUpdate(update) + } +} + +func (s *StreamAllocator) handleSignalSetAllowPause(event *Event) { + s.allowPause = event.Data.(bool) +} + +func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { + s.overriddenChannelCapacity = event.Data.(int64) + if s.overriddenChannelCapacity > 0 { + s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) + s.allocateAllTracks() + } else { + s.params.Logger.Infow("clearing override channel capacity") + } +} + +func (s *StreamAllocator) handleSignalNACK(event *Event) { + nackInfos := event.Data.([]sfu.NackInfo) + + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + s.videoTracksMu.Unlock() + + if track != nil { + track.UpdateNack(nackInfos) + } +} + +func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { + rr := event.Data.(rtcp.ReceptionReport) + + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + s.videoTracksMu.Unlock() + + if track != nil { + track.ProcessRTCPReceiverReport(rr) + } +} + +func (s *StreamAllocator) setState(state streamAllocatorState) { + if s.state == state { + return + } + + s.params.Logger.Infow("stream allocator: state change", "from", s.state, "to", state) + s.state = state + + // reset probe to enforce a delay after state change before probing + s.probeController.Reset() + // a fresh channel observer after state transition to get clean data + s.channelObserver = s.newChannelObserverNonProbe() +} + +func (s *StreamAllocator) adjustState() { + for _, track := range s.getTracks() { + if track.IsDeficient() { + s.setState(streamAllocatorStateDeficient) + return + } + } + + s.setState(streamAllocatorStateStable) +} + +func (s *StreamAllocator) handleNewEstimateInProbe() { + // always update NACKs, even if aborted + packetDelta, repeatedNackDelta := s.getNackDelta() + + if s.probeController.DoesProbeNeedFinalize() { + // waiting for aborted probe to finalize + return + } + + s.channelObserver.AddEstimate(s.lastReceivedEstimate) + s.channelObserver.AddNack(packetDelta, repeatedNackDelta) + + trend, _ := s.channelObserver.GetTrend() + s.probeController.CheckProbe(trend, s.channelObserver.GetHighestEstimate()) +} + +func (s *StreamAllocator) handleNewEstimateInNonProbe() { + s.channelObserver.AddEstimate(s.lastReceivedEstimate) + + packetDelta, repeatedNackDelta := s.getNackDelta() + s.channelObserver.AddNack(packetDelta, repeatedNackDelta) + + trend, reason := s.channelObserver.GetTrend() + if trend != ChannelTrendCongesting { + return + } + + var estimateToCommit int64 + expectedBandwidthUsage := s.getExpectedBandwidthUsage() + switch reason { + case ChannelCongestionReasonLoss: + estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*s.channelObserver.GetNackRatio())) + if estimateToCommit > s.lastReceivedEstimate { + estimateToCommit = s.lastReceivedEstimate + } + default: + estimateToCommit = s.lastReceivedEstimate + } + + s.params.Logger.Infow( + "stream allocator: channel congestion detected, updating channel capacity", + "reason", reason, + "old(bps)", s.committedChannelCapacity, + "new(bps)", estimateToCommit, + "lastReceived(bps)", s.lastReceivedEstimate, + "expectedUsage(bps)", expectedBandwidthUsage, + "channel", s.channelObserver.ToString(), + ) + s.params.Logger.Infow( + "stream allocator: channel congestion detected, updating channel capacity: experimental", + "rateHistory", s.rateMonitor.GetHistory(), + "expectedQueuing", s.rateMonitor.GetQueuingGuess(), + "nackHistory", s.channelObserver.GetNackHistory(), + "trackHistory", s.getTracksHistory(), + ) + s.committedChannelCapacity = estimateToCommit + + // reset to get new set of samples for next trend + s.channelObserver = s.newChannelObserverNonProbe() + + // reset probe to ensure it does not start too soon after a downward trend + s.probeController.Reset() + + s.allocateAllTracks() +} + +func (s *StreamAllocator) allocateTrack(track *Track) { + // abort any probe that may be running when a track specific change needs allocation + s.probeController.AbortProbe() + + // if not deficient, free pass allocate track + if !s.params.Config.Enabled || s.state == streamAllocatorStateStable || !track.IsManaged() { + update := NewStreamStateUpdate() + allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal) + updateStreamStateChange(track, allocation, update) + s.maybeSendUpdate(update) + return + } + + // + // In DEFICIENT state, + // Two possibilities + // 1. Available headroom is enough to accommodate track that needs change. + // Note that the track could be muted, hence stopping. + // 2. Have to steal bits from other tracks currently streaming. + // + // For both cases, do + // a. Find cooperative transition from track that needs allocation. + // b. If track is currently streaming at minimum, do not do anything. + // c. If track is giving back bits, apply the transition and use bits given + // back to boost any deficient track(s). + // + // If track needs more bits, i.e. upward transition (may need resume or higher layer subscription), + // a. Try to allocate using existing headroom. This can be tried to get the best + // possible fit for the available headroom. + // b. If there is not enough headroom to allocate anything, ask for best offer from + // other tracks that are currently streaming and try to use it. + // + track.ProvisionalAllocatePrepare() + transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) + + // track is currently streaming at minimum + if transition.BandwidthDelta == 0 { + return + } + + // downgrade, giving back bits + if transition.From.GreaterThan(transition.To) { + allocation := track.ProvisionalAllocateCommit() + + update := NewStreamStateUpdate() + updateStreamStateChange(track, allocation, update) + s.maybeSendUpdate(update) + + s.adjustState() + + // Use the bits given back to boost deficient track(s). + // Note layer downgrade may actually have positive delta (i.e. consume more bits) + // because of when the measurement is done. But, only available headroom after + // applying the transition will be used to boost deficient track(s). + s.maybeBoostDeficientTracks() + return + } + + // this track is currently not streaming and needs bits to start. + // first try an allocation using available headroom + availableChannelCapacity := s.getAvailableHeadroom(false) + if availableChannelCapacity > 0 { + track.ProvisionalAllocateReset() // to reset allocation from co-operative transition above and try fresh + + bestLayer := buffer.InvalidLayer + + alloc_loop: + for spatial := int32(0); spatial <= buffer.DefaultMaxLayerSpatial; spatial++ { + for temporal := int32(0); temporal <= buffer.DefaultMaxLayerTemporal; temporal++ { + layer := buffer.VideoLayer{ + Spatial: spatial, + Temporal: temporal, + } + + usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient) + if availableChannelCapacity < usedChannelCapacity { + break alloc_loop + } + + bestLayer = layer + } + } + + if bestLayer.IsValid() { + // found layer that can fit in available headroom + update := NewStreamStateUpdate() + allocation := track.ProvisionalAllocateCommit() + updateStreamStateChange(track, allocation, update) + s.maybeSendUpdate(update) + + s.adjustState() + return + } + + track.ProvisionalAllocateReset() + transition = track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom + } + + // if there is not enough headroom, try to redistribute starting with tracks that are closest to their desired. + bandwidthAcquired := int64(0) + var contributingTracks []*Track + + minDistanceSorted := s.getMinDistanceSorted(track) + for _, t := range minDistanceSorted { + t.ProvisionalAllocatePrepare() + } + + for _, t := range minDistanceSorted { + tx := t.ProvisionalAllocateGetBestWeightedTransition() + if tx.BandwidthDelta < 0 { + contributingTracks = append(contributingTracks, t) + + bandwidthAcquired += -tx.BandwidthDelta + if bandwidthAcquired >= transition.BandwidthDelta { + break + } + } + } + + update := NewStreamStateUpdate() + if bandwidthAcquired >= transition.BandwidthDelta { + // commit the tracks that contributed + for _, t := range contributingTracks { + allocation := t.ProvisionalAllocateCommit() + updateStreamStateChange(t, allocation, update) + } + + // STREAM-ALLOCATOR-TODO if got too much extra, can potentially give it to some deficient track + } + + // commit the track that needs change if enough could be acquired or pause not allowed + if !s.allowPause || bandwidthAcquired >= transition.BandwidthDelta { + allocation := track.ProvisionalAllocateCommit() + updateStreamStateChange(track, allocation, update) + } else { + // explicitly pause to ensure stream state update happens if a track coming out of mute cannot be allocated + allocation := track.Pause() + updateStreamStateChange(track, allocation, update) + } + + s.maybeSendUpdate(update) + + s.adjustState() +} + +func (s *StreamAllocator) onProbeDone(isNotFailing bool, isGoalReached bool) { + highestEstimateInProbe := s.channelObserver.GetHighestEstimate() + + // + // Reset estimator at the end of a probe irrespective of probe result to get fresh readings. + // With a failed probe, the latest estimate could be lower than committed estimate. + // As bandwidth estimator (remote in REMB case, local in TWCC case) holds state, + // subsequent estimates could start from the lower point. That should not trigger a + // downward trend and get latched to committed estimate as that would trigger a re-allocation. + // With fresh readings, as long as the trend is not going downward, it will not get latched. + // + // NOTE: With TWCC, it is possible to reset bandwidth estimation to clean state as + // the send side is in full control of bandwidth estimation. + // + channelObserverString := s.channelObserver.ToString() + s.channelObserver = s.newChannelObserverNonProbe() + s.params.Logger.Infow( + "probe done", + "isNotFailing", isNotFailing, + "isGoalReached", isGoalReached, + "committedEstimate", s.committedChannelCapacity, + "highestEstimate", highestEstimateInProbe, + "channel", channelObserverString, + ) + if !isNotFailing { + return + } + + if highestEstimateInProbe > s.committedChannelCapacity { + s.committedChannelCapacity = highestEstimateInProbe + } + + s.maybeBoostDeficientTracks() +} + +func (s *StreamAllocator) maybeBoostDeficientTracks() { + availableChannelCapacity := s.getAvailableHeadroom(false) + if availableChannelCapacity <= 0 { + return + } + + update := NewStreamStateUpdate() + + for _, track := range s.getMaxDistanceSortedDeficient() { + allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, FlagAllowOvershootInCatchup) + if !boosted { + continue + } + + updateStreamStateChange(track, allocation, update) + + availableChannelCapacity -= allocation.BandwidthDelta + if availableChannelCapacity <= 0 { + break + } + } + + s.maybeSendUpdate(update) + + s.adjustState() +} + +func (s *StreamAllocator) allocateAllTracks() { + if !s.params.Config.Enabled { + // nothing else to do when disabled + return + } + + // + // Goals: + // 1. Stream as many tracks as possible, i.e. no pauses. + // 2. Try to give fair allocation to all track. + // + // Start with the lowest layer and give each track a chance at that layer and keep going up. + // As long as there is enough bandwidth for tracks to stream at the lowest layer, the first goal is achieved. + // + // Tracks that have higher subscribed layer can use any additional available bandwidth. This tried to achieve the second goal. + // + // If there is not enough bandwidth even for the lowest layer, tracks at lower priorities will be paused. + // + update := NewStreamStateUpdate() + + availableChannelCapacity := s.getAvailableChannelCapacity(true) + + // + // This pass is to find out if there is any leftover channel capacity after allocating exempt tracks. + // Exempt tracks are given optimal allocation (i. e. no bandwidth constraint) so that they do not fail allocation. + // + videoTracks := s.getTracks() + for _, track := range videoTracks { + if track.IsManaged() { + continue + } + + allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient) + updateStreamStateChange(track, allocation, update) + + // STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? + availableChannelCapacity -= allocation.BandwidthRequested + } + + if availableChannelCapacity < 0 { + availableChannelCapacity = 0 + } + if availableChannelCapacity == 0 && s.allowPause { + // nothing left for managed tracks, pause them all + for _, track := range videoTracks { + if !track.IsManaged() { + continue + } + + allocation := track.Pause() + updateStreamStateChange(track, allocation, update) + } + } else { + sorted := s.getSorted() + for _, track := range sorted { + track.ProvisionalAllocatePrepare() + } + + for spatial := int32(0); spatial <= buffer.DefaultMaxLayerSpatial; spatial++ { + for temporal := int32(0); temporal <= buffer.DefaultMaxLayerTemporal; temporal++ { + layer := buffer.VideoLayer{ + Spatial: spatial, + Temporal: temporal, + } + + for _, track := range sorted { + usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient) + availableChannelCapacity -= usedChannelCapacity + if availableChannelCapacity < 0 { + availableChannelCapacity = 0 + } + } + } + } + + for _, track := range sorted { + allocation := track.ProvisionalAllocateCommit() + updateStreamStateChange(track, allocation, update) + } + } + + s.maybeSendUpdate(update) + + s.adjustState() +} + +func (s *StreamAllocator) maybeSendUpdate(update *StreamStateUpdate) { + if update.Empty() { + return + } + + // logging individual changes to make it easier for logging systems + for _, streamState := range update.StreamStates { + s.params.Logger.Debugw("streamed tracks changed", + "trackID", streamState.TrackID, + "state", streamState.State, + ) + } + if s.onStreamStateChange != nil { + err := s.onStreamStateChange(update) + if err != nil { + s.params.Logger.Errorw("could not send streamed tracks update", err) + } + } +} + +func (s *StreamAllocator) getAvailableChannelCapacity(allowOverride bool) int64 { + availableChannelCapacity := s.committedChannelCapacity + if s.params.Config.MinChannelCapacity > availableChannelCapacity { + availableChannelCapacity = s.params.Config.MinChannelCapacity + s.params.Logger.Debugw( + "stream allocator: overriding channel capacity with min channel capacity", + "actual", s.committedChannelCapacity, + "override", availableChannelCapacity, + ) + } + if allowOverride && s.overriddenChannelCapacity > 0 { + availableChannelCapacity = s.overriddenChannelCapacity + s.params.Logger.Debugw( + "stream allocator: overriding channel capacity", + "actual", s.committedChannelCapacity, + "override", availableChannelCapacity, + ) + } + + return availableChannelCapacity +} + +func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { + expected := int64(0) + for _, track := range s.getTracks() { + expected += track.BandwidthRequested() + } + + return expected +} + +func (s *StreamAllocator) getAvailableHeadroom(allowOverride bool) int64 { + return s.getAvailableChannelCapacity(allowOverride) - s.getExpectedBandwidthUsage() +} + +func (s *StreamAllocator) getNackDelta() (uint32, uint32) { + aggPacketDelta := uint32(0) + aggRepeatedNackDelta := uint32(0) + for _, track := range s.getTracks() { + packetDelta, nackDelta := track.GetNackDelta() + aggPacketDelta += packetDelta + aggRepeatedNackDelta += nackDelta + } + + return aggPacketDelta, aggRepeatedNackDelta +} + +func (s *StreamAllocator) newChannelObserverProbe() *ChannelObserver { + return NewChannelObserver( + ChannelObserverParams{ + Name: "probe", + Config: s.params.Config.ChannelObserverProbeConfig, + }, + s.params.Logger, + ) +} + +func (s *StreamAllocator) newChannelObserverNonProbe() *ChannelObserver { + return NewChannelObserver( + ChannelObserverParams{ + Name: "non-probe", + Config: s.params.Config.ChannelObserverNonProbeConfig, + }, + s.params.Logger, + ) +} + +func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { + expectedBandwidthUsage := s.getExpectedBandwidthUsage() + if float64(expectedBandwidthUsage) > 1.5*float64(s.committedChannelCapacity) { + // STREAM-ALLOCATOR-TODO-START + // Should probably skip probing if the expected usage is much higher than committed channel capacity. + // But, give that bandwidth estimate is volatile at times and can drop down to small values, + // not probing means streaming stuck in a well for long. + // Observe this and figure out if there is a threshold from practical use cases that can be used to + // skip probing safely + // STREAM-ALLOCATOR-TODO-END + s.params.Logger.Warnw( + "stream allocator: starting probe alarm", + fmt.Errorf("expected too high, expected: %d, committed: %d", expectedBandwidthUsage, s.committedChannelCapacity), + ) + } + + probeClusterId, probeGoalBps := s.probeController.InitProbe(probeGoalDeltaBps, expectedBandwidthUsage) + + channelState := "" + if s.channelObserver != nil { + channelState = s.channelObserver.ToString() + } + s.channelObserver = s.newChannelObserverProbe() + s.channelObserver.SeedEstimate(s.lastReceivedEstimate) + + s.params.Logger.Infow( + "stream allocator: starting probe", + "probeClusterId", probeClusterId, + "current usage", expectedBandwidthUsage, + "committed", s.committedChannelCapacity, + "lastReceived", s.lastReceivedEstimate, + "channel", channelState, + "probeGoalDeltaBps", probeGoalDeltaBps, + "goalBps", probeGoalBps, + ) +} + +func (s *StreamAllocator) maybeProbe() { + if s.overriddenChannelCapacity > 0 { + // do not probe if channel capacity is overridden + return + } + if !s.probeController.CanProbe() { + return + } + + switch s.params.Config.ProbeMode { + case config.CongestionControlProbeModeMedia: + s.maybeProbeWithMedia() + s.adjustState() + case config.CongestionControlProbeModePadding: + s.maybeProbeWithPadding() + } +} + +func (s *StreamAllocator) maybeProbeWithMedia() { + // boost deficient track farthest from desired layer + for _, track := range s.getMaxDistanceSortedDeficient() { + allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInBoost) + if !boosted { + continue + } + + update := NewStreamStateUpdate() + updateStreamStateChange(track, allocation, update) + s.maybeSendUpdate(update) + + s.probeController.Reset() + break + } +} + +func (s *StreamAllocator) maybeProbeWithPadding() { + // use deficient track farthest from desired layer to find how much to probe + for _, track := range s.getMaxDistanceSortedDeficient() { + transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe) + if !available || transition.BandwidthDelta < 0 { + continue + } + + s.initProbe(transition.BandwidthDelta) + break + } +} + +func (s *StreamAllocator) getTracks() []*Track { + s.videoTracksMu.RLock() + tracks := make([]*Track, 0, len(s.videoTracks)) + for _, track := range s.videoTracks { + tracks = append(tracks, track) + } + s.videoTracksMu.RUnlock() + + return tracks +} + +func (s *StreamAllocator) getSorted() TrackSorter { + s.videoTracksMu.RLock() + var trackSorter TrackSorter + for _, track := range s.videoTracks { + if !track.IsManaged() { + continue + } + + trackSorter = append(trackSorter, track) + } + s.videoTracksMu.RUnlock() + + sort.Sort(trackSorter) + + return trackSorter +} + +func (s *StreamAllocator) getMinDistanceSorted(exclude *Track) MinDistanceSorter { + s.videoTracksMu.RLock() + var minDistanceSorter MinDistanceSorter + for _, track := range s.videoTracks { + if !track.IsManaged() || track == exclude { + continue + } + + minDistanceSorter = append(minDistanceSorter, track) + } + s.videoTracksMu.RUnlock() + + sort.Sort(minDistanceSorter) + + return minDistanceSorter +} + +func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { + s.videoTracksMu.RLock() + var maxDistanceSorter MaxDistanceSorter + for _, track := range s.videoTracks { + if !track.IsManaged() || !track.IsDeficient() { + continue + } + + maxDistanceSorter = append(maxDistanceSorter, track) + } + s.videoTracksMu.RUnlock() + + sort.Sort(maxDistanceSorter) + + return maxDistanceSorter +} + +// STREAM-ALLOCATOR-EXPERIMENTAL-TODO +// Monitor sent rate vs estimate to figure out queuing on congestion. +// Idea here is to pause all managed tracks on congestion detection immediately till queue drains. +// That will allow channel to clear up without more traffic added and a re-allocation can start afresh. +// Some bits to work out +// - how good is queuing estimate? +// - should we pause unmanaged tracks also? But, they will restart at highest layer and request a key frame. +// - what should be the channel capacity to use when resume re-allocation happens? +func (s *StreamAllocator) monitorRate(estimate int64) { + managedBytesSent := uint32(0) + managedBytesRetransmitted := uint32(0) + unmanagedBytesSent := uint32(0) + unmanagedBytesRetransmitted := uint32(0) + for _, track := range s.getTracks() { + b, r := track.GetAndResetBytesSent() + if track.IsManaged() { + managedBytesSent += b + managedBytesRetransmitted += r + } else { + unmanagedBytesSent += b + unmanagedBytesRetransmitted += r + } + } + + s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) +} + +func (s *StreamAllocator) updateTracksHistory() { + for _, track := range s.getTracks() { + track.UpdateHistory() + } +} + +func (s *StreamAllocator) getTracksHistory() map[livekit.TrackID]string { + tracks := s.getTracks() + history := make(map[livekit.TrackID]string, len(tracks)) + for _, track := range tracks { + history[track.ID()] = track.GetHistory() + } + + return history +} + +// ------------------------------------------------ + +func updateStreamStateChange(track *Track, allocation sfu.VideoAllocation, update *StreamStateUpdate) { + updated := false + streamState := StreamStateInactive + switch allocation.PauseReason { + case sfu.VideoPauseReasonMuted: + fallthrough + + case sfu.VideoPauseReasonPubMuted: + streamState = StreamStateInactive + updated = track.SetStreamState(streamState) + + case sfu.VideoPauseReasonBandwidth: + streamState = StreamStatePaused + updated = track.SetStreamState(streamState) + } + + if updated { + update.HandleStreamingChange(track, streamState) + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamstateupdate.go b/pkg/sfu/streamallocator/streamstateupdate.go new file mode 100644 index 000000000..53156de3c --- /dev/null +++ b/pkg/sfu/streamallocator/streamstateupdate.go @@ -0,0 +1,85 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + + "github.com/livekit/protocol/livekit" +) + +// ------------------------------------------------ + +type StreamState int + +const ( + StreamStateInactive StreamState = iota + StreamStateActive + StreamStatePaused +) + +func (s StreamState) String() string { + switch s { + case StreamStateInactive: + return "INACTIVE" + case StreamStateActive: + return "ACTIVE" + case StreamStatePaused: + return "PAUSED" + default: + return fmt.Sprintf("UNKNOWN: %d", int(s)) + } +} + +// ------------------------------------------------ + +type StreamStateInfo struct { + ParticipantID livekit.ParticipantID + TrackID livekit.TrackID + State StreamState +} + +type StreamStateUpdate struct { + StreamStates []*StreamStateInfo +} + +func NewStreamStateUpdate() *StreamStateUpdate { + return &StreamStateUpdate{} +} + +func (s *StreamStateUpdate) HandleStreamingChange(track *Track, streamState StreamState) { + switch streamState { + case StreamStateInactive: + // inactive is not a notification, could get into this state because of mute + case StreamStateActive: + s.StreamStates = append(s.StreamStates, &StreamStateInfo{ + ParticipantID: track.PublisherID(), + TrackID: track.ID(), + State: StreamStateActive, + }) + case StreamStatePaused: + s.StreamStates = append(s.StreamStates, &StreamStateInfo{ + ParticipantID: track.PublisherID(), + TrackID: track.ID(), + State: StreamStatePaused, + }) + } +} + +func (s *StreamStateUpdate) Empty() bool { + return len(s.StreamStates) == 0 +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go new file mode 100644 index 000000000..6ccae215b --- /dev/null +++ b/pkg/sfu/streamallocator/track.go @@ -0,0 +1,439 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + "sort" + "time" + + "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/pion/rtcp" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +type Track struct { + downTrack *sfu.DownTrack + source livekit.TrackSource + isSimulcast bool + priority uint8 + publisherID livekit.ParticipantID + logger logger.Logger + + maxLayer buffer.VideoLayer + + totalPackets uint32 + totalRepeatedNacks uint32 + + nackInfos map[uint16]sfu.NackInfo + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + nackHistory []string + + receiverReportInitialized bool + totalLostAtLastRead uint32 + totalLost uint32 + highestSequenceNumberAtLastRead uint32 + highestSequenceNumber uint32 + maxRTT uint32 + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + receiverReportHistory []string + + isDirty bool + + streamState StreamState +} + +func NewTrack( + downTrack *sfu.DownTrack, + source livekit.TrackSource, + isSimulcast bool, + publisherID livekit.ParticipantID, + logger logger.Logger, +) *Track { + t := &Track{ + downTrack: downTrack, + source: source, + isSimulcast: isSimulcast, + publisherID: publisherID, + logger: logger, + nackInfos: make(map[uint16]sfu.NackInfo), + nackHistory: make([]string, 0, 10), + receiverReportHistory: make([]string, 0, 10), + streamState: StreamStateInactive, + } + t.SetPriority(0) + t.SetMaxLayer(downTrack.MaxLayer()) + + return t +} + +func (t *Track) SetDirty(isDirty bool) bool { + if t.isDirty == isDirty { + return false + } + + t.isDirty = isDirty + return true +} + +func (t *Track) SetStreamState(streamState StreamState) bool { + if t.streamState == streamState { + return false + } + + t.streamState = streamState + return true +} + +func (t *Track) SetPriority(priority uint8) bool { + if priority == 0 { + switch t.source { + case livekit.TrackSource_SCREEN_SHARE: + priority = PriorityDefaultScreenshare + default: + priority = PriorityDefaultVideo + } + } + + if t.priority == priority { + return false + } + + t.priority = priority + return true +} + +func (t *Track) Priority() uint8 { + return t.priority +} + +func (t *Track) DownTrack() *sfu.DownTrack { + return t.downTrack +} + +func (t *Track) IsManaged() bool { + return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast +} + +func (t *Track) ID() livekit.TrackID { + return livekit.TrackID(t.downTrack.ID()) +} + +func (t *Track) PublisherID() livekit.ParticipantID { + return t.publisherID +} + +func (t *Track) SetMaxLayer(layer buffer.VideoLayer) bool { + if t.maxLayer == layer { + return false + } + + t.maxLayer = layer + return true +} + +func (t *Track) WritePaddingRTP(bytesToSend int) int { + return t.downTrack.WritePaddingRTP(bytesToSend, false, false) +} + +func (t *Track) AllocateOptimal(allowOvershoot bool) sfu.VideoAllocation { + return t.downTrack.AllocateOptimal(allowOvershoot) +} + +func (t *Track) ProvisionalAllocatePrepare() { + t.downTrack.ProvisionalAllocatePrepare() +} + +func (t *Track) ProvisionalAllocateReset() { + t.downTrack.ProvisionalAllocateReset() +} + +func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { + return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layer, allowPause, allowOvershoot) +} + +func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) sfu.VideoTransition { + return t.downTrack.ProvisionalAllocateGetCooperativeTransition(allowOvershoot) +} + +func (t *Track) ProvisionalAllocateGetBestWeightedTransition() sfu.VideoTransition { + return t.downTrack.ProvisionalAllocateGetBestWeightedTransition() +} + +func (t *Track) ProvisionalAllocateCommit() sfu.VideoAllocation { + return t.downTrack.ProvisionalAllocateCommit() +} + +func (t *Track) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (sfu.VideoAllocation, bool) { + return t.downTrack.AllocateNextHigher(availableChannelCapacity, allowOvershoot) +} + +func (t *Track) GetNextHigherTransition(allowOvershoot bool) (sfu.VideoTransition, bool) { + return t.downTrack.GetNextHigherTransition(allowOvershoot) +} + +func (t *Track) Pause() sfu.VideoAllocation { + return t.downTrack.Pause() +} + +func (t *Track) IsDeficient() bool { + return t.downTrack.IsDeficient() +} + +func (t *Track) BandwidthRequested() int64 { + return t.downTrack.BandwidthRequested() +} + +func (t *Track) DistanceToDesired() float64 { + return t.downTrack.DistanceToDesired() +} + +func (t *Track) GetNackDelta() (uint32, uint32) { + totalPackets, totalRepeatedNacks := t.downTrack.GetNackStats() + + packetDelta := totalPackets - t.totalPackets + t.totalPackets = totalPackets + + nackDelta := totalRepeatedNacks - t.totalRepeatedNacks + t.totalRepeatedNacks = totalRepeatedNacks + + return packetDelta, nackDelta +} + +func (t *Track) UpdateNack(nackInfos []sfu.NackInfo) { + for _, ni := range nackInfos { + t.nackInfos[ni.SequenceNumber] = ni + } +} + +func (t *Track) GetAndResetNackStats() (lowest uint16, highest uint16, numNacked int, numNacks int, numRuns int) { + if len(t.nackInfos) == 0 { + return + } + + sns := make([]uint16, 0, len(t.nackInfos)) + for _, ni := range t.nackInfos { + if lowest == 0 || ni.SequenceNumber-lowest > (1<<15) { + lowest = ni.SequenceNumber + } + if highest == 0 || highest-ni.SequenceNumber > (1<<15) { + highest = ni.SequenceNumber + } + numNacks += int(ni.Attempts) + sns = append(sns, ni.SequenceNumber) + } + numNacked = len(t.nackInfos) + + // find number of runs, i. e. bursts of contiguous sequence numbers NACKed, does not include isolated NACKs + sort.Slice(sns, func(i, j int) bool { + return (sns[i] - sns[j]) > (1 << 15) + }) + + rsn := sns[0] + rsi := 0 + for i := 1; i < len(sns); i++ { + if sns[i] == rsn+1 { + continue + } + + if (i - rsi - 1) > 0 { + numRuns++ + } + + rsn = sns[i] + rsi = i + } + + t.nackInfos = make(map[uint16]sfu.NackInfo) + return +} + +func (t *Track) ProcessRTCPReceiverReport(rr rtcp.ReceptionReport) { + if !t.receiverReportInitialized { + t.receiverReportInitialized = true + t.totalLostAtLastRead = rr.TotalLost + t.highestSequenceNumberAtLastRead = rr.LastSequenceNumber + } + + t.totalLost = rr.TotalLost + t.highestSequenceNumber = rr.LastSequenceNumber + + if rtt, err := mediatransportutil.GetRttMsFromReceiverReportOnly(&rr); err != nil { + if rtt > t.maxRTT { + t.maxRTT = rtt + } + } + + t.updateReceiverReportHistory() +} + +func (t *Track) GetRTCPReceiverReportDelta() (uint32, uint32, uint32) { + deltaPackets := t.highestSequenceNumber - t.highestSequenceNumberAtLastRead + t.highestSequenceNumberAtLastRead = t.highestSequenceNumber + + deltaLost := t.totalLost - t.totalLostAtLastRead + t.totalLostAtLastRead = t.totalLost + + maxRTT := t.maxRTT + t.maxRTT = 0 + + return deltaLost, deltaPackets, maxRTT +} + +func (t *Track) GetAndResetBytesSent() (uint32, uint32) { + return t.downTrack.GetAndResetBytesSent() +} + +func (t *Track) UpdateHistory() { + t.updateNackHistory() +} + +func (t *Track) GetHistory() string { + return fmt.Sprintf("t: %+v, n: %+v, rr: %+v", time.Now(), t.nackHistory, t.receiverReportHistory) +} + +// STREAM-ALLOCATOR-EXPERIMENTAL-TODO: +// Idea is to check if this provides a good signal to detect congestion. +// This measures a few things +// 1. Spread: sequence number difference between highest and lowest NACK +// - shows how widespread the losses are +// 2. Number of runs of length more than 1: Counts number of burst losses. +// - could be a sign of congestion when losses are bursty +// 3. NACK density: how many sequence numbers in the spread were NACKed. +// - a high density could be a sign of congestion +// 4. NACK intensity: how many times those sequence numbers were NACKed. +// - high intensity could be a sign of congestion +// +// While these all could be good signals, some challenges in making use of these +// - aggregating across tracks +// - proper thresholing, i. e. something based on averages should not trip +// because of small numbers, e. g. a single NACK run of 2 sequence numbers +// is technically a burst, but is it a signal of congestion? +func (t *Track) updateNackHistory() { + if len(t.nackHistory) >= 10 { + t.nackHistory = t.nackHistory[1:] + } + + l, h, nnd, nns, nr := t.GetAndResetNackStats() + spread := h - l + 1 + density := float64(0.0) + if nnd != 0 { + density = float64(nnd) / float64(spread) + } else { + spread = 0 + } + intensity := float64(0.0) + if nnd != 0 { + intensity = float64(nns) / float64(nnd) + } + t.nackHistory = append( + t.nackHistory, + fmt.Sprintf("t: %+v, l: %d, h: %d, sp: %d, nnd: %d, dens: %.2f, nns: %d, int: %.2f, nr: %d", time.Now().UnixMilli(), l, h, spread, nnd, density, nns, intensity, nr), + ) +} + +func (t *Track) updateReceiverReportHistory() { + if len(t.receiverReportHistory) >= 10 { + t.receiverReportHistory = t.receiverReportHistory[1:] + } + + dl, dp, maxRTT := t.GetRTCPReceiverReportDelta() + t.receiverReportHistory = append( + t.receiverReportHistory, + fmt.Sprintf("t: %+v, l: %d, p: %d, rtt: %d", time.Now().Format(time.UnixDate), dl, dp, maxRTT), + ) +} + +// ------------------------------------------------ + +type TrackSorter []*Track + +func (t TrackSorter) Len() int { + return len(t) +} + +func (t TrackSorter) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +func (t TrackSorter) Less(i, j int) bool { + // + // TrackSorter is used to allocate layer-by-layer. + // So, higher priority track should come earlier so that it gets an earlier shot at each layer + // + if t[i].priority != t[j].priority { + return t[i].priority > t[j].priority + } + + if t[i].maxLayer.Spatial != t[j].maxLayer.Spatial { + return t[i].maxLayer.Spatial > t[j].maxLayer.Spatial + } + + return t[i].maxLayer.Temporal > t[j].maxLayer.Temporal +} + +// ------------------------------------------------ + +type MaxDistanceSorter []*Track + +func (m MaxDistanceSorter) Len() int { + return len(m) +} + +func (m MaxDistanceSorter) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +func (m MaxDistanceSorter) Less(i, j int) bool { + // + // MaxDistanceSorter is used to find a deficient track to use for probing during recovery from congestion. + // So, higher priority track should come earlier so that they have a chance to recover sooner. + // + if m[i].priority != m[j].priority { + return m[i].priority > m[j].priority + } + + return m[i].DistanceToDesired() > m[j].DistanceToDesired() +} + +// ------------------------------------------------ + +type MinDistanceSorter []*Track + +func (m MinDistanceSorter) Len() int { + return len(m) +} + +func (m MinDistanceSorter) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +func (m MinDistanceSorter) Less(i, j int) bool { + // + // MinDistanceSorter is used to find excess bandwidth in cooperative allocation. + // So, lower priority track should come earlier so that they contribute bandwidth to higher priority tracks. + // + if m[i].priority != m[j].priority { + return m[i].priority < m[j].priority + } + + return m[i].DistanceToDesired() < m[j].DistanceToDesired() +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go new file mode 100644 index 000000000..c54d29efa --- /dev/null +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -0,0 +1,244 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type TrendDirection int + +const ( + TrendDirectionNeutral TrendDirection = iota + TrendDirectionUpward + TrendDirectionDownward +) + +func (t TrendDirection) String() string { + switch t { + case TrendDirectionNeutral: + return "NEUTRAL" + case TrendDirectionUpward: + return "UPWARD" + case TrendDirectionDownward: + return "DOWNWARD" + default: + return fmt.Sprintf("%d", int(t)) + } +} + +// ------------------------------------------------ + +type trendDetectorSample struct { + value int64 + at time.Time +} + +// ------------------------------------------------ + +type TrendDetectorParams struct { + Name string + Logger logger.Logger + RequiredSamples int + DownwardTrendThreshold float64 + CollapseThreshold time.Duration + ValidityWindow time.Duration +} + +type TrendDetector struct { + params TrendDetectorParams + + startTime time.Time + numSamples int + samples []trendDetectorSample + lowestValue int64 + highestValue int64 + + direction TrendDirection +} + +func NewTrendDetector(params TrendDetectorParams) *TrendDetector { + return &TrendDetector{ + params: params, + startTime: time.Now(), + direction: TrendDirectionNeutral, + } +} + +func (t *TrendDetector) Seed(value int64) { + if len(t.samples) != 0 { + return + } + + t.samples = append(t.samples, trendDetectorSample{value: value, at: time.Now()}) +} + +func (t *TrendDetector) AddValue(value int64) { + t.numSamples++ + if t.lowestValue == 0 || value < t.lowestValue { + t.lowestValue = value + } + if value > t.highestValue { + t.highestValue = value + } + + // Ignore duplicate values in collapse window. + // + // Bandwidth estimate is received periodically. If the estimate does not change, it will be repeated. + // When there is congestion, there are several estimates received with decreasing values. + // + // Using a sliding window, collapsing repeated values and waiting for falling trend is to ensure that + // the reaction is not too fast, i. e. reacting to falling values too quick could mean a lot of re-allocation + // resulting in layer switches, key frames and more congestion. + // + // But, on the flip side, estimate could fall once or twice within a sliding window and stay there. + // In those cases, using a collapse window to record a value even if it is duplicate. By doing that, + // a trend could be detected eventually. If will be delayed, but that is fine with slow changing estimates. + var lastSample *trendDetectorSample + if len(t.samples) != 0 { + lastSample = &t.samples[len(t.samples)-1] + } + if lastSample != nil && lastSample.value == value && t.params.CollapseThreshold > 0 && time.Since(lastSample.at) < t.params.CollapseThreshold { + return + } + + t.samples = append(t.samples, trendDetectorSample{value: value, at: time.Now()}) + t.prune() + t.updateDirection() +} + +func (t *TrendDetector) GetLowest() int64 { + return t.lowestValue +} + +func (t *TrendDetector) GetHighest() int64 { + return t.highestValue +} + +func (t *TrendDetector) GetDirection() TrendDirection { + return t.direction +} + +func (t *TrendDetector) HasEnoughSamples() bool { + return t.numSamples >= t.params.RequiredSamples +} + +func (t *TrendDetector) ToString() string { + now := time.Now() + elapsed := now.Sub(t.startTime).Seconds() + samplesStr := "" + if len(t.samples) > 0 { + firstTime := t.samples[0].at + samplesStr += "[" + for i, sample := range t.samples { + suffix := ", " + if i == len(t.samples)-1 { + suffix = "" + } + samplesStr += fmt.Sprintf("%d(%d)%s", sample.value, sample.at.Sub(firstTime).Milliseconds(), suffix) + } + samplesStr += "]" + } + return fmt.Sprintf("n: %s, t: %+v|%+v|%.2fs, v: %d|%d|%d|%s|%.2f", + t.params.Name, + t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed, + t.numSamples, t.lowestValue, t.highestValue, samplesStr, kendallsTau(t.samples), + ) +} + +func (t *TrendDetector) prune() { + // prune based on a few rules + // 1. If there are more than required samples + if len(t.samples) > t.params.RequiredSamples { + t.samples = t.samples[len(t.samples)-t.params.RequiredSamples:] + } + + // 2. drop samples that are too old + if len(t.samples) != 0 && t.params.ValidityWindow > 0 { + cutoffTime := time.Now().Add(-t.params.ValidityWindow) + cutoffIndex := -1 + for i := 0; i < len(t.samples); i++ { + if t.samples[i].at.After(cutoffTime) { + cutoffIndex = i + break + } + } + if cutoffIndex >= 0 { + t.samples = t.samples[cutoffIndex:] + } + } + + // 3. If all sample values are same, collapse to just the last one + if len(t.samples) != 0 { + sameValue := true + firstValue := t.samples[0].value + for i := 0; i < len(t.samples); i++ { + if t.samples[i].value != firstValue { + sameValue = false + break + } + } + + if sameValue { + t.samples = t.samples[len(t.samples)-1:] + } + } +} + +func (t *TrendDetector) updateDirection() { + if len(t.samples) < t.params.RequiredSamples { + t.direction = TrendDirectionNeutral + return + } + + // using Kendall's Tau to find trend + kt := kendallsTau(t.samples) + + t.direction = TrendDirectionNeutral + switch { + case kt > 0: + t.direction = TrendDirectionUpward + case kt < t.params.DownwardTrendThreshold: + t.direction = TrendDirectionDownward + } +} + +// ------------------------------------------------ + +func kendallsTau(samples []trendDetectorSample) float64 { + concordantPairs := 0 + discordantPairs := 0 + + for i := 0; i < len(samples)-1; i++ { + for j := i + 1; j < len(samples); j++ { + if samples[i].value < samples[j].value { + concordantPairs++ + } else if samples[i].value > samples[j].value { + discordantPairs++ + } + } + } + + if (concordantPairs + discordantPairs) == 0 { + return 0.0 + } + + return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) +} diff --git a/pkg/sfu/streamtracker/interfaces.go b/pkg/sfu/streamtracker/interfaces.go index 3837f8470..f3ad5e699 100644 --- a/pkg/sfu/streamtracker/interfaces.go +++ b/pkg/sfu/streamtracker/interfaces.go @@ -1,8 +1,24 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( "fmt" "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // ------------------------------------------------------------ @@ -40,3 +56,15 @@ type StreamTrackerImpl interface { Observe(hasMarker bool, ts uint32) StreamStatusChange CheckStatus() StreamStatusChange } + +type StreamTrackerWorker interface { + Start() + Stop() + Reset() + OnStatusChanged(f func(status StreamStatus)) + OnBitrateAvailable(f func()) + Status() StreamStatus + BitrateTemporalCumulative() []int64 + SetPaused(paused bool) + Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.ExtDependencyDescriptor) +} diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index 9f3a611a2..b45723a79 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( @@ -5,8 +19,10 @@ import ( "sync" "time" - "github.com/livekit/protocol/logger" "go.uber.org/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" ) // ------------------------------------------------------------ @@ -174,6 +190,7 @@ func (s *StreamTracker) Observe( payloadSize int, hasMarker bool, ts uint32, + _ *buffer.ExtDependencyDescriptor, ) { s.lock.Lock() diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go new file mode 100644 index 000000000..c19b3fe85 --- /dev/null +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -0,0 +1,285 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamtracker + +import ( + "sync" + "time" + + "go.uber.org/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +type StreamTrackerDependencyDescriptor struct { + lock sync.RWMutex + paused bool + generation atomic.Uint32 + params StreamTrackerParams + maxSpatialLayer int32 + maxTemporalLayer int32 + + onStatusChanged [buffer.DefaultMaxLayerSpatial + 1]func(status StreamStatus) + onBitrateAvailable [buffer.DefaultMaxLayerSpatial + 1]func() + + lastBitrateReport time.Time + bytesForBitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 + bitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 + + isStopped bool +} + +func NewStreamTrackerDependencyDescriptor(params StreamTrackerParams) *StreamTrackerDependencyDescriptor { + return &StreamTrackerDependencyDescriptor{ + params: params, + maxSpatialLayer: buffer.InvalidLayerSpatial, + maxTemporalLayer: buffer.InvalidLayerTemporal, + } +} +func (s *StreamTrackerDependencyDescriptor) Start() { +} + +func (s *StreamTrackerDependencyDescriptor) Stop() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.isStopped { + return + } + s.isStopped = true + + // bump generation to trigger exit of worker + s.generation.Inc() +} + +func (s *StreamTrackerDependencyDescriptor) OnStatusChanged(layer int32, f func(status StreamStatus)) { + s.lock.Lock() + s.onStatusChanged[layer] = f + s.lock.Unlock() +} + +func (s *StreamTrackerDependencyDescriptor) OnBitrateAvailable(layer int32, f func()) { + s.lock.Lock() + s.onBitrateAvailable[layer] = f + s.lock.Unlock() +} + +func (s *StreamTrackerDependencyDescriptor) Status(layer int32) StreamStatus { + s.lock.RLock() + defer s.lock.RUnlock() + + if layer > s.maxSpatialLayer { + return StreamStatusStopped + } + + return StreamStatusActive +} + +func (s *StreamTrackerDependencyDescriptor) BitrateTemporalCumulative(layer int32) []int64 { + s.lock.RLock() + defer s.lock.RUnlock() + + if layer > s.maxSpatialLayer { + brs := make([]int64, len(s.bitrate[0])) + return brs + } + + return s.bitrate[layer][:] +} + +func (s *StreamTrackerDependencyDescriptor) Reset() { +} + +func (s *StreamTrackerDependencyDescriptor) resetLocked() { + // bump generation to trigger exit of current worker + s.generation.Inc() + + for i := 0; i < len(s.bytesForBitrate); i++ { + for j := 0; j < len(s.bytesForBitrate[i]); j++ { + s.bytesForBitrate[i][j] = 0 + } + } + for i := 0; i < len(s.bitrate); i++ { + for j := 0; j < len(s.bitrate[i]); j++ { + s.bitrate[i][j] = 0 + } + } +} + +func (s *StreamTrackerDependencyDescriptor) SetPaused(paused bool) { + s.lock.Lock() + if s.paused == paused { + s.lock.Unlock() + return + } + s.paused = paused + if !paused { + s.resetLocked() + } else { + s.lastBitrateReport = time.Now() + go s.worker(s.generation.Inc()) + + } + s.lock.Unlock() + +} + +func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.ExtDependencyDescriptor) { + s.lock.Lock() + + if s.isStopped || s.paused || payloadSize == 0 || ddVal == nil { + s.lock.Unlock() + return + } + + var notifyFns []func(status StreamStatus) + var notifyStatus StreamStatus + if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil && ddVal.ActiveDecodeTargetsUpdated { + var maxSpatial, maxTemporal int32 + for _, dt := range ddVal.DecodeTargets { + if *mask&(1< buffer.DefaultMaxLayerSpatial { + maxSpatial = buffer.DefaultMaxLayerSpatial + s.params.Logger.Warnw("max spatial layer exceeded", nil, "maxSpatial", maxSpatial) + } + if maxTemporal > buffer.DefaultMaxLayerTemporal { + maxTemporal = buffer.DefaultMaxLayerTemporal + s.params.Logger.Warnw("max temporal layer exceeded", nil, "maxTemporal", maxTemporal) + } + + s.params.Logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal) + oldMaxSpatial := s.maxSpatialLayer + s.maxSpatialLayer, s.maxTemporalLayer = maxSpatial, maxTemporal + if oldMaxSpatial == -1 { + s.lastBitrateReport = time.Now() + go s.worker(s.generation.Inc()) + } + + if oldMaxSpatial > s.maxSpatialLayer { + notifyStatus = StreamStatusStopped + for i := s.maxSpatialLayer + 1; i <= oldMaxSpatial; i++ { + notifyFns = append(notifyFns, s.onStatusChanged[i]) + } + } else if oldMaxSpatial < s.maxSpatialLayer { + notifyStatus = StreamStatusActive + for i := oldMaxSpatial + 1; i <= s.maxSpatialLayer; i++ { + notifyFns = append(notifyFns, s.onStatusChanged[i]) + } + } + } + + dtis := ddVal.Descriptor.FrameDependencies.DecodeTargetIndications + + for _, dt := range ddVal.DecodeTargets { + // we are not dropping discardable frames now, so only ingore not present frames + if dtis[dt.Target] == dd.DecodeTargetNotPresent { + continue + } + + s.bytesForBitrate[dt.Layer.Spatial][dt.Layer.Temporal] += int64(pktSize) + } + + s.lock.Unlock() + + for _, fn := range notifyFns { + if fn != nil { + fn(notifyStatus) + } + } +} + +func (s *StreamTrackerDependencyDescriptor) worker(generation uint32) { + tickerBitrate := time.NewTicker(s.params.BitrateReportInterval) + defer tickerBitrate.Stop() + + for { + <-tickerBitrate.C + if generation != s.generation.Load() { + return + } + s.bitrateReport() + } +} + +func (s *StreamTrackerDependencyDescriptor) bitrateReport() { + // run this even if paused to drain out bitrate if there are no packets coming in + s.lock.Lock() + now := time.Now() + diff := now.Sub(s.lastBitrateReport) + s.lastBitrateReport = now + + var availableChangedFns []func() + for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ { + bytesForBitrate := s.bytesForBitrate[spatial][:] + bitrateAvailabilityChanged := false + bitrates := s.bitrate[spatial][:] + for i := 0; i < len(bytesForBitrate); i++ { + bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds()) + if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) { + bitrateAvailabilityChanged = true + } + bitrates[i] = bitrate + bytesForBitrate[i] = 0 + } + + if bitrateAvailabilityChanged && s.onBitrateAvailable[spatial] != nil { + availableChangedFns = append(availableChangedFns, s.onBitrateAvailable[spatial]) + } + } + s.lock.Unlock() + + for _, fn := range availableChangedFns { + fn() + } +} + +func (s *StreamTrackerDependencyDescriptor) LayeredTracker(layer int32) *StreamTrackerDependencyDescriptorLayered { + return &StreamTrackerDependencyDescriptorLayered{ + StreamTrackerDependencyDescriptor: s, + layer: layer, + } +} + +// ---------------------------- +// Layered wrapper for StreamTrackerWorker +type StreamTrackerDependencyDescriptorLayered struct { + *StreamTrackerDependencyDescriptor + layer int32 +} + +func (s *StreamTrackerDependencyDescriptorLayered) OnStatusChanged(f func(status StreamStatus)) { + s.StreamTrackerDependencyDescriptor.OnStatusChanged(s.layer, f) +} + +func (s *StreamTrackerDependencyDescriptorLayered) OnBitrateAvailable(f func()) { + s.StreamTrackerDependencyDescriptor.OnBitrateAvailable(s.layer, f) +} + +func (s *StreamTrackerDependencyDescriptorLayered) Status() StreamStatus { + return s.StreamTrackerDependencyDescriptor.Status(s.layer) +} + +func (s *StreamTrackerDependencyDescriptorLayered) BitrateTemporalCumulative() []int64 { + return s.StreamTrackerDependencyDescriptor.BitrateTemporalCumulative(s.layer) +} diff --git a/pkg/sfu/streamtracker/streamtracker_dd_test.go b/pkg/sfu/streamtracker/streamtracker_dd_test.go new file mode 100644 index 000000000..f638e4e2d --- /dev/null +++ b/pkg/sfu/streamtracker/streamtracker_dd_test.go @@ -0,0 +1,98 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streamtracker + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +func createDescriptorDependencyForTargets(maxSpatial, maxTemporal int) *buffer.ExtDependencyDescriptor { + var targets []buffer.DependencyDescriptorDecodeTarget + var mask uint32 + for i := 0; i <= maxSpatial; i++ { + for j := 0; j <= maxTemporal; j++ { + targets = append(targets, buffer.DependencyDescriptorDecodeTarget{Target: len(targets), Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}}) + mask |= 1 << uint32(len(targets)-1) + } + } + + dtis := make([]dd.DecodeTargetIndication, len(targets)) + for _, t := range targets { + dtis[t.Target] = dd.DecodeTargetRequired + } + + return &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + ActiveDecodeTargetsBitmask: &mask, + FrameDependencies: &dd.FrameDependencyTemplate{ + DecodeTargetIndications: dtis, + }, + }, + DecodeTargets: targets, + ActiveDecodeTargetsUpdated: true, + } +} + +func checkStatues(t *testing.T, statuses []StreamStatus, expected StreamStatus, maxSpatial int) { + for i := 0; i <= maxSpatial; i++ { + require.Equal(t, expected, statuses[i]) + } + + for i := maxSpatial + 1; i < len(statuses); i++ { + require.NotEqual(t, expected, statuses[i]) + } +} + +func TestStreamTrackerDD(t *testing.T) { + ddTracker := NewStreamTrackerDependencyDescriptor(StreamTrackerParams{ + BitrateReportInterval: 1 * time.Second, + Logger: logger.GetLogger(), + }) + layeredTrackers := make([]StreamTrackerWorker, buffer.DefaultMaxLayerSpatial+1) + statuses := make([]StreamStatus, buffer.DefaultMaxLayerSpatial+1) + for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ { + layeredTrack := ddTracker.LayeredTracker(int32(i)) + layer := i + layeredTrack.OnStatusChanged(func(status StreamStatus) { + statuses[layer] = status + }) + layeredTrack.Start() + layeredTrackers[i] = layeredTrack + } + defer ddTracker.Stop() + + // no active layers + ddTracker.Observe(0, 1000, 1000, false, 0, nil) + checkStatues(t, statuses, StreamStatusActive, int(buffer.InvalidLayerSpatial)) + + // layer seen [0,1] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1)) + checkStatues(t, statuses, StreamStatusActive, 1) + + // layer seen [0,1,2] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(2, 1)) + checkStatues(t, statuses, StreamStatusActive, 2) + + // layer 2 gone, layer seen [0,1] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1)) + checkStatues(t, statuses, StreamStatusActive, 1) +} diff --git a/pkg/sfu/streamtracker/streamtracker_frame.go b/pkg/sfu/streamtracker/streamtracker_frame.go index 02d9ad7ae..db08d9343 100644 --- a/pkg/sfu/streamtracker/streamtracker_frame.go +++ b/pkg/sfu/streamtracker/streamtracker_frame.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( @@ -151,7 +165,7 @@ func (s *StreamTrackerFrame) updateEstimatedFrameRate() float64 { s.oldestTS = s.newestTS s.numFrames = 1 - factor := float64(1.0) + factor := 1.0 switch { case s.estimatedFrameRate < frameRate: // slow increase, prevents shortening eval interval too quickly on frame rate going up diff --git a/pkg/sfu/streamtracker/streamtracker_packet.go b/pkg/sfu/streamtracker/streamtracker_packet.go index 78866e40e..e95629580 100644 --- a/pkg/sfu/streamtracker/streamtracker_packet.go +++ b/pkg/sfu/streamtracker/streamtracker_packet.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_packet_test.go b/pkg/sfu/streamtracker/streamtracker_packet_test.go index 2aee13ecb..276483e5e 100644 --- a/pkg/sfu/streamtracker/streamtracker_packet_test.go +++ b/pkg/sfu/streamtracker/streamtracker_packet_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( @@ -42,7 +56,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, StreamStatusStopped, tracker.Status()) // observe first packet - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() { @@ -73,7 +87,7 @@ func TestStreamTracker(t *testing.T) { callbackStatusMu.Unlock() }) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { callbackStatusMu.RLock() defer callbackStatusMu.RUnlock() @@ -110,7 +124,7 @@ func TestStreamTracker(t *testing.T) { tracker.Start() require.Equal(t, StreamStatusStopped, tracker.Status()) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if tracker.Status() == StreamStatusActive { return "" @@ -121,11 +135,11 @@ func TestStreamTracker(t *testing.T) { tracker.setStatusLocked(StreamStatusStopped) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() require.Equal(t, StreamStatusStopped, tracker.Status()) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() require.Equal(t, StreamStatusActive, tracker.Status()) @@ -135,7 +149,7 @@ func TestStreamTracker(t *testing.T) { t.Run("changes to inactive when paused", func(t *testing.T) { tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond) tracker.Start() - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if tracker.Status() == StreamStatusActive { return "" @@ -161,7 +175,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, StreamStatusStopped, tracker.Status()) // observe first packet - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() == 1 { @@ -175,10 +189,10 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, uint32(1), callbackCalled.Load()) // observe a few more - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() // should still be active @@ -191,7 +205,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, uint32(2), callbackCalled.Load()) // first packet after reset - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() == 3 { diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index af4532af4..30ba98322 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -1,12 +1,28 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( "fmt" + "math" "sort" "sync" "time" "github.com/frostbyte73/core" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" @@ -14,6 +30,14 @@ import ( "github.com/livekit/protocol/logger" ) +const ( + senderReportThresholdSeconds = float64(60.0) + + minDurationForClockRateCalculation = 15 * time.Second +) + +// --------------------------------------------------- + type StreamTrackerManagerListener interface { OnAvailableLayersChanged() OnBitrateAvailabilityChanged() @@ -23,6 +47,14 @@ type StreamTrackerManagerListener interface { OnBitrateReport(availableLayers []int32, bitrates Bitrates) } +// --------------------------------------------------- + +type endsSenderReport struct { + first *buffer.RTCPSenderReportData + newest *buffer.RTCPSenderReportData + lastUpdated time.Time +} + type StreamTrackerManager struct { logger logger.Logger trackInfo *livekit.TrackInfo @@ -35,14 +67,16 @@ type StreamTrackerManager struct { maxPublishedLayer int32 maxTemporalLayerSeen int32 - trackers [DefaultMaxLayerSpatial + 1]*streamtracker.StreamTracker + ddTracker *streamtracker.StreamTrackerDependencyDescriptor + trackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker availableLayers []int32 maxExpectedLayer int32 paused bool senderReportMu sync.RWMutex - senderReports [DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportDataExt + senderReports [buffer.DefaultMaxLayerSpatial + 1]endsSenderReport + layerOffsets [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerSpatial + 1]uint32 closed core.Fuse @@ -60,8 +94,8 @@ func NewStreamTrackerManager( logger: logger, trackInfo: trackInfo, isSVC: isSVC, - maxPublishedLayer: InvalidLayerSpatial, - maxTemporalLayerSeen: InvalidLayerTemporal, + maxPublishedLayer: buffer.InvalidLayerSpatial, + maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, closed: core.NewFuse(), } @@ -77,7 +111,9 @@ func NewStreamTrackerManager( s.maxExpectedLayerFromTrackInfo() - go s.bitrateReporter() + if s.trackInfo.Type == livekit.TrackType_VIDEO { + go s.bitrateReporter() + } return s } @@ -125,28 +161,58 @@ func (s *StreamTrackerManager) createStreamTrackerFrame(layer int32) streamtrack return streamtracker.NewStreamTrackerFrame(params) } -func (s *StreamTrackerManager) AddTracker(layer int32) *streamtracker.StreamTracker { +func (s *StreamTrackerManager) AddDependencyDescriptorTrackers() { + bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[0] + if !ok { + return + } + s.lock.Lock() + var addAllTrackers bool + if s.ddTracker == nil { + s.ddTracker = streamtracker.NewStreamTrackerDependencyDescriptor(streamtracker.StreamTrackerParams{ + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", 0), + }) + addAllTrackers = true + } + s.lock.Unlock() + if addAllTrackers { + for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ { + s.AddTracker(int32(i)) + } + } +} + +func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrackerWorker { bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[layer] if !ok { return nil } - var trackerImpl streamtracker.StreamTrackerImpl - switch s.trackerConfig.StreamTrackerType { - case config.StreamTrackerTypePacket: - trackerImpl = s.createStreamTrackerPacket(layer) - case config.StreamTrackerTypeFrame: - trackerImpl = s.createStreamTrackerFrame(layer) - } - if trackerImpl == nil { - return nil + var tracker streamtracker.StreamTrackerWorker + s.lock.Lock() + if s.ddTracker != nil { + tracker = s.ddTracker.LayeredTracker(layer) } + s.lock.Unlock() + if tracker == nil { + var trackerImpl streamtracker.StreamTrackerImpl + switch s.trackerConfig.StreamTrackerType { + case config.StreamTrackerTypePacket: + trackerImpl = s.createStreamTrackerPacket(layer) + case config.StreamTrackerTypeFrame: + trackerImpl = s.createStreamTrackerFrame(layer) + } + if trackerImpl == nil { + return nil + } - tracker := streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ - StreamTrackerImpl: trackerImpl, - BitrateReportInterval: bitrateInterval, - Logger: s.logger.WithValues("layer", layer), - }) + tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ + StreamTrackerImpl: trackerImpl, + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", layer), + }) + } s.logger.Debugw("StreamTrackerManager add track", "layer", layer) tracker.OnStatusChanged(func(status streamtracker.StreamStatus) { @@ -205,6 +271,8 @@ func (s *StreamTrackerManager) RemoveAllTrackers() { s.availableLayers = make([]int32, 0) s.maxExpectedLayerFromTrackInfo() s.paused = false + ddTracker := s.ddTracker + s.ddTracker = nil s.lock.Unlock() for _, tracker := range trackers { @@ -212,9 +280,12 @@ func (s *StreamTrackerManager) RemoveAllTrackers() { tracker.Stop() } } + if ddTracker != nil { + ddTracker.Stop() + } } -func (s *StreamTrackerManager) GetTracker(layer int32) *streamtracker.StreamTracker { +func (s *StreamTrackerManager) GetTracker(layer int32) streamtracker.StreamTrackerWorker { s.lock.RLock() defer s.lock.RUnlock() @@ -262,7 +333,7 @@ func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 { // But, those conditions should be rare. In those cases, the restart will // take longer. // - var trackersToReset []*streamtracker.StreamTracker + var trackersToReset []streamtracker.StreamTrackerWorker for l := s.maxExpectedLayer + 1; l <= layer; l++ { if s.hasSpatialLayerLocked(l) { continue @@ -286,18 +357,18 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { s.lock.RLock() defer s.lock.RUnlock() - if s.paused { + if s.paused || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 { return 0 } - _, brs := s.getLayeredBitrateLocked() + al, brs := s.getLayeredBitrateLocked() - maxLayers := InvalidLayers + maxLayer := buffer.InvalidLayer done: for s := int32(len(brs)) - 1; s >= 0; s-- { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[s][t] != 0 { - maxLayers = VideoLayers{ + maxLayer = buffer.VideoLayer{ Spatial: s, Temporal: t, } @@ -306,27 +377,27 @@ done: } } - distance := float64(0.0) - for sp := maxLayers.Spatial; sp <= s.getMaxExpectedLayerLocked(); sp++ { - for t := maxLayers.Temporal; t <= s.maxTemporalLayerSeen; t++ { - distance++ + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range al { + if layer > maxLayer.Spatial { + maxLayer.Spatial = layer + maxLayer.Temporal = s.maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal } } - if s.maxTemporalLayerSeen < 0 { - return distance + adjustedMaxLayers := maxLayer + if !maxLayer.IsValid() { + adjustedMaxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} } - return distance / float64(s.maxTemporalLayerSeen+1) -} - -func (s *StreamTrackerManager) getMaxExpectedLayerLocked() int32 { - // find min of layer - maxExpectedLayer := s.maxExpectedLayer - if maxExpectedLayer > s.maxPublishedLayer { - maxExpectedLayer = s.maxPublishedLayer + distance := + ((s.maxExpectedLayer - adjustedMaxLayers.Spatial) * (s.maxTemporalLayerSeen + 1)) + + (s.maxTemporalLayerSeen - adjustedMaxLayers.Temporal) + if !maxLayer.IsValid() { + distance++ } - return maxExpectedLayer + + return float64(distance) / float64(s.maxTemporalLayerSeen+1) } func (s *StreamTrackerManager) GetMaxPublishedLayer() int32 { @@ -348,7 +419,7 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { for i, tracker := range s.trackers { if tracker != nil { - tls := make([]int64, DefaultMaxLayerTemporal+1) + tls := make([]int64, buffer.DefaultMaxLayerTemporal+1) if s.hasSpatialLayerLocked(int32(i)) { tls = tracker.BitrateTemporalCumulative() } @@ -359,7 +430,8 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { } } - if s.isSVC { + // accumulate bitrates for SVC streams without dependency descriptor + if s.isSVC && s.ddTracker == nil { for i := len(br) - 1; i >= 1; i-- { for j := len(br[i]) - 1; j >= 0; j-- { if br[i][j] != 0 { @@ -407,7 +479,7 @@ func (s *StreamTrackerManager) addAvailableLayer(layer int32) { // check if new layer is the max layer isMaxLayerChange := s.availableLayers[len(s.availableLayers)-1] == layer - s.logger.Infow( + s.logger.Debugw( "available layers changed - layer seen", "added", layer, "availableLayers", s.availableLayers, @@ -425,28 +497,27 @@ func (s *StreamTrackerManager) addAvailableLayer(layer int32) { func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { s.lock.Lock() - prevMaxLayer := InvalidLayerSpatial + prevMaxLayer := buffer.InvalidLayerSpatial if len(s.availableLayers) > 0 { prevMaxLayer = s.availableLayers[len(s.availableLayers)-1] } - newLayers := make([]int32, 0, DefaultMaxLayerSpatial+1) + newLayers := make([]int32, 0, buffer.DefaultMaxLayerSpatial+1) for _, l := range s.availableLayers { - // do not remove layers for non-simulcast - if l != layer || len(s.trackInfo.Layers) < 2 { + if l != layer { newLayers = append(newLayers, l) } } sort.Slice(newLayers, func(i, j int) bool { return newLayers[i] < newLayers[j] }) s.availableLayers = newLayers - s.logger.Infow( + s.logger.Debugw( "available layers changed - layer gone", "removed", layer, "availableLayers", newLayers, ) - curMaxLayer := InvalidLayerSpatial + curMaxLayer := buffer.InvalidLayerSpatial if len(s.availableLayers) > 0 { curMaxLayer = s.availableLayers[len(s.availableLayers)-1] } @@ -464,7 +535,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { } func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { - s.maxExpectedLayer = InvalidLayerSpatial + s.maxExpectedLayer = buffer.InvalidLayerSpatial for _, layer := range s.trackInfo.Layers { spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, s.trackInfo) if spatialLayer > s.maxExpectedLayer { @@ -473,7 +544,41 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } -func (s *StreamTrackerManager) SetRTCPSenderReportDataExt(layer int32, senderReport *buffer.RTCPSenderReportDataExt) { +func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { + srRef := s.senderReports[ref].newest + srOther := s.senderReports[other].newest + if srRef == nil || srRef.NTPTimestamp == 0 || srOther == nil || srOther.NTPTimestamp == 0 { + return + } + + ntpDiff := srRef.NTPTimestamp.Time().Sub(srOther.NTPTimestamp.Time()) + if math.Abs(ntpDiff.Seconds()) > senderReportThresholdSeconds { + // offset is updated only if the layers' sender reports are close enough. + // + // Rationale: higher layers could be paused for extended periods of time + // due to adaptive stream/dynacast or publisher constraints like CPU/bandwidth. + // The check is to avoid using very old reports. + return + } + rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 + + // calculate other layer's time stamp at the same time as ref layer's NTP time + normalizedOtherTS := srOther.RTPTimestamp + uint32(rtpDiff) + + // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers + offset := srRef.RTPTimestamp - normalizedOtherTS + + // use minimal offset to indicate value availability in the extremely unlikely case of + // both layers using the same timestamp + if offset == 0 { + s.logger.Infow("using default offset", "ref", ref, "other", other) + offset = 1 + } + + s.layerOffsets[ref][other] = offset +} + +func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() @@ -481,64 +586,79 @@ func (s *StreamTrackerManager) SetRTCPSenderReportDataExt(layer int32, senderRep return } - s.senderReports[layer] = senderReport + s.senderReports[layer].first = srFirst + s.senderReports[layer].newest = srNewest + s.senderReports[layer].lastUpdated = time.Now() + + // (re)fill offsets as necessary for received layer. + for i := int32(0); i < buffer.DefaultMaxLayerSpatial+1; i++ { + if i == layer { + continue + } + + // treating layer for which report was received as reference layer + s.updateLayerOffsetLocked(layer, i) + + // and the other way + s.updateLayerOffsetLocked(i, layer) + } } -func (s *StreamTrackerManager) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { +func (s *StreamTrackerManager) GetCalculatedClockRate(layer int32) uint32 { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() if layer < 0 || int(layer) >= len(s.senderReports) { - return nil + // invalid layer + return 0 } - return s.senderReports[layer] + srFirst := s.senderReports[layer].first + srNewest := s.senderReports[layer].newest + if srFirst == nil || srFirst.NTPTimestamp == 0 || srNewest == nil || srNewest.NTPTimestamp == 0 || srFirst.RTPTimestamp == srNewest.RTPTimestamp { + // sender reports invalid or same + return 0 + } + + if s.senderReports[layer].lastUpdated.IsZero() || time.Since(s.senderReports[layer].lastUpdated).Seconds() > senderReportThresholdSeconds { + // sender report updated too far back + return 0 + } + + tsf := srNewest.NTPTimestamp.Time().Sub(srFirst.NTPTimestamp.Time()) + if tsf < minDurationForClockRateCalculation { + // not enough time has elapsed to get a stable clock rate calculation + return 0 + } + + rdsf := srNewest.RTPTimestampExt - srFirst.RTPTimestampExt + return uint32(float64(rdsf) / tsf.Seconds()) } func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() - if layer < 0 || referenceLayer < 0 { + if layer < 0 || int(layer) >= len(s.layerOffsets[0]) || referenceLayer < 0 || int(referenceLayer) >= len(s.layerOffsets) { return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) } - if layer == referenceLayer { - return ts, nil + if layer != referenceLayer && s.layerOffsets[referenceLayer][layer] == 0 { + return 0, fmt.Errorf("offset unavailable, target: %d, reference: %d", layer, referenceLayer) } - var srLayer *buffer.RTCPSenderReportDataExt - if int(layer) < len(s.senderReports) { - srLayer = s.senderReports[layer] - } - if srLayer == nil || srLayer.SenderReportData.NTPTimestamp == 0 { - return 0, fmt.Errorf("layer rtcp sender report not available: %d", layer) - } + return ts + s.layerOffsets[referenceLayer][layer], nil +} - var srRef *buffer.RTCPSenderReportDataExt - if int(referenceLayer) < len(s.senderReports) { - srRef = s.senderReports[referenceLayer] - } - if srRef == nil || srRef.SenderReportData.NTPTimestamp == 0 { - return 0, fmt.Errorf("reference layer rtcp sender report not available: %d", referenceLayer) - } +func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { + s.lock.RLock() + defer s.lock.RUnlock() - // line up the RTP time stamps using NTP time of most recent sender report of layer and referenceLayer - // NOTE: It is possible that reference layer has stopped (due to dynacast/adaptive streaming OR publisher - // constraints). It should be okay even if the layer has stopped for a long time when using modulo arithmetic for - // RTP time stamp (uint32 arithmetic). - ntpDiff := float64(int64(srRef.SenderReportData.NTPTimestamp-srLayer.SenderReportData.NTPTimestamp)) / float64(1<<32) - normalizedTS := srLayer.SenderReportData.RTPTimestamp + uint32(ntpDiff*float64(s.clockRate)) - - // now that both RTP timestamps correspond to roughly the same NTP time, - // the diff between them is the offset in RTP timestamp units between layer and referenceLayer. - // Add the offset to layer's ts to map it to corresponding RTP timestamp in - // the reference layer. - return ts + (srRef.SenderReportData.RTPTimestamp - normalizedTS), nil + return s.maxTemporalLayerSeen } func (s *StreamTrackerManager) updateMaxTemporalLayerSeen(brs Bitrates) { - maxTemporalLayerSeen := InvalidLayerTemporal + maxTemporalLayerSeen := buffer.InvalidLayerTemporal done: for t := int32(len(brs[0])) - 1; t >= 0; t-- { for s := int32(len(brs)) - 1; s >= 0; s-- { diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index a3131eeee..38640d96b 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -1,6 +1,22 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( + "time" + "github.com/pion/rtp" "github.com/pion/webrtc/v3" @@ -18,7 +34,8 @@ type TestExtPacketParams struct { SSRC uint32 PayloadSize int PaddingSize byte - ArrivalTime int64 + ArrivalTime time.Time + VideoLayer buffer.VideoLayer } // ----------------------------------------------------------- @@ -44,10 +61,11 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) { } ep := &buffer.ExtPacket{ - Arrival: params.ArrivalTime, - Packet: &packet, - KeyFrame: params.IsKeyFrame, - RawPacket: raw, + VideoLayer: params.VideoLayer, + Arrival: params.ArrivalTime, + Packet: &packet, + KeyFrame: params.IsKeyFrame, + RawPacket: raw, } return ep, nil diff --git a/pkg/sfu/utils/wraparound.go b/pkg/sfu/utils/wraparound.go new file mode 100644 index 000000000..299002fbf --- /dev/null +++ b/pkg/sfu/utils/wraparound.go @@ -0,0 +1,146 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "unsafe" +) + +type number interface { + uint16 | uint32 +} + +type extendedNumber interface { + uint32 | uint64 +} + +type WrapAround[T number, ET extendedNumber] struct { + fullRange ET + + initialized bool + start T + highest T + cycles int +} + +func NewWrapAround[T number, ET extendedNumber]() *WrapAround[T, ET] { + var t T + return &WrapAround[T, ET]{ + fullRange: 1 << (unsafe.Sizeof(t) * 8), + } +} + +func (w *WrapAround[T, ET]) Seed(from *WrapAround[T, ET]) { + w.initialized = from.initialized + w.start = from.start + w.highest = from.highest + w.cycles = from.cycles +} + +type wrapAroundUpdateResult[ET extendedNumber] struct { + IsRestart bool + PreExtendedStart ET // valid only if IsRestart = true + PreExtendedHighest ET + ExtendedVal ET +} + +func (w *WrapAround[T, ET]) Update(val T) (result wrapAroundUpdateResult[ET]) { + if !w.initialized { + result.PreExtendedHighest = ET(val) - 1 + result.ExtendedVal = ET(val) + + w.start = val + w.highest = val + w.initialized = true + return + } + + result.PreExtendedHighest = w.GetExtendedHighest() + + gap := val - w.highest + if gap == 0 || gap > T(w.fullRange>>1) { + // duplicate OR out-of-order + result.IsRestart, result.PreExtendedStart, result.ExtendedVal = w.maybeAdjustStart(val) + return + } + + // in-order + if val < w.highest { + w.cycles++ + } + w.highest = val + + result.ExtendedVal = ET(w.cycles)*w.fullRange + ET(val) + return +} + +func (w *WrapAround[T, ET]) ResetHighest(val T) { + w.highest = val +} + +func (w *WrapAround[T, ET]) GetStart() T { + return w.start +} + +func (w *WrapAround[T, ET]) GetExtendedStart() ET { + return ET(w.start) +} + +func (w *WrapAround[T, ET]) GetHighest() T { + return w.highest +} + +func (w *WrapAround[T, ET]) GetExtendedHighest() ET { + return ET(w.cycles)*w.fullRange + ET(w.highest) +} + +func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (isRestart bool, preExtendedStart ET, extendedVal ET) { + isWrapBack := func() bool { + return ET(w.highest) < (w.fullRange>>1) && ET(val) >= (w.fullRange>>1) + } + + // re-adjust start if necessary. The conditions are + // 1. Not seen more than half the range yet + // 1. wrap around compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space + // 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space + cycles := w.cycles + totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1 + if totalNum > (w.fullRange >> 1) { + if isWrapBack() { + cycles-- + } + extendedVal = ET(cycles)*w.fullRange + ET(val) + return + } + + if val-w.start > T(w.fullRange>>1) { + // out-of-order with existing start => a new start + isRestart = true + preExtendedStart = w.GetExtendedStart() + + if val > w.highest { + // wrap around + w.cycles = 1 + cycles = 0 + } + w.start = val + } else { + if isWrapBack() { + cycles-- + } + } + extendedVal = ET(cycles)*w.fullRange + ET(val) + return +} diff --git a/pkg/sfu/utils/wraparound_test.go b/pkg/sfu/utils/wraparound_test.go new file mode 100644 index 000000000..9e3b8e555 --- /dev/null +++ b/pkg/sfu/utils/wraparound_test.go @@ -0,0 +1,339 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWrapAroundUint16(t *testing.T) { + w := NewWrapAround[uint16, uint32]() + testCases := []struct { + name string + input uint16 + updated wrapAroundUpdateResult[uint32] + start uint16 + extendedStart uint32 + highest uint16 + extendedHighest uint32 + }{ + // initialize + { + name: "initialize", + input: 10, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: 9, + ExtendedVal: 10, + }, + start: 10, + extendedStart: 10, + highest: 10, + extendedHighest: 10, + }, + // an older number without wrap around should reset start point + { + name: "reset start no wrap around", + input: 8, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: 10, + PreExtendedHighest: 10, + ExtendedVal: 8, + }, + start: 8, + extendedStart: 8, + highest: 10, + extendedHighest: 10, + }, + // an older number with wrap around should reset start point + { + name: "reset start wrap around", + input: (1 << 16) - 6, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: 8, + PreExtendedHighest: 10, + ExtendedVal: (1 << 16) - 6, + }, + start: (1 << 16) - 6, + extendedStart: (1 << 16) - 6, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // an older number with wrap around should reset start point again + { + name: "reset start again", + input: (1 << 16) - 12, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: (1 << 16) - 6, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) - 12, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // out of order with highest, wrap back, but no restart + { + name: "out of order - no restart", + input: (1 << 16) - 3, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) - 3, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // duplicate should return same as highest + { + name: "duplicate", + input: 10, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) + 10, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // a significant jump in order should not reset start + { + name: "big in-order jump", + input: (1 << 15) - 10, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) + (1 << 15) - 10, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, + }, + // now out-of-order should not reset start as half the range has been seen + { + name: "out-of-order after half range", + input: (1 << 15) - 11, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, + ExtendedVal: (1 << 16) + (1 << 15) - 11, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, + }, + // wrap back out-of-order + { + name: "wrap back out-of-order after half range", + input: (1 << 16) - 1, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, + ExtendedVal: (1 << 16) - 1, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, + }, + // in-order, should update highest + { + name: "in-order", + input: (1 << 15) + 3, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, + ExtendedVal: (1 << 16) + (1 << 15) + 3, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) + 3, + extendedHighest: (1 << 16) + (1 << 15) + 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.updated, w.Update(tc.input)) + require.Equal(t, tc.start, w.GetStart()) + require.Equal(t, tc.extendedStart, w.GetExtendedStart()) + require.Equal(t, tc.highest, w.GetHighest()) + require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) + }) + } +} + +func TestWrapAroundUint32(t *testing.T) { + w := NewWrapAround[uint32, uint64]() + testCases := []struct { + name string + input uint32 + updated wrapAroundUpdateResult[uint64] + start uint32 + extendedStart uint64 + highest uint32 + extendedHighest uint64 + }{ + // initialize + { + name: "initialize", + input: 10, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: 9, + ExtendedVal: 10, + }, + start: 10, + extendedStart: 10, + highest: 10, + extendedHighest: 10, + }, + // an older number without wrap around should reset start point + { + name: "reset start no wrap around", + input: 8, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: 10, + PreExtendedHighest: 10, + ExtendedVal: 8, + }, + start: 8, + extendedStart: 8, + highest: 10, + extendedHighest: 10, + }, + // an older number with wrap around should reset start point + { + name: "reset start wrap around", + input: (1 << 32) - 6, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: 8, + PreExtendedHighest: 10, + ExtendedVal: (1 << 32) - 6, + }, + start: (1 << 32) - 6, + extendedStart: (1 << 32) - 6, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // an older number with wrap around should reset start point again + { + name: "reset start again", + input: (1 << 32) - 12, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: (1 << 32) - 6, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) - 12, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // duplicate should return same as highest + { + name: "duplicate", + input: 10, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) + 10, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // a significant jump in order should not reset start + { + name: "big in-order jump", + input: 1 << 31, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) + (1 << 31), + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 1 << 31, + extendedHighest: (1 << 32) + (1 << 31), + }, + // now out-of-order should not reset start as half the range has been seen + { + name: "out-of-order after half range", + input: (1 << 31) - 1, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + (1 << 31), + ExtendedVal: (1 << 32) + (1 << 31) - 1, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 1 << 31, + extendedHighest: (1 << 32) + (1 << 31), + }, + // in-order, should update highest + { + name: "in-order", + input: (1 << 31) + 3, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + (1 << 31), + ExtendedVal: (1 << 32) + (1 << 31) + 3, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: (1 << 31) + 3, + extendedHighest: (1 << 32) + (1 << 31) + 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.updated, w.Update(tc.input)) + require.Equal(t, tc.start, w.GetStart()) + require.Equal(t, tc.extendedStart, w.GetExtendedStart()) + require.Equal(t, tc.highest, w.GetHighest()) + require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) + }) + } +} diff --git a/pkg/sfu/videolayerselector.go b/pkg/sfu/videolayerselector.go deleted file mode 100644 index 507bfe49f..000000000 --- a/pkg/sfu/videolayerselector.go +++ /dev/null @@ -1,202 +0,0 @@ -package sfu - -import ( - "fmt" - "sort" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" - "github.com/livekit/protocol/logger" -) - -type targetLayer struct { - Target int - Layer VideoLayers -} - -type DDVideoLayerSelector struct { - logger logger.Logger - - // DD-TODO : fields for frame chain detect - // frameNumberWrapper Uint16Wrapper - // expectKeyFrame bool - - decodeTargetLayer []targetLayer - layer VideoLayers - activeDecodeTargetsBitmask *uint32 - structure *dd.FrameDependencyStructure -} - -func NewDDVideoLayerSelector(logger logger.Logger) *DDVideoLayerSelector { - return &DDVideoLayerSelector{ - logger: logger, - layer: VideoLayers{Spatial: 2, Temporal: 2}, - } -} - -func (s *DDVideoLayerSelector) Select(expPkt *buffer.ExtPacket, tp *TranslationParams) (selected bool) { - tp.marker = expPkt.Packet.Marker - if expPkt.DependencyDescriptor == nil { - // packet don't have dependency descriptor, pass check - return true - } - - if expPkt.DependencyDescriptor.AttachedStructure != nil { - // update decode target layer and active decode targets - // DD-TODO : these targets info can be shared by all the downtracks, no need calculate in every selector - s.updateDependencyStructure(expPkt.DependencyDescriptor.AttachedStructure) - } - - // forward all packets before locking - if s.layer == InvalidLayers { - return true - } - - // DD-TODO : we don't have a rtp queue to ensure the order of packets now, - // so we don't know packet is lost/out of order, that cause us can't detect - // frame integrity, entire frame is forwareded, whether frame chain is broken. - // So use a simple check here, assume all the reference frame is forwarded and - // only check DTI of the active decode target. - // it is not effeciency, at last we need check frame chain integrity. - - activeDecodeTargets := expPkt.DependencyDescriptor.ActiveDecodeTargetsBitmask - if activeDecodeTargets != nil { - s.logger.Debugw("active decode targets", "activeDecodeTargets", *activeDecodeTargets) - } - - currentTarget := -1 - for _, dt := range s.decodeTargetLayer { - // find target match with selected layer - if dt.Layer.Spatial <= s.layer.Spatial && dt.Layer.Temporal <= s.layer.Temporal { - if activeDecodeTargets == nil || ((*activeDecodeTargets)&(1< maxSpatial { - maxSpatial = dt.Layer.Spatial - } - if dt.Layer.Temporal > maxTemporal { - maxTemporal = dt.Layer.Temporal - } - if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal { - activeBitMask |= 1 << dt.Target - } - } - if layer.Spatial == maxSpatial && layer.Temporal == maxTemporal { - // all the decode targets are selected - s.activeDecodeTargetsBitmask = nil - } else { - s.activeDecodeTargetsBitmask = &activeBitMask - } - s.logger.Debugw("select layer ", "layer", layer, "activeDecodeTargetsBitmask", s.activeDecodeTargetsBitmask) -} - -func (s *DDVideoLayerSelector) updateDependencyStructure(structure *dd.FrameDependencyStructure) { - s.structure = structure - s.decodeTargetLayer = s.decodeTargetLayer[:0] - - for target := 0; target < structure.NumDecodeTargets; target++ { - layer := VideoLayers{Spatial: 0, Temporal: 0} - for _, t := range structure.Templates { - if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { - if layer.Spatial < int32(t.SpatialId) { - layer.Spatial = int32(t.SpatialId) - } - if layer.Temporal < int32(t.TemporalId) { - layer.Temporal = int32(t.TemporalId) - } - } - } - s.decodeTargetLayer = append(s.decodeTargetLayer, targetLayer{target, layer}) - } - - // sort decode target layer by spatial and temporal from high to low - sort.Slice(s.decodeTargetLayer, func(i, j int) bool { - if s.decodeTargetLayer[i].Layer.Spatial == s.decodeTargetLayer[j].Layer.Spatial { - return s.decodeTargetLayer[i].Layer.Temporal > s.decodeTargetLayer[j].Layer.Temporal - } - return s.decodeTargetLayer[i].Layer.Spatial > s.decodeTargetLayer[j].Layer.Spatial - }) - s.logger.Debugw(fmt.Sprintf("update decode targets: %v", s.decodeTargetLayer)) -} - -// DD-TODO : use generic wrapper when updated to go 1.18 -type Uint16Wrapper struct { - last_value *uint16 - lastUnwrapped int32 -} - -func (w *Uint16Wrapper) Unwrap(value uint16) int32 { - if w.last_value == nil { - w.last_value = &value - w.lastUnwrapped = int32(value) - return int32(*w.last_value) - } - - diff := value - *w.last_value - w.lastUnwrapped += int32(diff) - if diff == 0x8000 && value < *w.last_value { - w.lastUnwrapped -= 0x10000 - } else if diff > 0x8000 { - w.lastUnwrapped -= 0x10000 - } - - *w.last_value = value - return w.lastUnwrapped -} diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go new file mode 100644 index 000000000..37b223948 --- /dev/null +++ b/pkg/sfu/videolayerselector/base.go @@ -0,0 +1,186 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" + "github.com/livekit/protocol/logger" +) + +type Base struct { + logger logger.Logger + + tls temporallayerselector.TemporalLayerSelector + + maxLayer buffer.VideoLayer + maxSeenLayer buffer.VideoLayer + + targetLayer buffer.VideoLayer + previousTargetLayer buffer.VideoLayer + + requestSpatial int32 + + parkedLayer buffer.VideoLayer + previousParkedLayer buffer.VideoLayer + + currentLayer buffer.VideoLayer + previousLayer buffer.VideoLayer +} + +func NewBase(logger logger.Logger) *Base { + return &Base{ + logger: logger, + maxLayer: buffer.InvalidLayer, + maxSeenLayer: buffer.InvalidLayer, + targetLayer: buffer.InvalidLayer, // start off with nothing, let streamallocator/opportunistic forwarder set the target + previousTargetLayer: buffer.InvalidLayer, + requestSpatial: buffer.InvalidLayerSpatial, + parkedLayer: buffer.InvalidLayer, + previousParkedLayer: buffer.InvalidLayer, + currentLayer: buffer.InvalidLayer, + previousLayer: buffer.InvalidLayer, + } +} + +func (b *Base) IsOvershootOkay() bool { + return false +} + +func (b *Base) SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector) { + b.tls = tls +} + +func (b *Base) SetMax(maxLayer buffer.VideoLayer) { + b.maxLayer = maxLayer +} + +func (b *Base) SetMaxSpatial(layer int32) { + b.maxLayer.Spatial = layer +} + +func (b *Base) SetMaxTemporal(layer int32) { + b.maxLayer.Temporal = layer +} + +func (b *Base) GetMax() buffer.VideoLayer { + return b.maxLayer +} + +func (b *Base) SetTarget(targetLayer buffer.VideoLayer) { + b.previousTargetLayer = targetLayer + b.targetLayer = targetLayer +} + +func (b *Base) GetTarget() buffer.VideoLayer { + return b.targetLayer +} + +func (b *Base) SetRequestSpatial(layer int32) { + b.requestSpatial = layer +} + +func (b *Base) GetRequestSpatial() int32 { + return b.requestSpatial +} + +func (b *Base) CheckSync() (locked bool, layer int32) { + layer = b.GetRequestSpatial() + locked = layer == b.GetCurrent().Spatial || b.GetParked().IsValid() + return +} + +func (b *Base) SetMaxSeen(maxSeenLayer buffer.VideoLayer) { + b.maxSeenLayer = maxSeenLayer +} + +func (b *Base) SetMaxSeenSpatial(layer int32) { + b.maxSeenLayer.Spatial = layer +} + +func (b *Base) SetMaxSeenTemporal(layer int32) { + b.maxSeenLayer.Temporal = layer +} + +func (b *Base) GetMaxSeen() buffer.VideoLayer { + return b.maxSeenLayer +} + +func (b *Base) SetParked(parkedLayer buffer.VideoLayer) { + b.parkedLayer = parkedLayer +} + +func (b *Base) GetParked() buffer.VideoLayer { + return b.parkedLayer +} + +func (b *Base) SetCurrent(currentLayer buffer.VideoLayer) { + b.currentLayer = currentLayer +} + +func (b *Base) GetCurrent() buffer.VideoLayer { + return b.currentLayer +} + +func (b *Base) Select(_extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + return +} + +func (b *Base) Rollback() { + b.logger.Infow( + "rolling back", + "previous", b.previousLayer, + "current", b.currentLayer, + "previousParked", b.previousParkedLayer, + "parked", b.parkedLayer, + "previousTarget", b.previousTargetLayer, + "target", b.targetLayer, + "max", b.maxLayer, + "req", b.requestSpatial, + "maxSeen", b.maxSeenLayer, + ) + b.parkedLayer = b.previousParkedLayer + b.currentLayer = b.previousLayer + b.targetLayer = b.previousTargetLayer +} + +func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) { + if b.tls != nil { + isSwitching := false + this, next := b.tls.Select(extPkt, b.currentLayer.Temporal, b.targetLayer.Temporal) + if next != b.currentLayer.Temporal { + isSwitching = true + + b.previousLayer = b.currentLayer + b.currentLayer.Temporal = next + + b.logger.Infow( + "updating temporal layer", + "previous", b.previousLayer, + "current", b.currentLayer, + "previousParked", b.previousParkedLayer, + "parked", b.parkedLayer, + "previousTarget", b.previousTargetLayer, + "target", b.targetLayer, + "max", b.maxLayer, + "req", b.requestSpatial, + "maxSeen", b.maxSeenLayer, + ) + } + return this, isSwitching + } + + return b.currentLayer.Temporal, false +} diff --git a/pkg/sfu/videolayerselector/decodetarget.go b/pkg/sfu/videolayerselector/decodetarget.go new file mode 100644 index 000000000..7204b329f --- /dev/null +++ b/pkg/sfu/videolayerselector/decodetarget.go @@ -0,0 +1,70 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "fmt" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +type DecodeTarget struct { + buffer.DependencyDescriptorDecodeTarget + chain *FrameChain + active bool +} + +type FrameDetectionResult struct { + TargetValid bool + DTI dd.DecodeTargetIndication +} + +func NewDecodeTarget(target buffer.DependencyDescriptorDecodeTarget, chain *FrameChain) *DecodeTarget { + return &DecodeTarget{ + DependencyDescriptorDecodeTarget: target, + chain: chain, + } +} + +func (dt *DecodeTarget) Valid() bool { + return dt.chain == nil || !dt.chain.Broken() +} + +func (dt *DecodeTarget) Active() bool { + return dt.active +} + +func (dt *DecodeTarget) UpdateActive(activeBitmask uint32) { + active := (activeBitmask & (1 << dt.Target)) != 0 + dt.active = active + if dt.chain != nil { + dt.chain.UpdateActive(active) + } +} + +func (dt *DecodeTarget) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate) (FrameDetectionResult, error) { + result := FrameDetectionResult{} + if len(fd.DecodeTargetIndications) <= dt.Target { + return result, fmt.Errorf("mismatch target %d and len(DecodeTargetIndications) %d", dt.Target, len(fd.DecodeTargetIndications)) + } + + result.DTI = fd.DecodeTargetIndications[dt.Target] + // The encoder can choose not to use frame chain in theory, and we need to trace every required frame is decodable in this case. + // But we don't observe this in browser and it makes no sense to not use the chain with svc, so only use chain to detect decode target broken now, + // and always return decodable if it is not protect by chain. + result.TargetValid = dt.Valid() + return result, nil +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go new file mode 100644 index 000000000..555437c6d --- /dev/null +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -0,0 +1,301 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "fmt" + "sync" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dede "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +type DependencyDescriptor struct { + *Base + + decisions *SelectorDecisionCache + + previousActiveDecodeTargetsBitmask *uint32 + activeDecodeTargetsBitmask *uint32 + structure *dede.FrameDependencyStructure + + chains []*FrameChain + + decodeTargetsLock sync.RWMutex + decodeTargets []*DecodeTarget +} + +func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { + return &DependencyDescriptor{ + Base: NewBase(logger), + decisions: NewSelectorDecisionCache(256, 80), + } +} + +func NewDependencyDescriptorFromNull(vls VideoLayerSelector) *DependencyDescriptor { + return &DependencyDescriptor{ + Base: vls.(*Null).Base, + decisions: NewSelectorDecisionCache(256, 80), + } +} + +func (d *DependencyDescriptor) IsOvershootOkay() bool { + return false +} + +func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + // a packet is always relevant for the svc codec + result.IsRelevant = true + + ddwdt := extPkt.DependencyDescriptor + if ddwdt == nil { + // packet doesn't have dependency descriptor + return + } + + dd := ddwdt.Descriptor + + extFrameNum := ddwdt.ExtFrameNum + + fd := dd.FrameDependencies + incomingLayer := buffer.VideoLayer{ + Spatial: int32(fd.SpatialId), + Temporal: int32(fd.TemporalId), + } + + // early return if this frame is already forwarded or dropped + sd, err := d.decisions.GetDecision(extFrameNum) + if err != nil { + // do not mark as dropped as only error is an old frame + return + } + switch sd { + case selectorDecisionDropped: + // a packet of an alreadty dropped frame, maintain decision + return + } + + if !d.currentLayer.IsValid() && !extPkt.KeyFrame { + d.decisions.AddDropped(extFrameNum) + return + } + + if ddwdt.StructureUpdated { + d.updateDependencyStructure(dd.AttachedStructure, ddwdt.DecodeTargets) + } + + if ddwdt.ActiveDecodeTargetsUpdated { + d.updateActiveDecodeTargets(*dd.ActiveDecodeTargetsBitmask) + } + + for _, chain := range d.chains { + chain.OnFrame(extFrameNum, fd) + } + + // find decode target closest to targetLayer + highestDecodeTarget := buffer.DependencyDescriptorDecodeTarget{ + Target: -1, + Layer: buffer.InvalidLayer, + } + var dti dede.DecodeTargetIndication + d.decodeTargetsLock.RLock() + + // decodeTargets be sorted from high to low, find the highest decode target that is active and integrity + for _, dt := range d.decodeTargets { + if !dt.Active() || dt.Layer.Spatial > d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal { + continue + } + + frameResult, err := dt.OnFrame(extFrameNum, fd) + if err != nil { + d.decodeTargetsLock.RUnlock() + // dtis error, dependency descriptor might lost + d.logger.Debugw(fmt.Sprintf("drop packet for frame detection error, incoming: %v", + incomingLayer, + ), "err", err) + d.decisions.AddDropped(extFrameNum) + return + } + + if frameResult.TargetValid { + highestDecodeTarget = dt.DependencyDescriptorDecodeTarget + dti = frameResult.DTI + break + } + } + d.decodeTargetsLock.RUnlock() + + if highestDecodeTarget.Target < 0 { + // no active decode target, do not select + // d.logger.Debugw(fmt.Sprintf("drop packet for no target found, decodeTargets %v, tagetLayer %v, incoming %v", + // d.decodeTargets, + // d.targetLayer, + // incomingLayer, + // )) + d.decisions.AddDropped(extFrameNum) + return + } + + // // DD-TODO : if bandwidth in congest, could drop the 'Discardable' frame + if dti == dede.DecodeTargetNotPresent { + // d.logger.Debugw(fmt.Sprintf("drop packet for decode target not present, highestDecodeTarget %d, incoming %v, fn: %d/%d", + // highestDecodeTarget, + // incomingLayer, + // dd.FrameNumber, + // extFrameNum, + // )) + d.decisions.AddDropped(extFrameNum) + return + } + + // check decodability using reference frames + isDecodable := true + for _, fdiff := range fd.FrameDiffs { + if fdiff == 0 { + continue + } + + // use relaxed check for frame diff that we have chain intact detection and don't want + // to drop packet due to out-of-order packet or recoverable packet loss + if sd, _ := d.decisions.GetDecision(extFrameNum - uint64(fdiff)); sd == selectorDecisionDropped { + isDecodable = false + break + } + } + if !isDecodable { + d.decisions.AddDropped(extFrameNum) + return + } + + if d.currentLayer != highestDecodeTarget.Layer { + result.IsSwitching = true + if !d.currentLayer.IsValid() { + result.IsResuming = true + d.logger.Infow( + "resuming at layer", + "current", incomingLayer, + "target", d.targetLayer, + "max", d.maxLayer, + "layer", fd.SpatialId, + "req", d.requestSpatial, + "maxSeen", d.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + } + + d.previousLayer = d.currentLayer + d.currentLayer = highestDecodeTarget.Layer + + d.previousActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask + d.activeDecodeTargetsBitmask = buffer.GetActiveDecodeTargetBitmask(d.currentLayer, ddwdt.DecodeTargets) + d.logger.Debugw("switch to target", "highest", highestDecodeTarget.Layer, "current", d.currentLayer, "bitmask", *d.activeDecodeTargetsBitmask) + } + + ddExtension := &dede.DependencyDescriptorExtension{ + Descriptor: dd, + Structure: d.structure, + } + if dd.AttachedStructure == nil { + if d.activeDecodeTargetsBitmask != nil { + // clone and override activebitmask + // DD-TODO: if the packet that contains the bitmask is acknowledged by RR, then we don't need it until it changed. + ddClone := *ddExtension.Descriptor + ddClone.ActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask + ddExtension.Descriptor = &ddClone + // d.logger.Debugw("set active decode targets bitmask", "activeDecodeTargetsBitmask", d.activeDecodeTargetsBitmask) + } + } + bytes, err := ddExtension.Marshal() + if err != nil { + d.logger.Warnw("error marshalling dependency descriptor extension", err) + } else { + result.DependencyDescriptorExtension = bytes + } + + if ddwdt.Integrity { + d.decisions.AddForwarded(extFrameNum) + } + result.RTPMarker = extPkt.Packet.Header.Marker || (dd.LastPacketInFrame && d.currentLayer.Spatial == int32(fd.SpatialId)) + result.IsSelected = true + return +} + +func (d *DependencyDescriptor) Rollback() { + d.activeDecodeTargetsBitmask = d.previousActiveDecodeTargetsBitmask + + d.Base.Rollback() +} + +func (d *DependencyDescriptor) updateDependencyStructure(structure *dede.FrameDependencyStructure, decodeTargets []buffer.DependencyDescriptorDecodeTarget) { + d.structure = structure + + d.chains = d.chains[:0] + + for chainIdx := 0; chainIdx < structure.NumChains; chainIdx++ { + d.chains = append(d.chains, NewFrameChain(d.decisions, chainIdx, d.logger)) + } + + newTargets := make([]*DecodeTarget, 0, len(decodeTargets)) + for _, dt := range decodeTargets { + var chain *FrameChain + // When chain_cnt > 0, each Decode target MUST be protected by exactly one Chain. + if structure.NumChains > 0 { + chainIdx := structure.DecodeTargetProtectedByChain[dt.Target] + if chainIdx >= len(d.chains) { + // should not happen + d.logger.Errorw("DecodeTargetProtectedByChain chainIdx out of range", nil, "chainIdx", chainIdx, "NumChains", len(d.chains)) + } else { + chain = d.chains[chainIdx] + } + } + newTargets = append(newTargets, NewDecodeTarget(dt, chain)) + } + d.decodeTargetsLock.Lock() + d.decodeTargets = newTargets + d.decodeTargetsLock.Unlock() +} + +func (d *DependencyDescriptor) updateActiveDecodeTargets(activeDecodeTargetsBitmask uint32) { + for _, chain := range d.chains { + chain.BeginUpdateActive() + } + + d.decodeTargetsLock.RLock() + for _, dt := range d.decodeTargets { + dt.UpdateActive(activeDecodeTargetsBitmask) + } + d.decodeTargetsLock.RUnlock() + + for _, chain := range d.chains { + chain.EndUpdateActive() + } +} + +func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) { + layer = d.GetRequestSpatial() + if d.GetParked().IsValid() { + return true, layer + } + + d.decodeTargetsLock.RLock() + defer d.decodeTargetsLock.RUnlock() + for _, dt := range d.decodeTargets { + if dt.Active() && dt.Layer.Spatial == layer && dt.Valid() { + return true, layer + } + } + return false, layer +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go new file mode 100644 index 000000000..e9e158fda --- /dev/null +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -0,0 +1,396 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "sort" + "testing" + + "github.com/pion/rtp" + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +func TestDecodeTarget(t *testing.T) { + target := buffer.DependencyDescriptorDecodeTarget{ + Target: 1, + Layer: buffer.VideoLayer{Spatial: 1, Temporal: 2}, + } + + t.Run("No Chain", func(t *testing.T) { + dt := NewDecodeTarget(target, nil) + require.True(t, dt.Valid()) + // no indication found + _, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{}, + }) + require.Error(t, err) + + ret, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired}, + }) + require.NoError(t, err) + require.True(t, ret.TargetValid) + require.Equal(t, dd.DecodeTargetRequired, ret.DTI) + }) + + t.Run("With Chain", func(t *testing.T) { + decisions := NewSelectorDecisionCache(256, 80) + chain := NewFrameChain(decisions, 1, logger.GetLogger()) + dt := NewDecodeTarget(target, chain) + chain.BeginUpdateActive() + dt.UpdateActive(1 << dt.Target) + chain.EndUpdateActive() + require.True(t, dt.Active()) + require.False(t, dt.Valid()) + + // chain intact + frame := &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired}, + ChainDiffs: []int{0, 0}, + } + chain.OnFrame(1, frame) + require.True(t, dt.Valid()) + ret, err := dt.OnFrame(1, frame) + require.NoError(t, err) + require.True(t, ret.TargetValid) + require.Equal(t, dd.DecodeTargetRequired, ret.DTI) + + }) +} + +func TestFrameChain(t *testing.T) { + decisions := NewSelectorDecisionCache(256, 3) + chain := NewFrameChain(decisions, 0, logger.GetLogger()) + require.True(t, chain.Broken()) + + // chain intact + frameNoDiff := &dd.FrameDependencyTemplate{ + ChainDiffs: []int{0}, + } + // not active + require.False(t, chain.OnFrame(1, frameNoDiff)) + + chain.BeginUpdateActive() + chain.UpdateActive(true) + chain.EndUpdateActive() + + require.True(t, chain.OnFrame(1, frameNoDiff)) + decisions.AddForwarded(1) + + frameDiff1 := &dd.FrameDependencyTemplate{ + ChainDiffs: []int{1}, + } + + require.True(t, chain.OnFrame(2, frameDiff1)) + decisions.AddForwarded(2) + + // frame 5 arrives first , but frame 4 can be recovered by NACK + require.True(t, chain.OnFrame(5, frameDiff1)) + decisions.AddForwarded(5) + + // frame 4 arrives, chain remains intact + require.True(t, chain.OnFrame(4, frameDiff1)) + decisions.AddForwarded(4) + + // frame 3 missed by out of nack range, chain broken + decisions.AddForwarded(7) + require.True(t, chain.Broken()) + + // recovery by non-diff frame + require.True(t, chain.OnFrame(1000, frameNoDiff)) + require.False(t, chain.Broken()) + decisions.AddForwarded(1000) + + // broken by dropped frame + require.True(t, chain.OnFrame(1002, frameDiff1)) + decisions.AddDropped(1001) + require.True(t, chain.Broken()) + + // recovery by non-diff frame + require.True(t, chain.OnFrame(2000, frameNoDiff)) + decisions.AddForwarded(2000) + decisions.AddDropped(2001) + require.False(t, chain.OnFrame(2002, frameDiff1)) + require.True(t, chain.Broken()) +} + +func TestDependencyDescriptor(t *testing.T) { + ddSelector := NewDependencyDescriptor(logger.GetLogger()) + targetLayer := buffer.VideoLayer{Spatial: 1, Temporal: 2} + ddSelector.SetTarget(targetLayer) + ddSelector.SetRequestSpatial(1) + + // no dd ext, dropped + ret := ddSelector.Select(&buffer.ExtPacket{}, 0) + require.False(t, ret.IsSelected) + require.True(t, ret.IsRelevant) + + // non key frame, dropped + ret = ddSelector.Select(&buffer.ExtPacket{ + KeyFrame: false, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: 1, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: int(targetLayer.Spatial), + TemporalId: int(targetLayer.Temporal), + }, + }, + }, + }, 0) + require.False(t, ret.IsSelected) + require.True(t, ret.IsRelevant) + + frames := createDDFrames(buffer.VideoLayer{Spatial: 2, Temporal: 2}, 3) + // key frame, update structure and decode targets + ret = ddSelector.Select(frames[0], 0) + require.True(t, ret.IsSelected) + require.Equal(t, ddSelector.GetCurrent(), ddSelector.GetTarget()) + sync, _ := ddSelector.CheckSync() + require.True(t, sync) + + // forward frame belongs to target layer + // drop frame exceeds target layer (not present in target layer or lower layer) + // forward frame not present in target layer but present in lower layer + var ( + belongTargetCase bool + exceedTargetCase bool + lowerTargetCase bool + ) + idx := 1 + var frameForwarded, frameDropped []*buffer.ExtPacket + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + switch { + case fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal): + require.True(t, ret.IsSelected) + belongTargetCase = true + frameForwarded = append(frameForwarded, frames[idx]) + case fd.SpatialId < int(targetLayer.Spatial) && fd.TemporalId == 0: + require.True(t, ret.IsSelected) + lowerTargetCase = true + frameForwarded = append(frameForwarded, frames[idx]) + case fd.SpatialId > int(targetLayer.Spatial) || fd.TemporalId > int(targetLayer.Temporal): + require.False(t, ret.IsSelected) + exceedTargetCase = true + frameDropped = append(frameDropped, frames[idx]) + } + + if belongTargetCase && exceedTargetCase && lowerTargetCase { + break + } + } + + require.True(t, belongTargetCase && exceedTargetCase && lowerTargetCase) + + // select frame already forwarded + ret = ddSelector.Select(frameForwarded[0], 0) + require.True(t, ret.IsSelected) + + // drop frame already dropped + ret = ddSelector.Select(frameDropped[0], 0) + require.False(t, ret.IsSelected) + + // drop frame present but not decodable (dependency frame missed) + idx++ + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) { + break + } + } + notDecodableFrame := frames[idx] + notDecodableFrame.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs = []int{ + int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber), + } + ret = ddSelector.Select(notDecodableFrame, 0) + require.False(t, ret.IsSelected) + + // target layer broken + idx++ + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) { + break + } + } + brokenFrame := frames[idx] + brokenFrame.DependencyDescriptor.Descriptor.FrameDependencies.ChainDiffs[targetLayer.Spatial] = + int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber) + ret = ddSelector.Select(brokenFrame, 0) + require.False(t, ret.IsSelected) + + // switch to lower layer, forward frame + idx++ + var switchToLower bool + for ; idx < len(frames); idx++ { + ret = ddSelector.Select(frames[idx], 0) + if ret.IsSelected { + require.True(t, targetLayer.GreaterThan(ddSelector.GetCurrent())) + switchToLower = true + break + } + } + require.True(t, switchToLower) + + // not sync with requested layer + ddSelector.SetRequestSpatial(targetLayer.Spatial) + locked, layer := ddSelector.CheckSync() + require.False(t, locked) + require.Equal(t, targetLayer.Spatial, layer) + + // request to current layer, sync + ddSelector.SetRequestSpatial(ddSelector.GetCurrent().Spatial) + locked, _ = ddSelector.CheckSync() + require.True(t, locked) +} + +func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buffer.ExtPacket { + var frames []*buffer.ExtPacket + var activeBitMask uint32 + var decodeTargets []buffer.DependencyDescriptorDecodeTarget + var decodeTargetsProtectByChain []int + for i := 0; i <= int(maxLayer.Spatial); i++ { + for j := 0; j <= int(maxLayer.Temporal); j++ { + decodeTargets = append(decodeTargets, buffer.DependencyDescriptorDecodeTarget{ + Target: i*int(maxLayer.Temporal+1) + j, + Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}, + }) + decodeTargetsProtectByChain = append(decodeTargetsProtectByChain, i) + activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j) + } + } + sort.Slice(decodeTargets, func(i, j int) bool { + return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer) + }) + + chainDiffs := make([]int, len(decodeTargets)) + dtis := make([]dd.DecodeTargetIndication, len(decodeTargets)) + for _, dt := range decodeTargets { + dtis[dt.Target] = dd.DecodeTargetSwitch + } + + templates := make([]*dd.FrameDependencyTemplate, len(decodeTargets)) + + for _, dt := range decodeTargets { + templates[dt.Target] = &dd.FrameDependencyTemplate{ + SpatialId: int(dt.Layer.Spatial), + TemporalId: int(dt.Layer.Temporal), + ChainDiffs: chainDiffs, + DecodeTargetIndications: dtis, + } + } + keyFrame := &buffer.ExtPacket{ + KeyFrame: true, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: startFrameNumber, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: 0, + TemporalId: 0, + ChainDiffs: chainDiffs, + DecodeTargetIndications: dtis, + }, + AttachedStructure: &dd.FrameDependencyStructure{ + NumDecodeTargets: int((maxLayer.Spatial + 1) * (maxLayer.Temporal + 1)), + NumChains: int(maxLayer.Spatial) + 1, + DecodeTargetProtectedByChain: decodeTargetsProtectByChain, + Templates: templates, + }, + ActiveDecodeTargetsBitmask: &activeBitMask, + }, + DecodeTargets: decodeTargets, + StructureUpdated: true, + ActiveDecodeTargetsUpdated: true, + Integrity: true, + ExtFrameNum: uint64(startFrameNumber), + }, + Packet: &rtp.Packet{ + Header: rtp.Header{ + SSRC: 1234, + }, + }, + } + + frames = append(frames, keyFrame) + + chainPrevFrame := make(map[int]int) + for i := 0; i <= int(maxLayer.Spatial); i++ { + chainPrevFrame[i] = int(startFrameNumber) + } + startFrameNumber++ + for i := 0; i < 10; i++ { + for j := len(decodeTargets) - 1; j >= 0; j-- { + dt := decodeTargets[j] + frameChainDiffs := make([]int, len(chainDiffs)) + for i := range frameChainDiffs { + frameChainDiffs[i] = int(startFrameNumber) - chainPrevFrame[i] + } + + frameDtis := make([]dd.DecodeTargetIndication, len(dtis)) + for k := range frameDtis { + if k >= dt.Target { + if dt.Layer.Temporal == 0 { + frameDtis[k] = dd.DecodeTargetRequired + } else { + frameDtis[k] = dd.DecodeTargetDiscardable + } + } else { + frameDtis[k] = dd.DecodeTargetNotPresent + } + } + + frame := &buffer.ExtPacket{ + KeyFrame: true, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: startFrameNumber, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: int(dt.Layer.Spatial), + TemporalId: int(dt.Layer.Temporal), + ChainDiffs: frameChainDiffs, + DecodeTargetIndications: frameDtis, + }, + }, + DecodeTargets: decodeTargets, + Integrity: true, + ExtFrameNum: uint64(startFrameNumber), + }, + Packet: &rtp.Packet{ + Header: rtp.Header{ + SSRC: 1234, + }, + }, + } + + startFrameNumber++ + + if dt.Layer.Temporal == 0 { + chainPrevFrame[int(dt.Layer.Spatial)] = int(startFrameNumber) + } + + frames = append(frames, frame) + } + } + + return frames +} diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go new file mode 100644 index 000000000..5edddfad9 --- /dev/null +++ b/pkg/sfu/videolayerselector/framechain.go @@ -0,0 +1,133 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +type FrameChain struct { + logger logger.Logger + decisions *SelectorDecisionCache + broken bool + chainIdx int + active bool + updatingActive bool + + expectFrames []uint64 +} + +func NewFrameChain(decisions *SelectorDecisionCache, chainIdx int, logger logger.Logger) *FrameChain { + return &FrameChain{ + logger: logger, + decisions: decisions, + broken: true, + chainIdx: chainIdx, + active: false, + } +} + +func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate) bool { + if !fc.active { + return false + } + + // A decodable frame with frame_chain_fdiff equal to 0 indicates that the Chain is intact. + if fd.ChainDiffs[fc.chainIdx] == 0 { + if fc.broken { + fc.broken = false + fc.logger.Debugw("frame chain intact", "chanIdx", fc.chainIdx) + } + fc.expectFrames = fc.expectFrames[:0] + return true + } + + if fc.broken { + return false + } + + prevFrameInChain := extFrameNum - uint64(fd.ChainDiffs[fc.chainIdx]) + sd, err := fc.decisions.GetDecision(prevFrameInChain) + if err != nil { + fc.logger.Debugw("could not get decision", "err", err, "frame", extFrameNum, "prevFrame", prevFrameInChain) + } + + var intact bool + switch { + case sd == selectorDecisionForwarded: + intact = true + + case sd == selectorDecisionUnknown: + // If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival, + // set up a expected callback here to determine if the chain is broken or intact + if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) { + intact = true + fc.expectFrames = append(fc.expectFrames, prevFrameInChain) + } + } + + if !intact { + fc.broken = true + fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", sd, "frame", extFrameNum, "prevFrame", prevFrameInChain) + } + return intact +} + +func (fc *FrameChain) OnExpectFrameChanged(frameNum uint64, decision selectorDecision) { + if fc.broken { + return + } + + for i, f := range fc.expectFrames { + if f == frameNum { + if decision != selectorDecisionForwarded { + fc.broken = true + fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", decision, "frame", frameNum) + } + fc.expectFrames[i] = fc.expectFrames[len(fc.expectFrames)-1] + fc.expectFrames = fc.expectFrames[:len(fc.expectFrames)-1] + break + } + } +} + +func (fc *FrameChain) Broken() bool { + return fc.broken +} + +func (fc *FrameChain) BeginUpdateActive() { + fc.updatingActive = false +} + +func (fc *FrameChain) UpdateActive(active bool) { + fc.updatingActive = fc.updatingActive || active +} + +func (fc *FrameChain) EndUpdateActive() { + active := fc.updatingActive + fc.updatingActive = false + + if active == fc.active { + return + } + + // if the chain transit from inactive to active, reset broken to wait a decodable SWITCH frame + if !fc.active { + fc.broken = true + } + + fc.active = active +} diff --git a/pkg/sfu/videolayerselector/null.go b/pkg/sfu/videolayerselector/null.go new file mode 100644 index 000000000..644a8f957 --- /dev/null +++ b/pkg/sfu/videolayerselector/null.go @@ -0,0 +1,29 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "github.com/livekit/protocol/logger" +) + +type Null struct { + *Base +} + +func NewNull(logger logger.Logger) *Null { + return &Null{ + Base: NewBase(logger), + } +} diff --git a/pkg/sfu/videolayerselector/selectordecisioncache.go b/pkg/sfu/videolayerselector/selectordecisioncache.go new file mode 100644 index 000000000..ea60086b0 --- /dev/null +++ b/pkg/sfu/videolayerselector/selectordecisioncache.go @@ -0,0 +1,197 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "fmt" +) + +// ---------------------------------------------------------------------- + +type selectorDecision int + +const ( + selectorDecisionMissing selectorDecision = iota + selectorDecisionDropped + selectorDecisionForwarded + selectorDecisionUnknown +) + +func (s selectorDecision) String() string { + switch s { + case selectorDecisionMissing: + return "MISSING" + case selectorDecisionDropped: + return "DROPPED" + case selectorDecisionForwarded: + return "FORWARDED" + case selectorDecisionUnknown: + return "UNKNOWN" + default: + return fmt.Sprintf("%d", int(s)) + } +} + +// ---------------------------------------------------------------------- + +type SelectorDecisionCache struct { + initialized bool + base uint64 + last uint64 + masks []uint64 + numEntries uint64 + numNackEntries uint64 + + onExpectEntityChanged map[uint64][]func(entity uint64, decision selectorDecision) +} + +func NewSelectorDecisionCache(maxNumElements uint64, numNackEntries uint64) *SelectorDecisionCache { + numElements := (maxNumElements*2 + 63) / 64 + return &SelectorDecisionCache{ + masks: make([]uint64, numElements), + numEntries: numElements * 32, // 2 bits per entry + numNackEntries: numNackEntries, + onExpectEntityChanged: make(map[uint64][]func(entity uint64, decision selectorDecision)), + } +} + +func (s *SelectorDecisionCache) AddForwarded(entity uint64) { + s.addEntity(entity, selectorDecisionForwarded) +} + +func (s *SelectorDecisionCache) AddDropped(entity uint64) { + s.addEntity(entity, selectorDecisionDropped) +} + +func (s *SelectorDecisionCache) GetDecision(entity uint64) (selectorDecision, error) { + if !s.initialized || entity < s.base { + return selectorDecisionMissing, nil + } + + if entity > s.last { + return selectorDecisionUnknown, nil + } + + offset := s.last - entity + if offset >= s.numEntries { + // asking for something too old + return selectorDecisionMissing, fmt.Errorf("too old, oldest: %d, asking: %d", s.last-s.numEntries+1, entity) + } + + return s.getEntity(entity), nil +} + +func (s *SelectorDecisionCache) ExpectDecision(entity uint64, f func(entity uint64, decision selectorDecision)) bool { + if !s.initialized || entity < s.base { + return false + } + + if entity < s.last { + offset := s.last - entity + if offset >= s.numEntries { + return false // too old + } + } + + s.onExpectEntityChanged[entity] = append(s.onExpectEntityChanged[entity], f) + return true +} + +func (s *SelectorDecisionCache) addEntity(entity uint64, sd selectorDecision) { + if !s.initialized { + s.initialized = true + s.base = entity + s.last = entity + s.setEntity(entity, sd) + return + } + + if entity <= s.base { + // before base, too old + return + } + + if entity <= s.last { + s.setEntity(entity, sd) + return + } + + for e := s.last + 1; e != entity; e++ { + s.setEntity(e, selectorDecisionUnknown) + } + + // update [last+1-nack, entity-nack) to missing + missingStart := s.last + if missingStart > s.numNackEntries+s.base { + missingStart -= s.numNackEntries + } else { + missingStart = s.base + } + missingEnd := entity + if missingEnd > s.numNackEntries+s.base { + missingEnd -= s.numNackEntries + } else { + missingEnd = s.base + } + if missingEnd > missingStart { + for e := missingStart; e != missingEnd; e++ { + s.setEntityIfUnknown(e, selectorDecisionMissing) + } + } + + s.setEntity(entity, sd) + s.last = entity + + for e, fns := range s.onExpectEntityChanged { + if e+s.numEntries < s.last { + delete(s.onExpectEntityChanged, e) + for _, f := range fns { + f(e, selectorDecisionMissing) + } + } + } +} + +func (s *SelectorDecisionCache) setEntityIfUnknown(entity uint64, sd selectorDecision) { + if s.getEntity(entity) == selectorDecisionUnknown { + s.setEntity(entity, sd) + } +} + +func (s *SelectorDecisionCache) setEntity(entity uint64, sd selectorDecision) { + index, bitpos := s.getPos(entity) + s.masks[index] &= ^(0x3 << bitpos) // clear before bitwise OR + s.masks[index] |= (uint64(sd) & 0x3) << bitpos + + if sd != selectorDecisionUnknown { + if fns, ok := s.onExpectEntityChanged[entity]; ok { + delete(s.onExpectEntityChanged, entity) + for _, f := range fns { + f(entity, sd) + } + } + } +} + +func (s *SelectorDecisionCache) getEntity(entity uint64) selectorDecision { + index, bitpos := s.getPos(entity) + return selectorDecision((s.masks[index] >> bitpos) & 0x3) +} + +func (s *SelectorDecisionCache) getPos(entity uint64) (int, int) { + // 2 bits per entity, a uint64 mask can hold 32 entities + offset := (entity - s.base) % s.numEntries + return int(offset >> 5), int(offset&0x1F) * 2 +} diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go new file mode 100644 index 000000000..7d7e173a8 --- /dev/null +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -0,0 +1,140 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type Simulcast struct { + *Base +} + +func NewSimulcast(logger logger.Logger) *Simulcast { + return &Simulcast{ + Base: NewBase(logger), + } +} + +func NewSimulcastFromNull(vls VideoLayerSelector) *Simulcast { + return &Simulcast{ + Base: vls.(*Null).Base, + } +} + +func (s *Simulcast) IsOvershootOkay() bool { + return true +} + +func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoLayerSelectorResult) { + populateSwitches := func(isSwitching bool, isActive bool, reason string) { + if isSwitching { + result.IsSwitching = true + } + + if !isActive { + result.IsResuming = true + } + + if reason != "" { + s.logger.Infow( + reason, + "previous", s.previousLayer, + "current", s.currentLayer, + "previousParked", s.previousParkedLayer, + "parked", s.parkedLayer, + "previousTarget", s.previousTargetLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + } + } + + if s.currentLayer.Spatial != s.targetLayer.Spatial { + currentLayer := s.currentLayer + + // Three things to check when not locked to target + // 1. Resumable layer - don't need a key frame + // 2. Opportunistic layer upgrade - needs a key frame + // 3. Need to downgrade - needs a key frame + isSwitching := true + isActive := s.currentLayer.IsValid() + found := false + reason := "" + if s.parkedLayer.IsValid() { + if s.parkedLayer.Spatial == layer { + reason = "resuming at parked layer" + currentLayer = s.parkedLayer + isSwitching = false + found = true + } + } else { + if extPkt.KeyFrame { + if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial { + reason = "upgrading layer" + found = true + } + + if layer < s.currentLayer.Spatial && layer >= s.targetLayer.Spatial { + reason = "downgrading layer" + found = true + } + + if found { + currentLayer.Spatial = layer + currentLayer.Temporal = extPkt.VideoLayer.Temporal + } + } + } + + if found { + s.previousParkedLayer = s.parkedLayer + s.parkedLayer = buffer.InvalidLayer + + s.previousLayer = s.currentLayer + s.currentLayer = currentLayer + + s.previousTargetLayer = s.targetLayer + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = s.currentLayer.Spatial + } + + populateSwitches(isSwitching, isActive, reason) + } + } + + // if locked to higher than max layer due to overshoot, check if it can be dialed back + if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame { + s.previousLayer = s.currentLayer + s.currentLayer.Spatial = layer + + s.previousTargetLayer = s.targetLayer + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = layer + } + + populateSwitches(true, true, "adjusting overshoot") + } + + result.RTPMarker = extPkt.Packet.Marker + result.IsSelected = layer == s.currentLayer.Spatial + result.IsRelevant = false + return +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/null.go b/pkg/sfu/videolayerselector/temporallayerselector/null.go new file mode 100644 index 000000000..51f2fb591 --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/null.go @@ -0,0 +1,31 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package temporallayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +type Null struct{} + +func NewNull() *Null { + return &Null{} +} + +func Select(_extPkt *buffer.ExtPacket, current int32, _target int32) (this int32, next int32) { + this = current + next = current + return +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go new file mode 100644 index 000000000..1d691b401 --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go @@ -0,0 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package temporallayerselector + +import "github.com/livekit/livekit-server/pkg/sfu/buffer" + +type TemporalLayerSelector interface { + Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32) +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go new file mode 100644 index 000000000..2d2660897 --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -0,0 +1,56 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package temporallayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type VP8 struct { + logger logger.Logger +} + +func NewVP8(logger logger.Logger) *VP8 { + return &VP8{ + logger: logger, + } +} + +func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32) { + this = current + next = current + if current == target { + return + } + + vp8, ok := extPkt.Payload.(buffer.VP8) + if !ok || !vp8.T { + return + } + + tid := int32(vp8.TID) + if current < target { + if tid > current && tid <= target && vp8.S && vp8.Y { + this = tid + next = tid + } + } else { + if extPkt.Packet.Marker { + next = target + } + } + return +} diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go new file mode 100644 index 000000000..545196eae --- /dev/null +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -0,0 +1,63 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" +) + +type VideoLayerSelectorResult struct { + IsSelected bool + IsRelevant bool + IsSwitching bool + IsResuming bool + RTPMarker bool + DependencyDescriptorExtension []byte +} + +type VideoLayerSelector interface { + IsOvershootOkay() bool + + SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector) + + SetMax(maxLayer buffer.VideoLayer) + SetMaxSpatial(layer int32) + SetMaxTemporal(layer int32) + GetMax() buffer.VideoLayer + + SetTarget(targetLayer buffer.VideoLayer) + GetTarget() buffer.VideoLayer + + SetRequestSpatial(layer int32) + GetRequestSpatial() int32 + + CheckSync() (locked bool, layer int32) + + SetMaxSeen(maxSeenLayer buffer.VideoLayer) + SetMaxSeenSpatial(layer int32) + SetMaxSeenTemporal(layer int32) + GetMaxSeen() buffer.VideoLayer + + SetParked(parkedLayer buffer.VideoLayer) + GetParked() buffer.VideoLayer + + SetCurrent(currentLayer buffer.VideoLayer) + GetCurrent() buffer.VideoLayer + + Select(extPkt *buffer.ExtPacket, layer int32) VideoLayerSelectorResult + SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) + Rollback() +} diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go new file mode 100644 index 000000000..62d5d35d9 --- /dev/null +++ b/pkg/sfu/videolayerselector/vp9.go @@ -0,0 +1,109 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" + "github.com/pion/rtp/codecs" +) + +type VP9 struct { + *Base +} + +func NewVP9(logger logger.Logger) *VP9 { + return &VP9{ + Base: NewBase(logger), + } +} + +func NewVP9FromNull(vls VideoLayerSelector) *VP9 { + return &VP9{ + Base: vls.(*Null).Base, + } +} + +func (v *VP9) IsOvershootOkay() bool { + return false +} + +func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + vp9, ok := extPkt.Payload.(codecs.VP9Packet) + if !ok { + return + } + + currentLayer := v.currentLayer + if v.currentLayer != v.targetLayer { + updatedLayer := v.currentLayer + + if !v.currentLayer.IsValid() { + if !extPkt.KeyFrame { + return + } + + updatedLayer = extPkt.VideoLayer + } else { + if v.currentLayer.Temporal != v.targetLayer.Temporal { + if v.currentLayer.Temporal < v.targetLayer.Temporal { + // temporal scale up + if extPkt.VideoLayer.Temporal > v.currentLayer.Temporal && extPkt.VideoLayer.Temporal <= v.targetLayer.Temporal && vp9.U && vp9.B { + currentLayer.Temporal = extPkt.VideoLayer.Temporal + updatedLayer.Temporal = extPkt.VideoLayer.Temporal + } + } else { + // temporal scale down + if vp9.E { + updatedLayer.Temporal = v.targetLayer.Temporal + } + } + } + + if v.currentLayer.Spatial != v.targetLayer.Spatial { + if v.currentLayer.Spatial < v.targetLayer.Spatial { + // spatial scale up + if extPkt.VideoLayer.Spatial > v.currentLayer.Spatial && extPkt.VideoLayer.Spatial <= v.targetLayer.Spatial && !vp9.P && vp9.B { + currentLayer.Spatial = extPkt.VideoLayer.Spatial + updatedLayer.Spatial = extPkt.VideoLayer.Spatial + } + } else { + // spatial scale down + if vp9.E { + updatedLayer.Spatial = v.targetLayer.Spatial + } + } + } + } + + if updatedLayer != v.currentLayer { + result.IsSwitching = true + if !v.currentLayer.IsValid() && updatedLayer.IsValid() { + result.IsResuming = true + } + + v.previousLayer = v.currentLayer + v.currentLayer = updatedLayer + } + } + + result.RTPMarker = extPkt.Packet.Marker + if vp9.E && extPkt.VideoLayer.Spatial == currentLayer.Spatial && (vp9.P || v.targetLayer.Spatial <= v.currentLayer.Spatial) { + result.RTPMarker = true + } + result.IsSelected = !extPkt.VideoLayer.GreaterThan(currentLayer) + result.IsRelevant = true + return +} diff --git a/pkg/sfu/vp8munger_test.go b/pkg/sfu/vp8munger_test.go deleted file mode 100644 index 20fbcd922..000000000 --- a/pkg/sfu/vp8munger_test.go +++ /dev/null @@ -1,509 +0,0 @@ -package sfu - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/testutils" -) - -func compare(expected *VP8Munger, actual *VP8Munger) bool { - return reflect.DeepEqual(expected.pictureIdWrapHandler, actual.pictureIdWrapHandler) && - expected.extLastPictureId == actual.extLastPictureId && - expected.pictureIdOffset == actual.pictureIdOffset && - expected.pictureIdUsed == actual.pictureIdUsed && - expected.lastTl0PicIdx == actual.lastTl0PicIdx && - expected.tl0PicIdxOffset == actual.tl0PicIdxOffset && - expected.tl0PicIdxUsed == actual.tl0PicIdxUsed && - expected.tidUsed == actual.tidUsed && - expected.lastKeyIdx == actual.lastKeyIdx && - expected.keyIdxOffset == actual.keyIdxOffset && - expected.keyIdxUsed == actual.keyIdxUsed -} - -func newVP8Munger() *VP8Munger { - return NewVP8Munger(logger.GetLogger()) -} - -func TestSetLast(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, err := testutils.GetTestExtPacketVP8(params, vp8) - require.NoError(t, err) - require.NotNil(t, extPkt) - - expectedVP8Munger := VP8Munger{ - VP8MungerParams: VP8MungerParams{ - pictureIdWrapHandler: VP8PictureIdWrapHandler{ - maxPictureId: 13466, - maxMBit: true, - totalWrap: 0, - lastWrap: 0, - }, - extLastPictureId: 13467, - pictureIdOffset: 0, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: 0, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: 0, - keyIdxUsed: 1, - lastDroppedPictureId: -1, - }, - } - - v.SetLast(extPkt) - require.True(t, compare(&expectedVP8Munger, v)) -} - -func TestUpdateOffsets(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - v.SetLast(extPkt) - - params = &testutils.TestExtPacketParams{ - SequenceNumber: 56789, - Timestamp: 0xabcdef, - SSRC: 0x87654321, - } - vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 345, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 12, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 4, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - v.UpdateOffsets(extPkt) - - expectedVP8Munger := VP8Munger{ - VP8MungerParams: VP8MungerParams{ - pictureIdWrapHandler: VP8PictureIdWrapHandler{ - maxPictureId: 344, - maxMBit: true, - totalWrap: 0, - lastWrap: 0, - }, - extLastPictureId: 13467, - pictureIdOffset: 345 - 13467 - 1, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: (12 - 233 - 1) & 0xff, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: (4 - 23 - 1) & 0x1f, - keyIdxUsed: 1, - lastDroppedPictureId: -1, - }, - } - require.True(t, compare(&expectedVP8Munger, v)) -} - -func TestOutOfOrderPictureId(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - v.SetLast(extPkt) - v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 2) - - // out-of-order sequence number not in the missing picture id cache - vp8.PictureID = 13466 - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingOutOfOrder, 2) - require.Error(t, err) - require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) - require.Nil(t, tp) - - // create a hole in picture id - vp8.PictureID = 13469 - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - - tpExpected := TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingGap, 2) - require.NoError(t, err) - require.NotNil(t, tp) - require.Equal(t, tpExpected, *tp) - - // all three, the last, the current and the in-between should have been added to missing picture id cache - value, ok := v.PictureIdOffset(13467) - require.True(t, ok) - require.EqualValues(t, 0, value) - - value, ok = v.PictureIdOffset(13468) - require.True(t, ok) - require.EqualValues(t, 0, value) - - value, ok = v.PictureIdOffset(13469) - require.True(t, ok) - require.EqualValues(t, 0, value) - - // out-of-order sequence number should be in the missing picture id cache - vp8.PictureID = 13468 - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - - tpExpected = TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingOutOfOrder, 2) - require.NoError(t, err) - require.NotNil(t, tp) - require.Equal(t, tpExpected, *tp) -} - -func TestTemporalLayerFiltering(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - v.SetLast(extPkt) - - // translate - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) - require.Error(t, err) - require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) - require.EqualValues(t, 1, v.pictureIdOffset) - - // another packet with the same picture id. - // It should be dropped, but offset should not be updated. - params.SequenceNumber = 23334 - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) - require.Error(t, err) - require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) - require.EqualValues(t, 1, v.pictureIdOffset) - - // another packet with the same picture id, but a gap in sequence number. - // It should be dropped, but offset should not be updated. - params.SequenceNumber = 23337 - extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) - require.Error(t, err) - require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) - require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) - require.EqualValues(t, 1, v.pictureIdOffset) -} - -func TestGapInSequenceNumberSamePicture(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 65533, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - PayloadSize: 33, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - v.SetLast(extPkt) - - tpExpected := TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - } - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 2) - require.NoError(t, err) - require.Equal(t, tpExpected, *tp) - - // telling there is a gap in sequence number will add pictures to missing picture cache - tpExpected = TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingGap, 2) - require.NoError(t, err) - require.Equal(t, tpExpected, *tp) - - value, ok := v.PictureIdOffset(13467) - require.True(t, ok) - require.EqualValues(t, 0, value) -} - -func TestUpdateAndGetPadding(t *testing.T) { - v := newVP8Munger() - - params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - PayloadSize: 20, - } - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - - v.SetLast(extPkt) - - // getting padding with repeat of last picture - blankVP8 := v.UpdateAndGetPadding(false) - expectedVP8 := buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - require.Equal(t, expectedVP8, *blankVP8) - - // getting padding with new picture - blankVP8 = v.UpdateAndGetPadding(true) - expectedVP8 = buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, - } - require.Equal(t, expectedVP8, *blankVP8) -} - -func TestVP8PictureIdWrapHandler(t *testing.T) { - v := &VP8PictureIdWrapHandler{} - - v.Init(109, false) - require.Equal(t, int32(109), v.MaxPictureId()) - require.False(t, v.maxMBit) - - v.UpdateMaxPictureId(109350, true) - require.Equal(t, int32(109350), v.MaxPictureId()) - require.True(t, v.maxMBit) - - // start with something close to the 15-bit wrap around point - v.Init(32766, true) - - // out-of-order, do not wrap - extPictureId := v.Unwrap(32750, true) - require.Equal(t, int32(32750), extPictureId) - require.Equal(t, int32(0), v.totalWrap) - require.Equal(t, int32(0), v.lastWrap) - - // wrap at 15-bits - extPictureId = v.Unwrap(5, false) - require.Equal(t, int32(32773), extPictureId) // 15-bit wrap at 32768 + 5 = 32773 - require.Equal(t, int32(32768), v.totalWrap) - require.Equal(t, int32(32768), v.lastWrap) - - // set things near 7-bit wrap point - v.UpdateMaxPictureId(32893, false) // 32768 + 125 - - // wrap at 7-bits - extPictureId = v.Unwrap(5, true) - require.Equal(t, int32(32901), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 5 = 32901 - require.Equal(t, int32(32896), v.totalWrap) // one 15-bit wrap + one 7-bit wrap - require.Equal(t, int32(128), v.lastWrap) - - // a new picture in 7-bit mode much with a gap in between. - // A big enough gap which would have been treated as out-of-order in 7-bit mode. - v.UpdateMaxPictureId(32901, false) - extPictureId = v.Unwrap(73, false) - require.Equal(t, int32(32841), extPictureId) // 15-bit wrap at 32768 + 73 = 32841 - - // a new picture in 15-bit mode much with a gap in between. - // A big enough gap which would have been treated as out-of-order in 7-bit mode. - v.UpdateMaxPictureId(32901, true) - v.lastWrap = int32(32768) - extPictureId = v.Unwrap(73, false) - require.Equal(t, int32(32969), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 73 = 32969 -} diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index 0373c33b6..8611337f1 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1c78a5486..35eaf668e 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( @@ -21,11 +35,9 @@ func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.Webho event.CreatedAt = time.Now().Unix() event.Id = utils.NewGuid("EV_") - t.webhookPool.Submit(func() { - if err := t.notifier.Notify(ctx, event); err != nil { - logger.Warnw("failed to notify webhook", err, "event", event.Event) - } - }) + if err := t.notifier.QueueNotify(ctx, event); err != nil { + logger.Warnw("failed to notify webhook", err, "event", event.Event) + } } func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) { @@ -93,14 +105,17 @@ func (t *telemetryService) ParticipantActive( room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, + isMigration bool, ) { t.enqueue(func() { - // consider participant joined only when they became active - t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventParticipantJoined, - Room: room, - Participant: participant, - }) + if !isMigration { + // consider participant joined only when they became active + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventParticipantJoined, + Room: room, + Participant: participant, + }) + } worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)) if !ok { @@ -411,6 +426,16 @@ func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.Egre }) } +func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventEgressUpdated, + EgressInfo: info, + }) + t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info)) + }) +} + func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) { t.enqueue(func() { t.NotifyEvent(ctx, &livekit.WebhookEvent{ @@ -422,6 +447,46 @@ func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.Egress }) } +func (t *telemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_CREATED, info)) + }) +} + +func (t *telemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_DELETED, info)) + }) +} + +func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventIngressStarted, + IngressInfo: info, + }) + + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_STARTED, info)) + }) +} + +func (t *telemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_UPDATED, info)) + }) +} + +func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventIngressEnded, + IngressInfo: info, + }) + + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_ENDED, info)) + }) +} + // returns a livekit.Room with only name and sid filled out // returns nil if room is not found func (t *telemetryService) getRoomDetails(participantID livekit.ParticipantID) *livekit.Room { @@ -476,3 +541,12 @@ func newEgressEvent(event livekit.AnalyticsEventType, egress *livekit.EgressInfo Egress: egress, } } + +func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressInfo) *livekit.AnalyticsEvent { + return &livekit.AnalyticsEvent{ + Type: event, + Timestamp: timestamppb.Now(), + IngressId: ingress.IngressId, + Ingress: ingress, + } +} diff --git a/pkg/telemetry/events_test.go b/pkg/telemetry/events_test.go index 852701180..e83b61ba9 100644 --- a/pkg/telemetry/events_test.go +++ b/pkg/telemetry/events_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry_test import ( @@ -69,7 +83,7 @@ func Test_OnParticipantLeft_EventIsSent(t *testing.T) { participantInfo := &livekit.ParticipantInfo{Sid: partSID} // do - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false) fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true) time.Sleep(time.Millisecond * 500) @@ -159,7 +173,7 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) { ClientConnectTime: 420, } - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false) time.Sleep(time.Millisecond * 500) require.Equal(t, 2, fixture.analytics.SendEventCallCount()) diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 9393052c1..dad09977e 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -1,9 +1,22 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( "time" - "github.com/mackerelio/go-osstat/loadavg" "github.com/mackerelio/go-osstat/memory" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" @@ -95,10 +108,12 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { initPacketStats(nodeID, nodeType, env) initRoomStats(nodeID, nodeType, env) + initPSRPCStats(nodeID, nodeType, env) + initQualityStats(nodeID, nodeType, env) } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { - loadAvg, err := loadavg.Get() + loadAvg, err := getLoadAvg() if err != nil { return nil, false, err } diff --git a/pkg/telemetry/prometheus/node_linux.go b/pkg/telemetry/prometheus/node_linux.go index b2e06c3d9..baa6919f8 100644 --- a/pkg/telemetry/prometheus/node_linux.go +++ b/pkg/telemetry/prometheus/node_linux.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build linux // +build linux @@ -5,37 +19,10 @@ package prometheus import ( "fmt" - "sync" "github.com/florianl/go-tc" - "github.com/mackerelio/go-osstat/cpu" ) -var ( - cpuStatsLock sync.RWMutex - lastCPUTotal, lastCPUIdle uint64 -) - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - cpuInfo, err := cpu.Get() - if err != nil { - return - } - - cpuStatsLock.Lock() - if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { - cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) - } - - lastCPUTotal = cpuInfo.Total - lastCPUIdle = cpuInfo.Idle // + cpu.Iowait - cpuStatsLock.Unlock() - - numCPUs = uint32(cpuInfo.CPUCount) - - return -} - func getTCStats() (packets, drops uint32, err error) { rtnl, err := tc.Open(&tc.Config{}) if err != nil { diff --git a/pkg/telemetry/prometheus/node_nonlinux.go b/pkg/telemetry/prometheus/node_nonlinux.go index 581456413..22fc11c7b 100644 --- a/pkg/telemetry/prometheus/node_nonlinux.go +++ b/pkg/telemetry/prometheus/node_nonlinux.go @@ -1,39 +1,21 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !linux package prometheus -import ( - "runtime" - "sync" - - "github.com/mackerelio/go-osstat/cpu" -) - -var ( - cpuStatsLock sync.RWMutex - lastCPUTotal, lastCPUIdle uint64 -) - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - cpuInfo, err := cpu.Get() - if err != nil { - return - } - - cpuStatsLock.Lock() - if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { - cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) - } - - lastCPUTotal = cpuInfo.Total - lastCPUIdle = cpuInfo.Idle - cpuStatsLock.Unlock() - - numCPUs = uint32(runtime.NumCPU()) - - return -} - func getTCStats() (packets, drops uint32, err error) { // linux only return diff --git a/pkg/telemetry/prometheus/node_nonwindows.go b/pkg/telemetry/prometheus/node_nonwindows.go new file mode 100644 index 000000000..b765ce271 --- /dev/null +++ b/pkg/telemetry/prometheus/node_nonwindows.go @@ -0,0 +1,56 @@ +//go:build !windows + +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import ( + "runtime" + "sync" + + "github.com/mackerelio/go-osstat/cpu" + "github.com/mackerelio/go-osstat/loadavg" +) + +var ( + cpuStatsLock sync.RWMutex + lastCPUTotal, lastCPUIdle uint64 +) + +func getLoadAvg() (*loadavg.Stats, error) { + return loadavg.Get() +} + +func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { + cpuInfo, err := cpu.Get() + if err != nil { + return + } + + cpuStatsLock.Lock() + if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { + cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) + } + + lastCPUTotal = cpuInfo.Total + lastCPUIdle = cpuInfo.Idle + cpuStatsLock.Unlock() + + numCPUs = uint32(runtime.NumCPU()) + + return +} diff --git a/pkg/telemetry/prometheus/node_windows.go b/pkg/telemetry/prometheus/node_windows.go new file mode 100644 index 000000000..eb0ba48a9 --- /dev/null +++ b/pkg/telemetry/prometheus/node_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import "github.com/mackerelio/go-osstat/loadavg" + +func getLoadAvg() (*loadavg.Stats, error) { + return &loadavg.Stats{}, nil +} + +func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { + return 1, 1, nil +} diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 09525053c..3048367a8 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/prometheus/psrpc.go b/pkg/telemetry/prometheus/psrpc.go new file mode 100644 index 000000000..f07c90034 --- /dev/null +++ b/pkg/telemetry/prometheus/psrpc.go @@ -0,0 +1,125 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" + "github.com/livekit/psrpc/pkg/middleware" +) + +var ( + psrpcRequestTime *prometheus.HistogramVec + psrpcStreamSendTime *prometheus.HistogramVec + psrpcStreamReceiveTotal *prometheus.CounterVec + psrpcStreamCurrent *prometheus.GaugeVec + psrpcErrorTotal *prometheus.CounterVec +) + +func initPSRPCStats(nodeID string, nodeType livekit.NodeType, env string) { + labels := []string{"role", "kind", "service", "method"} + streamLabels := []string{"role", "service", "method"} + + psrpcRequestTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "request_time_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, + }, labels) + psrpcStreamSendTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_send_time_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, + }, streamLabels) + psrpcStreamReceiveTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_receive_total", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, streamLabels) + psrpcStreamCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_count", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, streamLabels) + psrpcErrorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "error_total", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, labels) + + prometheus.MustRegister(psrpcRequestTime) + prometheus.MustRegister(psrpcStreamSendTime) + prometheus.MustRegister(psrpcStreamReceiveTotal) + prometheus.MustRegister(psrpcStreamCurrent) + prometheus.MustRegister(psrpcErrorTotal) +} + +var _ middleware.MetricsObserver = PSRPCMetricsObserver{} + +type PSRPCMetricsObserver struct{} + +func (o PSRPCMetricsObserver) OnUnaryRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Inc() + } else if role == middleware.ClientRole { + psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } else { + psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnMultiRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, responseCount int, errorCount int) { + if responseCount == 0 { + psrpcErrorTotal.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Inc() + } else if role == middleware.ClientRole { + psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } else { + psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnStreamSend(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() + } else { + psrpcStreamSendTime.WithLabelValues(role.String(), info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnStreamRecv(role middleware.MetricRole, info psrpc.RPCInfo, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() + } else { + psrpcStreamReceiveTotal.WithLabelValues(role.String(), info.Service, info.Method).Inc() + } +} + +func (o PSRPCMetricsObserver) OnStreamOpen(role middleware.MetricRole, info psrpc.RPCInfo) { + psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Inc() +} + +func (o PSRPCMetricsObserver) OnStreamClose(role middleware.MetricRole, info psrpc.RPCInfo) { + psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Dec() +} diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go new file mode 100644 index 000000000..481b7558f --- /dev/null +++ b/pkg/telemetry/prometheus/quality.go @@ -0,0 +1,61 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/livekit/protocol/livekit" +) + +var ( + qualityRating prometheus.Histogram + qualityScore prometheus.Histogram + qualityDrop *prometheus.CounterVec +) + +func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) { + qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "rating", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{0, 1, 2}, + }) + qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "score", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5}, + }) + qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "drop", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, []string{"direction"}) + + prometheus.MustRegister(qualityRating) + prometheus.MustRegister(qualityScore) + prometheus.MustRegister(qualityDrop) +} + +func RecordQuality(rating livekit.ConnectionQuality, score float32, numUpDrops int, numDownDrops int) { + qualityRating.Observe(float64(rating)) + qualityScore.Observe(float64(score)) + qualityDrop.WithLabelValues("up").Add(float64(numUpDrops)) + qualityDrop.WithLabelValues("down").Add(float64(numDownDrops)) +} diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index d50ce2d15..ef320ad73 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index b218582cd..840f126c3 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( @@ -53,31 +67,31 @@ func (s *BytesTrackStats) Report() { s.report(true) } -func (p *BytesTrackStats) report(force bool) { +func (s *BytesTrackStats) report(force bool) { now := time.Now() if !force { - lr := p.lastStatsReport.Load().(*time.Time) + lr := s.lastStatsReport.Load().(*time.Time) if time.Since(*lr) < statsReportInterval { return } - if !p.lastStatsReport.CompareAndSwap(lr, &now) { + if !s.lastStatsReport.CompareAndSwap(lr, &now) { return } } else { - p.lastStatsReport.Store(&now) + s.lastStatsReport.Store(&now) } - if recv := p.recv.Swap(0); recv > 0 { - p.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, p.pID, p.trackID), &livekit.AnalyticsStat{ + if recv := s.recv.Swap(0); recv > 0 { + s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ {PrimaryBytes: recv}, }, }) } - if send := p.send.Swap(0); send > 0 { - p.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, p.pID, p.trackID), &livekit.AnalyticsStat{ + if send := s.send.Swap(0); send > 0 { + s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ {PrimaryBytes: send}, }, diff --git a/pkg/telemetry/stats.go b/pkg/telemetry/stats.go index 81e9c2556..79d6717cb 100644 --- a/pkg/telemetry/stats.go +++ b/pkg/telemetry/stats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 5dd8b82c3..1758c8b45 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry_test import ( diff --git a/pkg/telemetry/statsconn.go b/pkg/telemetry/statsconn.go index 91f60af94..10ac45fb5 100644 --- a/pkg/telemetry/statsconn.go +++ b/pkg/telemetry/statsconn.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index 9447703b6..b86175368 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( @@ -8,6 +22,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -143,7 +158,9 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } // find aggregates across streams - score := float32(0.0) + scoreSum := float32(0.0) // used for average + minScore := float32(0.0) // min score in batched stats + var scores []float32 // used for median maxRtt := uint32(0) maxJitter := uint32(0) coalescedVideoLayers := make(map[int32]*livekit.AnalyticsVideoLayer) @@ -154,7 +171,17 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { continue } - score += stat.Score + // only consider non-zero scores + if stat.Score > 0 { + if minScore == 0 { + minScore = stat.Score + } else if stat.Score < minScore { + minScore = stat.Score + } + scoreSum += stat.Score + scores = append(scores, stat.Score) + } + for _, analyticsStream := range stat.Streams { if analyticsStream.Rtt > maxRtt { maxRtt = analyticsStream.Rtt @@ -201,10 +228,16 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } } - return &livekit.AnalyticsStat{ - Score: score / float32(len(stats)), - Streams: []*livekit.AnalyticsStream{coalescedStream}, + stat := &livekit.AnalyticsStat{ + MinScore: minScore, + MedianScore: utils.MedianFloat32(scores), + Streams: []*livekit.AnalyticsStream{coalescedStream}, } + numScores := len(scores) + if numScores > 0 { + stat.Score = scoreSum / float32(numScores) + } + return stat } func isValid(stat *livekit.AnalyticsStat) bool { diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 4e6cff95e..fd3ae6ff5 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -22,23 +22,60 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.EgressInfo } + EgressUpdatedStub func(context.Context, *livekit.EgressInfo) + egressUpdatedMutex sync.RWMutex + egressUpdatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.EgressInfo + } FlushStatsStub func() flushStatsMutex sync.RWMutex flushStatsArgsForCall []struct { } + IngressCreatedStub func(context.Context, *livekit.IngressInfo) + ingressCreatedMutex sync.RWMutex + ingressCreatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressDeletedStub func(context.Context, *livekit.IngressInfo) + ingressDeletedMutex sync.RWMutex + ingressDeletedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressEndedStub func(context.Context, *livekit.IngressInfo) + ingressEndedMutex sync.RWMutex + ingressEndedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressStartedStub func(context.Context, *livekit.IngressInfo) + ingressStartedMutex sync.RWMutex + ingressStartedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressUpdatedStub func(context.Context, *livekit.IngressInfo) + ingressUpdatedMutex sync.RWMutex + ingressUpdatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } NotifyEventStub func(context.Context, *livekit.WebhookEvent) notifyEventMutex sync.RWMutex notifyEventArgsForCall []struct { arg1 context.Context arg2 *livekit.WebhookEvent } - ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta) + ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) participantActiveMutex sync.RWMutex participantActiveArgsForCall []struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta + arg5 bool } ParticipantJoinedStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool) participantJoinedMutex sync.RWMutex @@ -274,6 +311,39 @@ func (fake *FakeTelemetryService) EgressStartedArgsForCall(i int) (context.Conte return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) EgressUpdated(arg1 context.Context, arg2 *livekit.EgressInfo) { + fake.egressUpdatedMutex.Lock() + fake.egressUpdatedArgsForCall = append(fake.egressUpdatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.EgressInfo + }{arg1, arg2}) + stub := fake.EgressUpdatedStub + fake.recordInvocation("EgressUpdated", []interface{}{arg1, arg2}) + fake.egressUpdatedMutex.Unlock() + if stub != nil { + fake.EgressUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) EgressUpdatedCallCount() int { + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() + return len(fake.egressUpdatedArgsForCall) +} + +func (fake *FakeTelemetryService) EgressUpdatedCalls(stub func(context.Context, *livekit.EgressInfo)) { + fake.egressUpdatedMutex.Lock() + defer fake.egressUpdatedMutex.Unlock() + fake.EgressUpdatedStub = stub +} + +func (fake *FakeTelemetryService) EgressUpdatedArgsForCall(i int) (context.Context, *livekit.EgressInfo) { + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() + argsForCall := fake.egressUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) FlushStats() { fake.flushStatsMutex.Lock() fake.flushStatsArgsForCall = append(fake.flushStatsArgsForCall, struct { @@ -298,6 +368,171 @@ func (fake *FakeTelemetryService) FlushStatsCalls(stub func()) { fake.FlushStatsStub = stub } +func (fake *FakeTelemetryService) IngressCreated(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressCreatedMutex.Lock() + fake.ingressCreatedArgsForCall = append(fake.ingressCreatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressCreatedStub + fake.recordInvocation("IngressCreated", []interface{}{arg1, arg2}) + fake.ingressCreatedMutex.Unlock() + if stub != nil { + fake.IngressCreatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressCreatedCallCount() int { + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + return len(fake.ingressCreatedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressCreatedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressCreatedMutex.Lock() + defer fake.ingressCreatedMutex.Unlock() + fake.IngressCreatedStub = stub +} + +func (fake *FakeTelemetryService) IngressCreatedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + argsForCall := fake.ingressCreatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressDeleted(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressDeletedMutex.Lock() + fake.ingressDeletedArgsForCall = append(fake.ingressDeletedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressDeletedStub + fake.recordInvocation("IngressDeleted", []interface{}{arg1, arg2}) + fake.ingressDeletedMutex.Unlock() + if stub != nil { + fake.IngressDeletedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressDeletedCallCount() int { + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + return len(fake.ingressDeletedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressDeletedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressDeletedMutex.Lock() + defer fake.ingressDeletedMutex.Unlock() + fake.IngressDeletedStub = stub +} + +func (fake *FakeTelemetryService) IngressDeletedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + argsForCall := fake.ingressDeletedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressEnded(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressEndedMutex.Lock() + fake.ingressEndedArgsForCall = append(fake.ingressEndedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressEndedStub + fake.recordInvocation("IngressEnded", []interface{}{arg1, arg2}) + fake.ingressEndedMutex.Unlock() + if stub != nil { + fake.IngressEndedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressEndedCallCount() int { + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + return len(fake.ingressEndedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressEndedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressEndedMutex.Lock() + defer fake.ingressEndedMutex.Unlock() + fake.IngressEndedStub = stub +} + +func (fake *FakeTelemetryService) IngressEndedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + argsForCall := fake.ingressEndedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressStarted(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressStartedMutex.Lock() + fake.ingressStartedArgsForCall = append(fake.ingressStartedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressStartedStub + fake.recordInvocation("IngressStarted", []interface{}{arg1, arg2}) + fake.ingressStartedMutex.Unlock() + if stub != nil { + fake.IngressStartedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressStartedCallCount() int { + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() + return len(fake.ingressStartedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressStartedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressStartedMutex.Lock() + defer fake.ingressStartedMutex.Unlock() + fake.IngressStartedStub = stub +} + +func (fake *FakeTelemetryService) IngressStartedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() + argsForCall := fake.ingressStartedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressUpdated(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressUpdatedMutex.Lock() + fake.ingressUpdatedArgsForCall = append(fake.ingressUpdatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressUpdatedStub + fake.recordInvocation("IngressUpdated", []interface{}{arg1, arg2}) + fake.ingressUpdatedMutex.Unlock() + if stub != nil { + fake.IngressUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressUpdatedCallCount() int { + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() + return len(fake.ingressUpdatedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressUpdatedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressUpdatedMutex.Lock() + defer fake.ingressUpdatedMutex.Unlock() + fake.IngressUpdatedStub = stub +} + +func (fake *FakeTelemetryService) IngressUpdatedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() + argsForCall := fake.ingressUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) NotifyEvent(arg1 context.Context, arg2 *livekit.WebhookEvent) { fake.notifyEventMutex.Lock() fake.notifyEventArgsForCall = append(fake.notifyEventArgsForCall, struct { @@ -331,19 +566,20 @@ func (fake *FakeTelemetryService) NotifyEventArgsForCall(i int) (context.Context return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta) { +func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta, arg5 bool) { fake.participantActiveMutex.Lock() fake.participantActiveArgsForCall = append(fake.participantActiveArgsForCall, struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta - }{arg1, arg2, arg3, arg4}) + arg5 bool + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.ParticipantActiveStub - fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.participantActiveMutex.Unlock() if stub != nil { - fake.ParticipantActiveStub(arg1, arg2, arg3, arg4) + fake.ParticipantActiveStub(arg1, arg2, arg3, arg4, arg5) } } @@ -353,17 +589,17 @@ func (fake *FakeTelemetryService) ParticipantActiveCallCount() int { return len(fake.participantActiveArgsForCall) } -func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta)) { +func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool)) { fake.participantActiveMutex.Lock() defer fake.participantActiveMutex.Unlock() fake.ParticipantActiveStub = stub } -func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta) { +func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) { fake.participantActiveMutex.RLock() defer fake.participantActiveMutex.RUnlock() argsForCall := fake.participantActiveArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } func (fake *FakeTelemetryService) ParticipantJoined(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.ClientInfo, arg5 *livekit.AnalyticsClientMeta, arg6 bool) { @@ -1109,8 +1345,20 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.egressEndedMutex.RUnlock() fake.egressStartedMutex.RLock() defer fake.egressStartedMutex.RUnlock() + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() fake.flushStatsMutex.RLock() defer fake.flushStatsMutex.RUnlock() + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() fake.notifyEventMutex.RLock() defer fake.notifyEventMutex.RUnlock() fake.participantActiveMutex.RLock() diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index bcef252ae..c74619eda 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( @@ -5,8 +19,6 @@ import ( "sync" "time" - "github.com/gammazero/workerpool" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -24,8 +36,8 @@ type TelemetryService interface { // ParticipantJoined - a participant establishes signal connection to a room ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool) // ParticipantActive - a participant establishes media connection - ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta) - // ParticipantResumed - there has been an ICE restart or connection resume attempt + ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool) + // ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) // ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool) @@ -54,7 +66,13 @@ type TelemetryService interface { TrackPublishRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType string, layer int, stats *livekit.RTPStats) TrackSubscribeRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType string, stats *livekit.RTPStats) EgressStarted(ctx context.Context, info *livekit.EgressInfo) + EgressUpdated(ctx context.Context, info *livekit.EgressInfo) EgressEnded(ctx context.Context, info *livekit.EgressInfo) + IngressCreated(ctx context.Context, info *livekit.IngressInfo) + IngressDeleted(ctx context.Context, info *livekit.IngressInfo) + IngressStarted(ctx context.Context, info *livekit.IngressInfo) + IngressUpdated(ctx context.Context, info *livekit.IngressInfo) + IngressEnded(ctx context.Context, info *livekit.IngressInfo) // helpers AnalyticsService @@ -63,7 +81,6 @@ type TelemetryService interface { } const ( - maxWebhookWorkers = 50 workerCleanupWait = 3 * time.Minute jobQueueBufferSize = 10000 ) @@ -71,22 +88,20 @@ const ( type telemetryService struct { AnalyticsService - notifier webhook.Notifier - webhookPool *workerpool.WorkerPool - jobsChan chan func() + notifier webhook.QueuedNotifier + jobsChan chan func() lock sync.RWMutex workers map[livekit.ParticipantID]*StatsWorker } -func NewTelemetryService(notifier webhook.Notifier, analytics AnalyticsService) TelemetryService { +func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService { t := &telemetryService{ AnalyticsService: analytics, - notifier: notifier, - webhookPool: workerpool.New(maxWebhookWorkers), - jobsChan: make(chan func(), jobQueueBufferSize), - workers: make(map[livekit.ParticipantID]*StatsWorker), + notifier: notifier, + jobsChan: make(chan func(), jobQueueBufferSize), + workers: make(map[livekit.ParticipantID]*StatsWorker), } go t.run() diff --git a/pkg/testutils/timeout.go b/pkg/testutils/timeout.go index ab6414711..11debef92 100644 --- a/pkg/testutils/timeout.go +++ b/pkg/testutils/timeout.go @@ -1,11 +1,23 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( "context" "testing" "time" - - "github.com/stretchr/testify/require" ) var ( @@ -19,7 +31,9 @@ func WithTimeout(t *testing.T, f func() string) { for { select { case <-ctx.Done(): - require.Empty(t, lastErr) + if lastErr != "" { + t.Fatalf("did not reach expected state after %v: %s", ConnectTimeout, lastErr) + } case <-time.After(10 * time.Millisecond): lastErr = f() if lastErr == "" { diff --git a/pkg/utils/math.go b/pkg/utils/math.go new file mode 100644 index 000000000..9e75c1ad8 --- /dev/null +++ b/pkg/utils/math.go @@ -0,0 +1,36 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import "sort" + +// MedianFloat32 gets median value for an array of float32 +func MedianFloat32(input []float32) float32 { + num := len(input) + if num == 0 { + return 0 + } else if num == 1 { + return input[0] + } + sort.Slice(input, func(i, j int) bool { + return input[i] < input[j] + }) + if num%2 != 0 { + return input[num/2] + } + left := input[num/2-1] + right := input[num/2] + return (left + right) / 2 +} diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 473430a3a..3e992461c 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/test/client/client.go b/test/client/client.go index b886d90f8..70b135a6d 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package client import ( @@ -8,6 +22,7 @@ import ( "net/http" "net/url" "path/filepath" + "strings" "sync" "time" @@ -19,6 +34,7 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -26,6 +42,11 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" ) +type SignalRequestHandler func(msg *livekit.SignalRequest) error +type SignalRequestInterceptor func(msg *livekit.SignalRequest, next SignalRequestHandler) error +type SignalResponseHandler func(msg *livekit.SignalResponse) error +type SignalResponseInterceptor func(msg *livekit.SignalResponse, next SignalResponseHandler) error + type RTCClient struct { id livekit.ParticipantID conn *websocket.Conn @@ -43,10 +64,14 @@ type RTCClient struct { localParticipant *livekit.ParticipantInfo remoteParticipants map[livekit.ParticipantID]*livekit.ParticipantInfo + signalRequestInterceptor SignalRequestInterceptor + signalResponseInterceptor SignalResponseInterceptor + subscriberAsPrimary atomic.Bool publisherFullyEstablished atomic.Bool subscriberFullyEstablished atomic.Bool pongReceivedAt atomic.Int64 + lastAnswer atomic.Pointer[webrtc.SessionDescription] // tracks waiting to be acked, cid => trackInfo pendingPublishedTracks map[string]*livekit.TrackInfo @@ -59,6 +84,8 @@ type RTCClient struct { // map of livekit.ParticipantID and last packet lastPackets map[livekit.ParticipantID]*rtp.Packet bytesReceived map[livekit.ParticipantID]uint64 + + subscriptionResponse atomic.Pointer[livekit.SubscriptionResponse] } var ( @@ -78,8 +105,12 @@ var ( ) type Options struct { - AutoSubscribe bool - Publish string + AutoSubscribe bool + Publish string + ClientInfo *livekit.ClientInfo + DisabledCodecs []webrtc.RTPCodecCapability + SignalRequestInterceptor SignalRequestInterceptor + SignalResponseInterceptor SignalResponseInterceptor } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { @@ -92,8 +123,18 @@ func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error connectUrl := u.String() if opts != nil { - connectUrl = fmt.Sprintf("%s&auto_subscribe=%t&publish=%s", - connectUrl, opts.AutoSubscribe, opts.Publish) + connectUrl = fmt.Sprintf("%s&auto_subscribe=%t", connectUrl, opts.AutoSubscribe) + if opts.Publish != "" { + connectUrl += encodeQueryParam("publish", opts.Publish) + } + if opts.ClientInfo != nil { + if opts.ClientInfo.DeviceModel != "" { + connectUrl += encodeQueryParam("device_model", opts.ClientInfo.DeviceModel) + } + if opts.ClientInfo.Os != "" { + connectUrl += encodeQueryParam("os", opts.ClientInfo.Os) + } + } } conn, _, err := websocket.DefaultDialer.Dial(connectUrl, requestHeader) return conn, err @@ -103,7 +144,7 @@ func SetAuthorizationToken(header http.Header, token string) { header.Set("Authorization", "Bearer "+token) } -func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { +func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { var err error c := &RTCClient{ @@ -120,11 +161,14 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { c.ctx, c.cancel = context.WithCancel(context.Background()) conf := rtc.WebRTCConfig{ - Configuration: rtcConf, + WebRTCConfig: rtcconfig.WebRTCConfig{ + Configuration: rtcConf, + }, } conf.SettingEngine.SetLite(false) conf.SettingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleClient) - codecs := []*livekit.Codec{ + var codecs []*livekit.Codec + for _, codec := range []*livekit.Codec{ { Mime: "audio/opus", }, @@ -134,7 +178,21 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { { Mime: "video/h264", }, + } { + var disabled bool + if opts != nil { + for _, dc := range opts.DisabledCodecs { + if strings.EqualFold(dc.MimeType, codec.Mime) && (dc.SDPFmtpLine == "" || dc.SDPFmtpLine == codec.FmtpLine) { + disabled = true + break + } + } + } + if !disabled { + codecs = append(codecs, codec) + } } + // // The signal targets are from point of view of server. // From client side, they are flipped, @@ -231,6 +289,11 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { }) }) + if opts != nil { + c.signalRequestInterceptor = opts.SignalRequestInterceptor + c.signalResponseInterceptor = opts.SignalResponseInterceptor + } + return c, nil } @@ -256,87 +319,101 @@ func (c *RTCClient) Run() error { logger.Errorw("error while reading", err) return err } - switch msg := res.Message.(type) { - case *livekit.SignalResponse_Join: - c.localParticipant = msg.Join.Participant - c.id = livekit.ParticipantID(msg.Join.Participant.Sid) - c.lock.Lock() - for _, p := range msg.Join.OtherParticipants { - c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p - } - c.lock.Unlock() - // if publish only, negotiate - if !msg.Join.SubscriberPrimary { - c.subscriberAsPrimary.Store(false) - c.publisher.Negotiate(false) - } else { - c.subscriberAsPrimary.Store(true) - } - - logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) - case *livekit.SignalResponse_Answer: - // logger.Debugw("received server answer", - // "participant", c.localParticipant.Identity, - // "answer", msg.Answer.Sdp) - c.handleAnswer(rtc.FromProtoSessionDescription(msg.Answer)) - case *livekit.SignalResponse_Offer: - logger.Infow("received server offer", - "participant", c.localParticipant.Identity, - ) - desc := rtc.FromProtoSessionDescription(msg.Offer) - c.handleOffer(desc) - case *livekit.SignalResponse_Trickle: - candidateInit, err := rtc.FromProtoTrickle(msg.Trickle) - if err != nil { - return err - } - if msg.Trickle.Target == livekit.SignalTarget_PUBLISHER { - c.publisher.AddICECandidate(candidateInit) - } else { - c.subscriber.AddICECandidate(candidateInit) - } - case *livekit.SignalResponse_Update: - c.lock.Lock() - for _, p := range msg.Update.Participants { - if livekit.ParticipantID(p.Sid) != c.id { - if p.State != livekit.ParticipantInfo_DISCONNECTED { - c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p - } else { - delete(c.remoteParticipants, livekit.ParticipantID(p.Sid)) - } - } - } - c.lock.Unlock() - - case *livekit.SignalResponse_TrackPublished: - logger.Debugw("track published", "trackID", msg.TrackPublished.Track.Name, "participant", c.localParticipant.Sid, - "cid", msg.TrackPublished.Cid, "trackSid", msg.TrackPublished.Track.Sid) - c.lock.Lock() - c.pendingPublishedTracks[msg.TrackPublished.Cid] = msg.TrackPublished.Track - c.lock.Unlock() - case *livekit.SignalResponse_RefreshToken: - c.lock.Lock() - c.refreshToken = msg.RefreshToken - c.lock.Unlock() - case *livekit.SignalResponse_TrackUnpublished: - sid := msg.TrackUnpublished.TrackSid - c.lock.Lock() - sender := c.trackSenders[sid] - if sender != nil { - if err := c.publisher.RemoveTrack(sender); err != nil { - logger.Errorw("Could not unpublish track", err) - } - c.publisher.Negotiate(false) - } - delete(c.trackSenders, sid) - delete(c.localTracks, sid) - c.lock.Unlock() - case *livekit.SignalResponse_Pong: - c.pongReceivedAt.Store(msg.Pong) + if c.signalResponseInterceptor != nil { + err = c.signalResponseInterceptor(res, c.handleSignalResponse) + } else { + err = c.handleSignalResponse(res) + } + if err != nil { + return err } } } +func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { + switch msg := res.Message.(type) { + case *livekit.SignalResponse_Join: + c.localParticipant = msg.Join.Participant + c.id = livekit.ParticipantID(msg.Join.Participant.Sid) + c.lock.Lock() + for _, p := range msg.Join.OtherParticipants { + c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p + } + c.lock.Unlock() + // if publish only, negotiate + if !msg.Join.SubscriberPrimary { + c.subscriberAsPrimary.Store(false) + c.publisher.Negotiate(false) + } else { + c.subscriberAsPrimary.Store(true) + } + + logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) + case *livekit.SignalResponse_Answer: + // logger.Debugw("received server answer", + // "participant", c.localParticipant.Identity, + // "answer", msg.Answer.Sdp) + c.handleAnswer(rtc.FromProtoSessionDescription(msg.Answer)) + case *livekit.SignalResponse_Offer: + logger.Infow("received server offer", + "participant", c.localParticipant.Identity, + ) + desc := rtc.FromProtoSessionDescription(msg.Offer) + c.handleOffer(desc) + case *livekit.SignalResponse_Trickle: + candidateInit, err := rtc.FromProtoTrickle(msg.Trickle) + if err != nil { + return err + } + if msg.Trickle.Target == livekit.SignalTarget_PUBLISHER { + c.publisher.AddICECandidate(candidateInit) + } else { + c.subscriber.AddICECandidate(candidateInit) + } + case *livekit.SignalResponse_Update: + c.lock.Lock() + for _, p := range msg.Update.Participants { + if livekit.ParticipantID(p.Sid) != c.id { + if p.State != livekit.ParticipantInfo_DISCONNECTED { + c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p + } else { + delete(c.remoteParticipants, livekit.ParticipantID(p.Sid)) + } + } + } + c.lock.Unlock() + + case *livekit.SignalResponse_TrackPublished: + logger.Debugw("track published", "trackID", msg.TrackPublished.Track.Name, "participant", c.localParticipant.Sid, + "cid", msg.TrackPublished.Cid, "trackSid", msg.TrackPublished.Track.Sid) + c.lock.Lock() + c.pendingPublishedTracks[msg.TrackPublished.Cid] = msg.TrackPublished.Track + c.lock.Unlock() + case *livekit.SignalResponse_RefreshToken: + c.lock.Lock() + c.refreshToken = msg.RefreshToken + c.lock.Unlock() + case *livekit.SignalResponse_TrackUnpublished: + sid := msg.TrackUnpublished.TrackSid + c.lock.Lock() + sender := c.trackSenders[sid] + if sender != nil { + if err := c.publisher.RemoveTrack(sender); err != nil { + logger.Errorw("Could not unpublish track", err) + } + c.publisher.Negotiate(false) + } + delete(c.trackSenders, sid) + delete(c.localTracks, sid) + c.lock.Unlock() + case *livekit.SignalResponse_Pong: + c.pongReceivedAt.Store(msg.Pong) + case *livekit.SignalResponse_SubscriptionResponse: + c.subscriptionResponse.Store(msg.SubscriptionResponse) + } + return nil +} + func (c *RTCClient) WaitUntilConnected() error { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -437,6 +514,10 @@ func (c *RTCClient) PongReceivedAt() int64 { return c.pongReceivedAt.Load() } +func (c *RTCClient) GetSubscriptionResponseAndClear() *livekit.SubscriptionResponse { + return c.subscriptionResponse.Swap(nil) +} + func (c *RTCClient) SendPing() error { return c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_Ping{ @@ -446,6 +527,14 @@ func (c *RTCClient) SendPing() error { } func (c *RTCClient) SendRequest(msg *livekit.SignalRequest) error { + if c.signalRequestInterceptor != nil { + return c.signalRequestInterceptor(msg, c.sendRequest) + } else { + return c.sendRequest(msg) + } +} + +func (c *RTCClient) sendRequest(msg *livekit.SignalRequest) error { payload, err := proto.Marshal(msg) if err != nil { return err @@ -607,6 +696,11 @@ func (c *RTCClient) GetPublishedTrackIDs() []string { return trackIDs } +// LastAnswer return SDP of the last answer for the publisher connection +func (c *RTCClient) LastAnswer() *webrtc.SessionDescription { + return c.lastAnswer.Load() +} + func (c *RTCClient) ensurePublisherConnected() error { if c.publisher.HasEverConnected() { return nil @@ -630,7 +724,7 @@ func (c *RTCClient) ensurePublisherConnected() error { } } -func (c *RTCClient) handleDataMessage(kind livekit.DataPacket_Kind, data []byte) { +func (c *RTCClient) handleDataMessage(_ livekit.DataPacket_Kind, data []byte) { dp := &livekit.DataPacket{} err := proto.Unmarshal(data, dp) if err != nil { @@ -651,6 +745,8 @@ func (c *RTCClient) handleOffer(desc webrtc.SessionDescription) { // the client handles answer on the publisher PC func (c *RTCClient) handleAnswer(desc webrtc.SessionDescription) { logger.Infow("handling server answer", "participant", c.localParticipant.Identity) + + c.lastAnswer.Store(&desc) // remote answered the offer, establish connection c.publisher.HandleRemoteDescription(desc) } @@ -741,3 +837,7 @@ func (c *RTCClient) SendNacks(count int) { _ = c.subscriber.WriteRTCP(packets) } + +func encodeQueryParam(key, value string) string { + return fmt.Sprintf("&%s=%s", url.QueryEscape(key), url.QueryEscape(value)) +} diff --git a/test/client/trackwriter.go b/test/client/trackwriter.go index e475a2ed9..9104f9cd2 100644 --- a/test/client/trackwriter.go +++ b/test/client/trackwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package client import ( diff --git a/test/integration_helpers.go b/test/integration_helpers.go index ab2f10a23..82f50f825 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -12,7 +26,6 @@ import ( "github.com/twitchtv/twirp" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/service" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -42,7 +55,7 @@ const ( var roomClient livekit.RoomService func init() { - serverlogger.InitFromConfig(config.LoggingConfig{ + config.InitLoggerFromConfig(config.LoggingConfig{ Config: logger.Config{Level: "debug"}, }) @@ -171,6 +184,7 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer { conf.RTC.TCPPort = port + 2 conf.Redis.Address = "localhost:6379" conf.Keys = map[string]string{testApiKey: testApiSecret} + conf.SignalRelay.Enabled = true currentNode, err := routing.NewLocalNode(conf) if err != nil { @@ -196,7 +210,7 @@ func createRTCClient(name string, port int, opts *testclient.Options) *testclien panic(err) } - c, err := testclient.NewRTCClient(ws) + c, err := testclient.NewRTCClient(ws, opts) if err != nil { panic(err) } @@ -213,7 +227,7 @@ func createRTCClientWithToken(token string, port int, opts *testclient.Options) panic(err) } - c, err := testclient.NewRTCClient(ws) + c, err := testclient.NewRTCClient(ws, opts) if err != nil { panic(err) } diff --git a/test/multinode_roomservice_test.go b/test/multinode_roomservice_test.go index f822ebdb1..a1ec1cb8b 100644 --- a/test/multinode_roomservice_test.go +++ b/test/multinode_roomservice_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/multinode_test.go b/test/multinode_test.go index 57a2ed57e..8b1eb1d32 100644 --- a/test/multinode_test.go +++ b/test/multinode_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -11,6 +25,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/livekit-server/test/client" ) func TestMultiNodeRouting(t *testing.T) { @@ -261,3 +276,46 @@ func TestMultiNodeRevokePublishPermission(t *testing.T) { return "" }) } + +func TestCloseDisconnectedParticipantOnSignalClose(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestCloseDisconnectedParticipantOnSignalClose") + defer finish() + + c1 := createRTCClient("c1", secondServerPort, nil) + waitUntilConnected(t, c1) + + c2 := createRTCClient("c2", defaultServerPort, &client.Options{ + SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error { + switch msg.Message.(type) { + case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave: + return nil + default: + return next(msg) + } + }, + SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error { + switch msg.Message.(type) { + case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer: + return nil + default: + return next(msg) + } + }, + }) + + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 1 { + return "c1 did not see c2 join" + } + return "" + }) + + c2.Stop() + + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 0 { + return "c1 did not see c2 removed" + } + return "" + }) +} diff --git a/test/scenarios.go b/test/scenarios.go index d5c7878e0..d72578614 100644 --- a/test/scenarios.go +++ b/test/scenarios.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/singlenode_test.go b/test/singlenode_test.go index 5af6602e0..fe2e55b07 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -8,6 +22,7 @@ import ( "testing" "time" + "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" "github.com/thoas/go-funk" @@ -22,6 +37,11 @@ import ( testclient "github.com/livekit/livekit-server/test/client" ) +const ( + waitTick = 10 * time.Millisecond + waitTimeout = 5 * time.Second +) + func TestClientCouldConnect(t *testing.T) { if testing.Short() { t.SkipNow() @@ -477,3 +497,170 @@ func TestSingleNodeUpdateSubscriptionPermissions(t *testing.T) { } }) } + +// TestDeviceCodecOverride checks that codecs that are incompatible with a device is not +// negotiated by the server +func TestDeviceCodecOverride(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + _, finish := setupSingleNodeTest("TestDeviceCodecOverride") + defer finish() + + // simulate device that isn't compatible with H.264 + c1 := createRTCClient("c1", defaultServerPort, &testclient.Options{ + ClientInfo: &livekit.ClientInfo{ + Os: "android", + DeviceModel: "Xiaomi 2201117TI", + }, + }) + defer c1.Stop() + waitUntilConnected(t, c1) + + // it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs + tw, err := c1.AddStaticTrack("video/h264", "video", "webcam") + require.NoError(t, err) + defer stopWriters(tw) + + // wait for server to receive track + require.Eventually(t, func() bool { + return c1.LastAnswer() != nil + }, waitTimeout, waitTick, "did not receive answer") + + sd := webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: c1.LastAnswer().SDP, + } + answer, err := sd.Unmarshal() + require.NoError(t, err) + + // video and data channel + require.Len(t, answer.MediaDescriptions, 2) + var desc *sdp.MediaDescription + for _, md := range answer.MediaDescriptions { + if md.MediaName.Media == "video" { + desc = md + break + } + } + require.NotNil(t, desc) + hasSeenVP8 := false + for _, a := range desc.Attributes { + if a.Key == "rtpmap" { + require.NotContains(t, a.Value, "H264", "should not contain H264 codec") + if strings.Contains(a.Value, "VP8") { + hasSeenVP8 = true + } + } + } + require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP") +} + +func TestSubscribeToCodecUnsupported(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + _, finish := setupSingleNodeTest("TestSubscribeToCodecUnsupported") + defer finish() + + c1 := createRTCClient("c1", defaultServerPort, nil) + // create a client that doesn't support H264 + c2 := createRTCClient("c2", defaultServerPort, &testclient.Options{ + AutoSubscribe: true, + DisabledCodecs: []webrtc.RTPCodecCapability{ + {MimeType: "video/H264"}, + }, + }) + waitUntilConnected(t, c1, c2) + + // publish a vp8 video track and ensure c2 receives it ok + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 was not subscribed to tracks from c1" + } + + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if strings.EqualFold(t.Codec().MimeType, "video/vp8") { + return "" + } + } + return "did not receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) + + // publish a h264 track and ensure c2 got subscription error + t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ + MimeType: "video/h264", + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, "videoscreen", "screen") + defer t3.Stop() + require.NoError(t, err) + + var h264TrackID string + require.Eventually(t, func() bool { + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + require.NotNil(t, remoteC1) + for _, track := range remoteC1.Tracks { + if strings.EqualFold(track.MimeType, "video/h264") { + h264TrackID = track.Sid + return true + } + } + return false + }, time.Second, 10*time.Millisecond, "did not receive track info with h264") + + require.Eventually(t, func() bool { + sr := c2.GetSubscriptionResponseAndClear() + if sr == nil { + return false + } + require.Equal(t, h264TrackID, sr.TrackSid) + require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err) + return true + }, 5*time.Second, 10*time.Millisecond, "did not receive subscription response") + + // publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it + t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2") + require.NoError(t, err) + defer t4.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 3 { + return "c2 was not subscribed to tracks from c1" + } + + var vp8Count int + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if strings.EqualFold(t.Codec().MimeType, "video/vp8") { + vp8Count++ + } + } + if vp8Count == 2 { + return "" + } + return "did not 2 receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) +} diff --git a/test/webhook_test.go b/test/webhook_test.go index 22e901e88..678c48c80 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/tools/tools.go b/tools/tools.go index 5a341066b..f77dcd861 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build tools // +build tools diff --git a/version/version.go b/version/version.go index 1f31716a0..1df2fdae5 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package version -const Version = "1.3.5" +const Version = "1.4.4"