diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1a55147..864055f60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,67 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.13.6] - 2026-08-26 + +### Added + +- Expand room details in webhook events (#4730) +- Add configurable read-message size limit on signalling WebSockets (#4743) +- Add per-participant concurrent TURN allocation quota (#4744) +- Limit number of pending tracks per participant. (#4750) +- Limit API request body size (#4757) +- Experimental WARP (#4649) +- Log invalid APIKey on API failures. (#4762) +- Add a small cache for data messages received via SendData API. (#4781) +- Add bytes cap on migration data cache size (#4782) +- TEL-912 Adding response for TransferSIPParticipant rpc (#4783) + +### Changed + +- Remove H.264 baseline (42001f) from default enabled codecs (#4723) +- Apply ICE preference when switching to TCP on unstable UDP (#4703) +- Include data track susbcriptions in WaitForSubscription. (#4727) +- Return incompatible in SetCodecWithState if the codec PT changed (#4729) +- return psrpc.FailedPrecondition for "participant client version does not support moving" error (#4736) +- Update actions/setup-go action to v7 (#4720) +- Validate TURN config to guard against invalid values (#4742) +- Use request id to make api idempotence on sdk retry (#4694) +- Set relay quota per participant at 12 default for dual peer connection + resume scenarios (#4745) +- Record publish time on participant close for pending tracks. (#4738) +- Make subscription limit log Debugw as it could spam in a large room. (#4748) +- Fail server start up on partial prom config. (#4749) +- sample codec payload mismatch error log (#4751) +- Process NACK retransmissions in a single worker per DownTrack (#4758) +- Update module github.com/moby/moby/client to v0.5.1 (#4769) +- Update renovate and pinning behavior, run tools from go.mod (#4759) +- Bump github.com/cilium/ebpf in the go_modules group across 1 directory (#4770) +- bump protocol for passthrough + UpdateEgress removal (#4774) +- Check for ICE connection before closing participant on signal close. (#4780) +- Join failure is a canceled connection attempt. (#4788) +- Update github.com/livekit/mediatransportutil digest to f234b53 (#4785) +- Reduce locking in media track + telemetry listener on move participant. (#4790) +- Group livekit dependency updates in renovate (#4791) +- Return created ingress info from io service (#4787) +- config: clarify AdvertiseInternalIP applies to explicit node_ip too (#4754) + +### Fixed + +- Fix AgentHandler.DrainConnections deadlock on worker close. (#4710) +- Check for pictureID existence in VP8 and VP9 (#4721) +- Do not report end time for participant if the participant is migrating (#4728) +- Check layer value in dependency descriptor and keep it in bounds. (#4739) +- Fix publish track count on migration in. (#4740) +- Cover a couple of more cases on data track runt packet handling. (#4741) +- Close web socket connections in all paths. (#4747) +- Check slice length before access in a couple of more places (#4752) +- Remove auth token from log/being sent back to client on invalid token error (#4756) +- Flush sequencer on stream restart; bound frame-integrity loops (#4760) +- Redact stream keys in UpdateStream API log fields (#4763) +- Fix deadlock in regress codec check. (#4775) +- Flush pending signal responses before closing the web socket. (#4776) +- Strip packet trailers from every VP9 layer frame (#4773) +- Fix simulcast RTX pairing broken by pion/webrtc#3470 (#4800) + ## [1.13.5] - 2026-07-31 ### Added diff --git a/Dockerfile b/Dockerfile index c71b191d3..1629557ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # This image is also the single source of truth for the Go toolchain: CI reads the # version out of this line (see .github/scripts/go-version.sh) so tests and images # always run the same runtime. -FROM golang:1.26.6-alpine3.24@sha256:af8d6740070b8906d12eae1c3e3ea0957fb63f492051ea05e354c38ef9fe88df AS builder +FROM golang:1.26.7-alpine3.24@sha256:28d89ee9cc0ff9fec75c82ca201e6bf7fdf9a679d4b7b24dfa04f2bb766bb468 AS builder ARG TARGETPLATFORM ARG TARGETARCH diff --git a/README.md b/README.md index c49da91db..7d74cd3bf 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,12 @@ -# LiveKit: Real-time video, audio and data for developers +# LiveKit: Realtime infrastructure for voice, video, and AI agents -[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. -It's designed to provide everything you need to build real-time video audio data capabilities in your applications. +[LiveKit](https://livekit.com) is an open source platform for building voice, video, and physical AI agents. +This repository is the LiveKit server: a scalable, distributed WebRTC SFU that moves realtime audio, video, and +data between people, devices, and AI models. The SDKs, agents frameworks, and companion services are linked in +the table at the bottom of this page. LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -23,182 +25,86 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/livekit/livekit/buildtest.yaml?branch=master)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) [![License](https://img.shields.io/github/license/livekit/livekit)](https://github.com/livekit/livekit/blob/master/LICENSE) + +> [!IMPORTANT] +> If you're building Voice AI, [LiveKit Agents](https://github.com/livekit/agents) is the SDK for code-first realtime voice agents. STT, LLM, TTS, turn detection, [expressive speech](https://docs.livekit.io/agents/models/tts/expressive/), [keyterm accuracy](https://docs.livekit.io/agents/models/stt/keyterms/), tool usage, and telephony all come bundled in the framework. It's available in both [Python](https://github.com/livekit/agents) and [Node.js](https://github.com/livekit/agents-js). +> +> ```python +> # agent.py +> from livekit import agents +> from livekit.agents import Agent, AgentServer, AgentSession, STTContextOptions, TurnHandlingOptions, inference +> +> server = AgentServer() +> +> +> @server.rtc_session(agent_name="my-agent") +> async def my_agent(ctx: agents.JobContext): +> session = AgentSession( +> stt=inference.STT(model="deepgram/nova-3", language="multi"), +> llm=inference.LLM(model="google/gemma-4-31b-it"), +> tts=inference.TTS(model="inworld/inworld-tts-2", voice="Ashley"), +> turn_handling=TurnHandlingOptions(turn_detection=inference.TurnDetector()), +> stt_context_options=STTContextOptions(keyterms=["LiveKit", "Acme Corp"]), +> expressive=True, +> ) +> await session.start(room=ctx.room, agent=Agent(instructions="You are a helpful voice AI assistant.")) +> await session.generate_reply(instructions="Greet the user and offer your assistance.") +> +> +> if __name__ == "__main__": +> agents.cli.run_app(server) +> ``` +> +> Models come from [LiveKit Inference](https://docs.livekit.io/agents/models/) with no per-provider API keys, and LiveKit Cloud handles [deployment](https://docs.livekit.io/deploy/agents/) and [observability](https://docs.livekit.io/deploy/observability/). Visit the docs for more info at [docs.livekit.io/agents](https://docs.livekit.io/agents/). + + +## Used in production by + +LiveKit carries billions of calls a year for companies including Salesforce, Nvidia, Oracle, SAP, +Deutsche Telekom, Spotify, Tinder, Coursera, Headspace, Skydio, Retell, Decagon, Cresta, and HeyGen. Read how +[Assort Health](https://livekit.com/customers/assort-health), [Playback](https://livekit.com/customers/playback), and +[Polymath Robotics](https://livekit.com/customers/polymath) use it, or see [more customers](https://livekit.com/customers). + ## Features - Scalable, distributed WebRTC SFU (Selective Forwarding Unit) -- Modern, full-featured client SDKs +- People, devices, and AI agents join the same room as participants, with + [agent dispatch](https://docs.livekit.io/agents/server/agent-dispatch/) to route agents in automatically or on demand +- Modern, full-featured SDKs for web, mobile, desktop, embedded, and server - Built for production, supports JWT authentication - Robust networking and connectivity, UDP/TCP/TURN - Easy to deploy: single binary, Docker or Kubernetes - Advanced features including: - - [speaker detection](https://docs.livekit.io/home/client/tracks/subscribe/#speaker-detection) - - [simulcast](https://docs.livekit.io/home/client/tracks/publish/#video-simulcast) - - [end-to-end optimizations](https://blog.livekit.io/livekit-one-dot-zero/) - - [selective subscription](https://docs.livekit.io/home/client/tracks/subscribe/#selective-subscription) - - [moderation APIs](https://docs.livekit.io/home/server/managing-participants/) - - end-to-end encryption + - [speaker detection](https://docs.livekit.io/transport/media/subscribe/) + - [simulcast](https://docs.livekit.io/transport/media/publish/) + - [selective subscription](https://docs.livekit.io/transport/media/subscribe/) + - [moderation APIs](https://docs.livekit.io/intro/basics/rooms-participants-tracks/participants/) + - [end-to-end encryption](https://docs.livekit.io/transport/media/encryption/) - SVC codecs (VP9, AV1) - - [webhooks](https://docs.livekit.io/home/server/webhooks/) - - [distributed and multi-region](https://docs.livekit.io/home/self-hosting/distributed/) + - [data tracks](https://docs.livekit.io/transport/data/data-tracks/) for low-latency telemetry and teleoperation + - [telephony](https://docs.livekit.io/telephony/) over SIP + - [webhooks](https://docs.livekit.io/intro/basics/rooms-participants-tracks/webhooks-events/) + - [distributed and multi-region](https://docs.livekit.io/transport/self-hosting/distributed/) ## Documentation & Guides https://docs.livekit.io +Working with a coding agent? Give it the [LiveKit Docs MCP server](https://docs.livekit.io/mcp/), or start with the +[coding agents guide](https://docs.livekit.io/intro/coding-agents/). + ## Live Demos +- [Talk to a voice agent](https://livekit.com) built with LiveKit Agents - [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit-examples/meet)) - [Spatial Audio](https://spatial-audio-demo.livekit.io/) ([source](https://github.com/livekit-examples/spatial-audio)) - Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) -- [AI voice assistant using ChatGPT](https://livekit.io/kitt) ([source](https://github.com/livekit-examples/kitt)) - -## Ecosystem - -- [Agents](https://github.com/livekit/agents): build real-time multimodal AI applications with programmable backend participants -- [Egress](https://github.com/livekit/egress): record or multi-stream rooms and export individual tracks -- [Ingress](https://github.com/livekit/ingress): ingest streams from external sources like RTMP, WHIP, HLS, or OBS Studio - -## SDKs & Tools - -### Client SDKs - -Client SDKs enable your frontend to include interactive, multi-user experiences. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
LanguageRepo - Declarative UI - Links
JavaScript (TypeScript) - client-sdk-js - - React - - docs - | - JS example - | - React example -
Swift (iOS / MacOS) - client-sdk-swift - Swift UI - docs - | - example -
Kotlin (Android) - client-sdk-android - Compose - docs - | - example - | - Compose example -
Flutter (all platforms) - client-sdk-flutter - native - docs - | - example -
Unity WebGL - client-sdk-unity-web - - docs -
React Native (beta) - client-sdk-react-native - native
Rust - client-sdk-rust -
- -### Server SDKs - -Server SDKs enable your backend to generate [access tokens](https://docs.livekit.io/home/get-started/authentication/), -call [server APIs](https://docs.livekit.io/reference/server/server-apis/), and -receive [webhooks](https://docs.livekit.io/home/server/webhooks/). In addition, the Go SDK includes client capabilities, -enabling you to build automations that behave like end-users. - -| Language | Repo | Docs | -| :---------------------- | :-------------------------------------------------------------------------------------- | :---------------------------------------------------------- | -| Go | [server-sdk-go](https://github.com/livekit/server-sdk-go) | [docs](https://pkg.go.dev/github.com/livekit/server-sdk-go) | -| JavaScript (TypeScript) | [server-sdk-js](https://github.com/livekit/server-sdk-js) | [docs](https://docs.livekit.io/server-sdk-js/) | -| Ruby | [server-sdk-ruby](https://github.com/livekit/server-sdk-ruby) | | -| Java (Kotlin) | [server-sdk-kotlin](https://github.com/livekit/server-sdk-kotlin) | | -| Python (community) | [python-sdks](https://github.com/livekit/python-sdks) | | -| PHP (community) | [agence104/livekit-server-sdk-php](https://github.com/agence104/livekit-server-sdk-php) | | - -### Tools - -- [CLI](https://github.com/livekit/livekit-cli) - command line interface & load tester -- [Docker image](https://hub.docker.com/r/livekit/livekit-server) -- [Helm charts](https://github.com/livekit/livekit-helm) ## Install > [!TIP] > We recommend installing [LiveKit CLI](https://github.com/livekit/livekit-cli) along with the server. It lets you access -> server APIs, create tokens, and generate test traffic. +> server APIs, create tokens, generate test traffic, and scaffold and deploy agents. The following will install LiveKit's media server: @@ -229,11 +135,11 @@ API Key: devkey API Secret: secret ``` -To customize your setup for production, refer to our [deployment docs](https://docs.livekit.io/deploy/) +To customize your setup for production, refer to our [deployment docs](https://docs.livekit.io/transport/self-hosting/deployment/) ### Creating access token -A user connecting to a LiveKit room requires an [access token](https://docs.livekit.io/home/get-started/authentication/#creating-a-token). Access +A user connecting to a LiveKit room requires an [access token](https://docs.livekit.io/frontends/build/authentication/). Access tokens (JWT) encode the user's identity and the room permissions they've been granted. You can generate a token with our CLI: @@ -247,7 +153,7 @@ lk token create \ ### Test with example app Head over to our [example app](https://example.livekit.io) and enter a generated token to connect to your LiveKit -server. This app is built with our [React SDK](https://github.com/livekit/livekit-react). +server. Once connected, your video and audio are now being published to your new LiveKit instance! @@ -266,24 +172,34 @@ This command publishes a looped demo video to a room. Due to how the video clip there's a slight delay before the browser has sufficient data to begin rendering frames. This is an artifact of the simulation. +### Adding an agent + +Agents join rooms as participants, the same way a browser or a phone does. Follow the +[Voice AI quickstart](https://docs.livekit.io/agents/start/voice-ai/) to build one. An agent connects to a self-hosted +server the same way it connects to LiveKit Cloud; when running without Cloud, use +[model plugins](https://docs.livekit.io/agents/models/#plugins) in place of LiveKit Inference. + ## Deployment ### Use LiveKit Cloud -LiveKit Cloud is the fastest and most reliable way to run LiveKit. Every project gets free monthly bandwidth and -transcoding credits. +LiveKit Cloud is the fastest and most reliable way to run LiveKit. It runs in 19+ regions with 99.99% uptime and adds +agent hosting, model inference, telephony, and observability on top of the server. The Build plan is free, with no +credit card required. Sign up for [LiveKit Cloud](https://cloud.livekit.io/). ### Self-host -Read our [deployment docs](https://docs.livekit.io/transport/self-hosting/) for more information. +Read our [deployment docs](https://docs.livekit.io/transport/self-hosting/) for more information. Official +[Docker images](https://hub.docker.com/r/livekit/livekit-server) and [Helm charts](https://github.com/livekit/livekit-helm) +are available. ## Building from source Pre-requisites: -- Go 1.23+ is installed +- Go 1.26+ is installed - GOPATH/bin is in your PATH Then run @@ -298,7 +214,8 @@ mage ## Contributing We welcome your contributions toward improving LiveKit! Please join us -[on Slack](http://livekit.io/join-slack) to discuss your ideas and/or PRs. +[on Slack](http://livekit.io/join-slack) or in the [Developer Community](https://community.livekit.io) to discuss your +ideas and/or PRs. ## License diff --git a/cmd/test-server/Dockerfile b/cmd/test-server/Dockerfile index e0dc826db..f0ff258cc 100644 --- a/cmd/test-server/Dockerfile +++ b/cmd/test-server/Dockerfile @@ -17,7 +17,7 @@ # # Pinned by digest so the build is reproducible even if the tag is republished. # The tag is kept alongside it for readability; Renovate updates both together. -FROM golang:1.26.6-alpine3.24@sha256:af8d6740070b8906d12eae1c3e3ea0957fb63f492051ea05e354c38ef9fe88df AS builder +FROM golang:1.26.7-alpine3.24@sha256:28d89ee9cc0ff9fec75c82ca201e6bf7fdf9a679d4b7b24dfa04f2bb766bb468 AS builder ARG TARGETARCH diff --git a/cmd/test-server/handlers.go b/cmd/test-server/handlers.go index 62c1b6c8b..dcb00e712 100644 --- a/cmd/test-server/handlers.go +++ b/cmd/test-server/handlers.go @@ -24,7 +24,6 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils/protojson" @@ -107,7 +106,7 @@ func init() { reg[livekit.ListSIPDispatchRuleRequest, livekit.ListSIPDispatchRuleResponse]("livekit.SIP/ListSIPDispatchRule") reg[livekit.DeleteSIPDispatchRuleRequest, livekit.SIPDispatchRuleInfo]("livekit.SIP/DeleteSIPDispatchRule") reg[livekit.CreateSIPParticipantRequest, livekit.SIPParticipantInfo]("livekit.SIP/CreateSIPParticipant") - reg[livekit.TransferSIPParticipantRequest, emptypb.Empty]("livekit.SIP/TransferSIPParticipant") + reg[livekit.TransferSIPParticipantRequest, livekit.TransferSIPParticipantResponse]("livekit.SIP/TransferSIPParticipant") // Connector reg[livekit.DialWhatsAppCallRequest, livekit.DialWhatsAppCallResponse]("livekit.Connector/DialWhatsAppCall") diff --git a/cmd/test-server/signal.go b/cmd/test-server/signal.go index 376359ab4..22d52651b 100644 --- a/cmd/test-server/signal.go +++ b/cmd/test-server/signal.go @@ -15,7 +15,11 @@ package main import ( + "bytes" + "compress/gzip" + "encoding/base64" "encoding/json" + "io" "net/http" "strconv" "strings" @@ -216,7 +220,7 @@ func (h *mockHandler) handleSignal(w http.ResponseWriter, r *http.Request) { return } - reconnect := r.URL.Query().Get("reconnect") == "1" + reconnect := r.URL.Query().Get("reconnect") == "1" || joinRequestSaysReconnect(r.URL.Query().Get("join_request")) conn, err := signalUpgrader.Upgrade(w, r, nil) if err != nil { @@ -416,6 +420,45 @@ func reconnectResponse(regionIndex int) *livekit.SignalResponse { } } +// joinRequestSaysReconnect unpacks the v1 `join_request` param and reports its +// Reconnect flag. Any decode error is ignored (treated as non-reconnect); this +// is a test mock, not a validator. +func joinRequestSaysReconnect(param string) bool { + if param == "" { + return false + } + wrappedBytes, err := base64.URLEncoding.DecodeString(param) + if err != nil { + return false + } + wrapped := &livekit.WrappedJoinRequest{} + if err := proto.Unmarshal(wrappedBytes, wrapped); err != nil { + return false + } + var joinBytes []byte + switch wrapped.Compression { + case livekit.WrappedJoinRequest_NONE: + joinBytes = wrapped.JoinRequest + case livekit.WrappedJoinRequest_GZIP: + gz, err := gzip.NewReader(bytes.NewReader(wrapped.JoinRequest)) + if err != nil { + return false + } + defer func() { _ = gz.Close() }() + joinBytes, err = io.ReadAll(gz) + if err != nil { + return false + } + default: + return false + } + join := &livekit.JoinRequest{} + if err := proto.Unmarshal(joinBytes, join); err != nil { + return false + } + return join.Reconnect +} + func leaveResponse(action livekit.LeaveRequest_Action) *livekit.SignalResponse { return &livekit.SignalResponse{ Message: &livekit.SignalResponse_Leave{ diff --git a/config-sample.yaml b/config-sample.yaml index 1cbf8a9e1..5aca3bea7 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -67,7 +67,8 @@ rtc: use_external_ip: true # # when set to true, advertises both mapped external and internal IPs to clients as server candidates. # # useful when clients connect from both private and public networks. - # # works only when `use_external_ip` is set to true. + # # works when `use_external_ip` is set to true, and also when `node_ip` is set explicitly below. + # # in both cases the node's local candidate is kept alongside the mapped one instead of being replaced by it. # # when both this and `external_ip_only` are set, SFU advertises all private IPs with their mapped external IPs and skips # # private IPs that do not have a mapped external IP. # advertise_internal_ip: true @@ -267,6 +268,15 @@ keys: # backoff: 500ms # # number of messages to buffer before dropping # buffer_size: 1000 +# # optional gzip compression of bus payloads +# compression: +# # gzip level 1-9; 0 disables. every node on the bus must support +# # compression before enabling it +# quality: 0 +# # payload bytes below which compression is skipped +# threshold: 1024 +# # cap on an inbound payload after decompression, 0 for unlimited +# max_decompressed_size: 0 # customize audio level sensitivity # audio: @@ -307,6 +317,19 @@ keys: # # set external_tls to true if using a L4 load balancer to terminate TLS. when enabled, # # LiveKit expects unencrypted traffic on tls_port, and still advertise tls_port as a TURN/TLS candidate. # external_tls: true +# # set proxy_protocol to true if the proxy or load balancer in front of tls_port does not preserve +# # the client address (it terminates TLS, or dials LiveKit from its own IP) and can send a +# # PROXY protocol v1/v2 header instead. Without it TURN reports the proxy's address to the +# # client as XOR-MAPPED-ADDRESS, which Firefox rejects when it is loopback or wildcard. +# # every connection on tls_port must then carry the header; connections without it are rejected. +# # prefer PROXY protocol v2 on the proxy side: the v1 text header must arrive in a single read. +# proxy_protocol: false +# # proxies whose PROXY header is trusted. connections from any other address are closed, so a +# # client that reaches tls_port directly cannot claim an arbitrary source address. +# # defaults to loopback, for a proxy running on the same host. +# proxy_protocol_trusted_cidrs: +# - 127.0.0.0/8 +# - ::1/128 # # needs to match tls cert domain # domain: turn.myhost.com # # optional (set only if not using external TLS termination) @@ -336,6 +359,15 @@ keys: # rtmp_base_url: "rtmp://my.domain.com/live" # # Prefix used to generate WHIP URLs for WHIP ingress. # whip_base_url: "http://my.domain.com/whip" +# # Allow URL pull ingress from udp:// source URLs. Disabled by default. +# # Only enable this if you trust both the callers allowed to create ingresses and the network the +# # ingress handlers run on. Unlike an http or srt source url, a udp source url doesn't make the +# # handler connect out to the url host: the handler binds a local socket on the address and port +# # taken from the url, and joins the multicast group if one is given. This lets the caller choose +# # which local port the handler binds, feed the session unauthenticated and easily spoofed traffic, +# # and make the handler join arbitrary multicast groups and republish whatever it receives into a +# # room, using the ingress as a relay for streams on the handler's local network. +# enable_udp_url_pull: false # Region of the current node. Required if using regionaware node selector # region: us-west-2 diff --git a/go.mod b/go.mod index 5b31f4014..99e97062c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/livekit/livekit-server -go 1.26 +go 1.26.0 require ( github.com/bep/debounce v1.2.1 @@ -20,9 +20,9 @@ require ( github.com/jellydator/ttlcache/v3 v3.4.1 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 - github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 - github.com/livekit/protocol v1.50.5-0.20260831015717-ba22cc1ce840 - github.com/livekit/psrpc v0.7.3 + github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 + github.com/livekit/protocol v1.51.1-0.20260905133529-a4f4b5c0c23f + github.com/livekit/psrpc v0.7.7 github.com/mackerelio/go-osstat v0.2.8 github.com/magefile/mage v1.17.2 github.com/mitchellh/go-homedir v1.1.0 @@ -38,15 +38,16 @@ require ( github.com/pion/sctp v1.11.1 github.com/pion/sdp/v3 v3.0.19 github.com/pion/transport/v4 v4.1.0 - github.com/pion/turn/v5 v5.0.12 + github.com/pion/turn/v5 v5.0.13 github.com/pion/webrtc/v4 v4.2.18 + github.com/pires/go-proxyproto v0.15.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.24.1 github.com/quic-go/quic-go v0.61.0 github.com/quic-go/webtransport-go v0.12.0 github.com/redis/go-redis/v9 v9.22.0 github.com/rs/cors v1.11.1 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/thoas/go-funk v0.9.3 github.com/tomnomnom/linkheader v0.0.0-20250811210735-e5fe3b51442e github.com/twitchtv/twirp v8.1.3+incompatible @@ -55,13 +56,14 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.28.0 - golang.org/x/mod v0.38.0 + golang.org/x/mod v0.40.0 golang.org/x/sync v0.22.0 - google.golang.org/protobuf v1.36.11 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) require ( + cel.dev/cel-go v0.32.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cilium/ebpf v0.22.0 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect @@ -85,80 +87,78 @@ require ( github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.3.0 // indirect github.com/olekukonko/ll v0.1.8 // indirect - github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 // indirect + github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 // indirect - golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect + golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect + golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect golang.org/x/time v0.15.0 // indirect ) require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1 // indirect - buf.build/go/protovalidate v1.2.0 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.1 // indirect + buf.build/go/protovalidate v1.4.0 // indirect buf.build/go/protoyaml v0.7.0 // indirect - cel.dev/expr v0.25.2 // indirect + cel.dev/expr v0.25.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-logr/logr v1.4.4 // indirect - github.com/google/cel-go v0.30.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/lithammer/shortuuid/v4 v4.2.0 // indirect + github.com/lithammer/shortuuid/v4 v4.3.0 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mdlayher/netlink v1.11.2 // indirect github.com/mdlayher/socket v0.6.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nats.go v1.53.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pion/logging v0.2.4 - github.com/pion/mdns/v2 v2.1.0 // indirect + github.com/pion/mdns/v2 v2.2.0 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/srtp/v3 v3.0.13 // indirect - github.com/pion/stun/v3 v3.1.6 + github.com/pion/stun/v3 v3.1.7 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.70.1 // indirect - github.com/prometheus/procfs v0.21.1 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.71.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect github.com/urfave/cli/v3 v3.10.1 github.com/wlynxg/anet v0.0.5 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/crypto v0.56.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/grpc v1.83.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a // indirect + google.golang.org/grpc v1.83.2 // indirect ) tool ( @@ -172,4 +172,4 @@ replace github.com/pion/webrtc/v4 => github.com/livekit/webrtc-pion/v4 v4.2.18-w replace github.com/pion/dtls/v3 => github.com/livekit/dtls/v3 v3.1.5-warp.1 -replace github.com/pion/ice/v4 => github.com/livekit/ice/v4 v4.4.0-warp.1 +replace github.com/pion/ice/v4 => github.com/livekit/ice/v4 v4.4.0-warp.2 diff --git a/go.sum b/go.sum index e9db5cc97..49d89faf0 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1 h1:fXh8CsdNpjRr8R5vFdqtIxPt/Lno2IIJlYOdZBIZn0w= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= -buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= -buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.1 h1:Slv0uGxx219srASyiaI5C9cDlyG8kNDcXpTSYcuAeE4= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.1/go.mod h1:TCt1lluMFnctISJXvkIQ4x3ABrPuUKCWKyjKdkJNBpw= +buf.build/go/protovalidate v1.4.0 h1:UjLrYbt5VX7+TMOs2+pG5FhZhIG1mSfK4EIopbb4LcM= +buf.build/go/protovalidate v1.4.0/go.mod h1:8vJfzNT6NIG2qm3uFsJDXMlRmG+bQJzbcIn1Aa0vPGs= buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -44,8 +46,6 @@ github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -54,8 +54,6 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= -github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elliotchance/orderedmap/v3 v3.1.1 h1:eV7lfZ5fVL8d36b8Wogqi/eqm7R/kZcftA9Yiyj+63M= @@ -87,8 +85,6 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.30.0 h1:ll54AkzKunWkBn9wSoiUXbFZXYZTkdJGNXTBXUoolGo= -github.com/google/cel-go v0.30.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -143,8 +139,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -158,20 +154,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= -github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/lithammer/shortuuid/v4 v4.3.0 h1:XRr80OYPOlqxgnCv2/iuElkl/ZtXtPtLKk2AvpXGdWA= +github.com/lithammer/shortuuid/v4 v4.3.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= github.com/livekit/dtls/v3 v3.1.5-warp.1 h1:5jo2qQreDnUrgw6Al1F+9xVyXHvWQ2IaQkjF53FHEyU= github.com/livekit/dtls/v3 v3.1.5-warp.1/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU= -github.com/livekit/ice/v4 v4.4.0-warp.1 h1:P9Y1f02VVx6kkRJxY3jzoS7HnblFbYkgMwiu6iOqCOU= -github.com/livekit/ice/v4 v4.4.0-warp.1/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU= +github.com/livekit/ice/v4 v4.4.0-warp.2 h1:AYfwxksKNs2LGgLIifsrB/K4ixaQDuURxBgrA5dJBo4= +github.com/livekit/ice/v4 v4.4.0-warp.2/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 h1:XHNNzebIKZRkLimla/hFGrAIX5EMWHctrgt3hLw7s+I= -github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.50.5-0.20260831015717-ba22cc1ce840 h1:VGdD4zhgRk5o/J+CPX/qT9lU/7l78eVwW8SPvrKBP8I= -github.com/livekit/protocol v1.50.5-0.20260831015717-ba22cc1ce840/go.mod h1:/kYxa0dlTuH981LaBFHG/Swyr969d0+2+/6Lm7fFc34= -github.com/livekit/psrpc v0.7.3 h1:bekuZt/ZQzg8+/M8G6G5jq7bvV9fAKdPHSOZeTwrIIc= -github.com/livekit/psrpc v0.7.3/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= +github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= +github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= +github.com/livekit/protocol v1.51.1-0.20260905133529-a4f4b5c0c23f h1:+48IWNrsoTgbB0JGv+xJl4umx6e/vh1BX1Tcokuacwc= +github.com/livekit/protocol v1.51.1-0.20260905133529-a4f4b5c0c23f/go.mod h1:zxowkRnQlJ2VMn6ZyinXMDi985wcKXuWNeXmEERqFAs= +github.com/livekit/psrpc v0.7.7 h1:eZ/jYlayQ3Y+C/3+NPtwTJBXZo/aN4vJf2MmBK6AfT4= +github.com/livekit/psrpc v0.7.7/go.mod h1:Twno03W8gpTNRpmc2cQwSoxYEo/nJxf+65iMq+9IuHo= github.com/livekit/webrtc-pion/v4 v4.2.18-warp.1 h1:fH+v4W+NFp9FfPzON6FaUFNmazGcctaAhb2P+Ksf+1s= github.com/livekit/webrtc-pion/v4 v4.2.18-warp.1/go.mod h1:rbKGHo2OpNUImWTvRIV776/3xjjq/t47H3IZiTtwluc= github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= @@ -215,8 +211,8 @@ github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJ github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= -github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nats.go v1.53.1 h1:Otsq3uLc/kLdjmkNHkXH0jBqwUquwdKFoe3fq6/3/Xo= +github.com/nats-io/nats.go v1.53.1/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -239,16 +235,16 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoPME= github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= -github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 h1:VmZ6mKVkxavKEhEy4ZYyV7BwBYBFBP0TwIqmLk84fpU= -github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 h1:lcWAnrqr2nNfDiArwFNHCE4787Mw2tCdVSOXCru0/0E= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo= github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= -github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= -github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= +github.com/pion/mdns/v2 v2.2.0 h1:AlAZ9MTUKtWgO+4itk35JdNak4sk5k7G/X4xnIBWHyA= +github.com/pion/mdns/v2 v2.2.0/go.mod h1:IJddx58QMlojqhQYjHcOUmvuBQ5MnLNetkb80VMvk2Y= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw= @@ -261,14 +257,16 @@ github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= github.com/pion/srtp/v3 v3.0.13 h1:FmQaqgNbN1vUtMhEsmj8trldc3lNZr1xmN7nl8CyX+Q= github.com/pion/srtp/v3 v3.0.13/go.mod h1:7qR3L69t8RX0EPVQwGNwCa1Gy9keKKNDpWwQzZbeXDY= -github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8= -github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= +github.com/pion/stun/v3 v3.1.7 h1:uRXMTlGLf89WgItGNyZ6aR5jMTX0NBbybXADpQCzn+E= +github.com/pion/stun/v3 v3.1.7/go.mod h1:Nq77RW4aRrSNrltf2ksUJLjxWeipj4lnlgdsYIxC8g8= github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= github.com/pion/transport/v4 v4.1.0 h1:8S+nF2reM2cJuqC6g78OVy2BBgmbdns+acx3jA97BvQ= github.com/pion/transport/v4 v4.1.0/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= -github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI= -github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg= +github.com/pion/turn/v5 v5.0.13 h1:erHOsJyxuV6QK54+PjWJhe8u1O7BM3a/US0zYJJsnx4= +github.com/pion/turn/v5 v5.0.13/go.mod h1:btdOovUYdYc8iBnvt87JHN4Pa1XV5UiLaCYe4ay3o9A= +github.com/pires/go-proxyproto v0.15.0 h1:dTshmNbFm/D+0+sbrxUuddPOZ5Y0B7c5NhtsBkm6LqI= +github.com/pires/go-proxyproto v0.15.0/go.mod h1:OXsCrKwrK2tXS9YrI5tkHx5xaQlO8FH3lFW76orFh24= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -276,22 +274,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= -github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= +github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= -github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= -github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= -github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= -github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA= -github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= -github.com/quic-go/webtransport-go v0.12.0 h1:CpnKNwZvdV0LD73xoHO8QaR0NI3llqpWRwnazdZS0sE= -github.com/quic-go/webtransport-go v0.12.0/go.mod h1:GHne8aRFJ24h73pAMrcywXtuaz/ShBXCLXLvG/NPFdU= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= @@ -307,8 +297,8 @@ github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tomnomnom/linkheader v0.0.0-20250811210735-e5fe3b51442e h1:tD38/4xg4nuQCASJ/JxcvCHNb46w0cdAaJfkzQOO1bA= @@ -331,28 +321,26 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= @@ -365,12 +353,12 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -385,8 +373,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -416,33 +404,33 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A= -golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a h1:i3TAXhpKc7TUP1VAPiBBrv45kamjoizCC3rOC0cAbOs= +google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:CvYJHpbzPlT0fb/PsgtAamdwru/GVxUsomFdXTpOTI8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a h1:3Dnd1cDaZlB68lziofO+bJXpjOy8UfRv8Unt+yH8tQ4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go index 00116528c..229820683 100644 --- a/pkg/agent/worker.go +++ b/pkg/agent/worker.go @@ -160,6 +160,13 @@ type WorkerRegistration struct { Endpoints []*livekit.AgentHttp_AgentEndpoint InstanceID string EndpointSettings *livekit.AgentHttp_AgentEndpointSettings + + // KindDetails, when set by the server, are stamped onto the participant + // join token minted for every job assigned to this worker (e.g. marking + // hosted/cloud agents so they can be distinguished from self-hosted ones in + // observability). This is server-controlled and never populated from the + // worker's register request. + KindDetails []livekit.ParticipantInfo_KindDetail } func MakeWorkerRegistration() WorkerRegistration { @@ -441,6 +448,7 @@ func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job, hook Assignmen res.ParticipantMetadata, attributes, w.Permissions, + w.KindDetails..., ) if err != nil { w.logger.Errorw("failed to build agent token", err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2ef6a3efe..3ca89107e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -279,6 +279,19 @@ type TURNConfig struct { RelayPortRangeEnd uint16 `yaml:"relay_range_end,omitempty"` ExternalTLS bool `yaml:"external_tls,omitempty"` BindAddresses []string `yaml:"bind_addresses,omitempty"` + // ProxyProtocol makes the TURN TCP listener require a PROXY protocol (v1 or v2) + // header on every connection and use the client address it carries. Needed when + // a proxy in front of TURN dials from its own address (a TLS-terminating L4 + // proxy, a TCP reverse proxy): TURN echoes the address it sees back to the + // client as XOR-MAPPED-ADDRESS, and browsers such as Firefox reject a loopback + // or wildcard address there and abandon the allocation. Connections without + // the header are rejected. + ProxyProtocol bool `yaml:"proxy_protocol,omitempty"` + // ProxyProtocolTrustedCIDRs lists the proxies whose PROXY header is believed. + // Connections from any other address are closed, so a client that reaches + // tls_port directly cannot claim an arbitrary source address. Defaults to + // loopback only, for a proxy on the same host. + ProxyProtocolTrustedCIDRs []string `yaml:"proxy_protocol_trusted_cidrs,omitempty"` // PerUserRelayAllocationLimit caps the number of concurrent relay allocations // a single participant credential may hold, keyed by the participant ID. This // stops one authenticated participant from consuming the shared relay-port @@ -439,6 +452,19 @@ func (l LimitConfig) CanAddDataBlob(dataBlobs []*livekit.DataBlob, toAdd *liveki type IngressConfig struct { RTMPBaseURL string `yaml:"rtmp_base_url,omitempty"` WHIPBaseURL string `yaml:"whip_base_url,omitempty"` + // Allow URL pull ingresses with a udp:// source URL. Disabled by default, and should only be + // enabled if both the callers allowed to create ingresses and the network the ingress handlers + // run on are trusted. Unlike an http or srt source url, a udp source url doesn't make the ingress + // handler connect out to the url host: the handler binds a local socket on the address and port + // taken from the url, and joins the multicast group if one is given. This lets the caller: + // - Choose which local port the handler binds, potentially colliding with other services on + // the host. + // - Feed the session unauthenticated traffic. UDP is connectionless, so any host able to reach + // that port can inject media, or spoof the sender address to disrupt a legitimate feed. + // - Make the handler join arbitrary multicast groups and republish whatever it receives into a + // room, using the ingress as a relay for streams on the handler's local network the caller has + // no direct access to. + EnableUDPURLPull bool `yaml:"enable_udp_url_pull,omitempty"` } type SIPConfig struct{} @@ -572,6 +598,7 @@ var DefaultConfig = Config{ TURN: TURNConfig{ Enabled: false, BindAddresses: []string{"0.0.0.0"}, + ProxyProtocolTrustedCIDRs: []string{"127.0.0.0/8", "::1/128"}, TTLSeconds: DefaultTURNTTLSeconds, PerUserRelayAllocationLimit: DefaultTURNPerUserRelayAllocationLimit, }, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 3e9cfbf48..0b6439945 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -61,6 +61,27 @@ func TestConfig_SignalMessageSizeLimitOverride(t *testing.T) { require.Equal(t, int64(0), conf.Limit.AgentSignalMessageSizeLimit) } +func TestConfig_PSRPCCompressionDefaults(t *testing.T) { + conf, err := NewConfig("", true, nil, nil) + require.NoError(t, err) + require.Equal(t, 0, conf.PSRPC.Compression.Quality) + require.Equal(t, 1024, conf.PSRPC.Compression.Threshold) + require.Equal(t, 0, conf.PSRPC.Compression.MaxDecompressedSize) +} + +func TestConfig_PSRPCCompressionOverride(t *testing.T) { + const content = `psrpc: + compression: + quality: 6 + max_decompressed_size: 4096` + conf, err := NewConfig(content, true, nil, nil) + require.NoError(t, err) + require.Equal(t, 6, conf.PSRPC.Compression.Quality) + require.Equal(t, 4096, conf.PSRPC.Compression.MaxDecompressedSize) + require.Equal(t, 1024, conf.PSRPC.Compression.Threshold) + require.Equal(t, 3, conf.PSRPC.MaxAttempts) +} + func TestConfig_UnknownKeys(t *testing.T) { const content = `unknown: 10 room: diff --git a/pkg/routing/selector/cpuload.go b/pkg/routing/selector/cpuload.go index f84b4a3ce..3c31236f1 100644 --- a/pkg/routing/selector/cpuload.go +++ b/pkg/routing/selector/cpuload.go @@ -27,21 +27,13 @@ type CPULoadSelector struct { } func (s *CPULoadSelector) filterNodes(nodes []*livekit.Node) ([]*livekit.Node, error) { - nodes = GetAvailableNodes(nodes) - if len(nodes) == 0 { - return nil, ErrNoAvailableNodes + nodes, err := FilterNodesByCriteria(nodes, s.CPULoadLimit, func(node *livekit.Node) float32 { + return node.Stats.CpuLoad + }) + if err != nil { + return nil, err } - nodesLowLoad := make([]*livekit.Node, 0) - for _, node := range nodes { - stats := node.Stats - if stats.CpuLoad < s.CPULoadLimit { - nodesLowLoad = append(nodesLowLoad, node) - } - } - if len(nodesLowLoad) > 0 { - nodes = nodesLowLoad - } return nodes, nil } diff --git a/pkg/routing/selector/nodesfilter.go b/pkg/routing/selector/nodesfilter.go new file mode 100644 index 000000000..3dd49d3b5 --- /dev/null +++ b/pkg/routing/selector/nodesfilter.go @@ -0,0 +1,21 @@ +package selector + +import "github.com/livekit/protocol/livekit" + +func FilterNodesByCriteria(nodes []*livekit.Node, criteriaThreshold float32, calculateCriteriaFunc func(*livekit.Node) float32) ([]*livekit.Node, error) { + nodes = GetAvailableNodes(nodes) + if len(nodes) == 0 { + return nil, ErrNoAvailableNodes + } + + filteredNodes := make([]*livekit.Node, 0) + for _, node := range nodes { + if calculateCriteriaFunc(node) < criteriaThreshold { + filteredNodes = append(filteredNodes, node) + } + } + if len(filteredNodes) > 0 { + nodes = filteredNodes + } + return nodes, nil +} diff --git a/pkg/routing/selector/sysload.go b/pkg/routing/selector/sysload.go index 285fca493..88af4a19c 100644 --- a/pkg/routing/selector/sysload.go +++ b/pkg/routing/selector/sysload.go @@ -27,20 +27,11 @@ type SystemLoadSelector struct { } func (s *SystemLoadSelector) filterNodes(nodes []*livekit.Node) ([]*livekit.Node, error) { - nodes = GetAvailableNodes(nodes) - if len(nodes) == 0 { - return nil, ErrNoAvailableNodes + nodes, err := FilterNodesByCriteria(nodes, s.SysloadLimit, GetNodeSysload) + if err != nil { + return nil, err } - nodesLowLoad := make([]*livekit.Node, 0) - for _, node := range nodes { - if GetNodeSysload(node) < s.SysloadLimit { - nodesLowLoad = append(nodesLowLoad, node) - } - } - if len(nodesLowLoad) > 0 { - nodes = nodesLowLoad - } return nodes, nil } diff --git a/pkg/rtc/datadowntrack.go b/pkg/rtc/datadowntrack.go index 45f5e19a8..a1b0fdc35 100644 --- a/pkg/rtc/datadowntrack.go +++ b/pkg/rtc/datadowntrack.go @@ -19,8 +19,8 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/datatrack.go b/pkg/rtc/datatrack.go index fd22f6258..1d93c296d 100644 --- a/pkg/rtc/datatrack.go +++ b/pkg/rtc/datatrack.go @@ -21,9 +21,9 @@ import ( "github.com/frostbyte73/core" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -127,7 +127,7 @@ func (d *DataTrack) AddSubscriber(sub types.LocalParticipant) (types.DataDownTra sub.ID(), sub.Kind(), sub.KindDetails(), - sub.GetTelemetryListener(), + sub.GetTelemetryListener, sub.GetReporter(), ) dataDownTrack, err := NewDataDownTrack(DataDownTrackParams{ diff --git a/pkg/rtc/datatrack/extension_participant_sid.go b/pkg/rtc/datatrack/extension_participant_sid.go deleted file mode 100644 index 35ada18f1..000000000 --- a/pkg/rtc/datatrack/extension_participant_sid.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package datatrack - -import ( - "errors" - - "github.com/livekit/protocol/livekit" -) - -type ExtensionParticipantSid struct { - participantID livekit.ParticipantID -} - -func NewExtensionParticipantSid(participantID livekit.ParticipantID) (*ExtensionParticipantSid, error) { - if len(participantID) >= 256 { - return nil, errors.New("participantID too long") - } - - return &ExtensionParticipantSid{participantID}, nil -} - -func (e *ExtensionParticipantSid) ParticipantID() livekit.ParticipantID { - return e.participantID -} - -func (e *ExtensionParticipantSid) Marshal() (Extension, error) { - data := make([]byte, len(e.participantID)) - copy(data, e.participantID) - return Extension{ - id: uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID), - data: data, - }, nil -} - -func (e *ExtensionParticipantSid) Unmarshal(ext Extension) error { - if ext.id != uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID) { - return errors.New("invalid extension ID") - } - - if len(ext.data) == 0 { - return errors.New("empty extension data") - } - - e.participantID = livekit.ParticipantID(ext.data) - return nil -} diff --git a/pkg/rtc/datatrack/extension_participant_sid_test.go b/pkg/rtc/datatrack/extension_participant_sid_test.go deleted file mode 100644 index 7128e8bdc..000000000 --- a/pkg/rtc/datatrack/extension_participant_sid_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package datatrack - -import ( - "testing" - - "github.com/livekit/protocol/livekit" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestExtensionParticipantSid(t *testing.T) { - longTestParticipantID := livekit.ParticipantID(make([]byte, 256)) - _, err := NewExtensionParticipantSid(longTestParticipantID) - require.Error(t, err) - - testParticipantID := livekit.ParticipantID("test") - extParticipantSid, err := NewExtensionParticipantSid(testParticipantID) - require.NoError(t, err) - - expectedExt := Extension{ - id: uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID), - data: []byte{'t', 'e', 's', 't'}, - } - ext, err := extParticipantSid.Marshal() - require.NoError(t, err) - require.Equal(t, expectedExt, ext) - - var unmarshaled ExtensionParticipantSid - err = unmarshaled.Unmarshal(ext) - require.NoError(t, err) - assert.Equal(t, testParticipantID, unmarshaled.ParticipantID()) -} diff --git a/pkg/rtc/datatrack/packet.go b/pkg/rtc/datatrack/packet.go deleted file mode 100644 index 42fe1eef2..000000000 --- a/pkg/rtc/datatrack/packet.go +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package datatrack - -import ( - "encoding/binary" - "errors" - "fmt" -) - -var ( - errHeaderSizeInsufficient = errors.New("data track packet header size insufficient") - errBufferSizeInsufficient = errors.New("data track packet buffer size insufficient") - errExtensionSizeInsufficient = errors.New("data track packet extension size insufficient") - errExtensionNotFound = errors.New("data track packet extension not found") - errExtensionSizeTooBig = errors.New("extension size is too big") -) - -const ( - headerLength = 12 - - versionShift = 5 - versionMask = (1 << 3) - 1 - - startOfFrameShift = 4 - startOfFrameMask = (1 << 1) - 1 - - finalOfFrameShift = 3 - finalOfFrameMask = (1 << 1) - 1 - - extensionsShift = 2 - extensionsMask = (1 << 1) - 1 - - handleOffset = 2 - handleLength = 2 - - seqNumOffset = 4 - seqNumLength = 2 - - frameNumOffset = 6 - frameNumLength = 2 - - timestampOffset = 8 - timestampLength = 4 - - extensionsSizeOffset = headerLength - extensionsSizeLength = 2 - - extensionIDLength = 1 - extensionSizeLength = 1 -) - -type Extension struct { - id uint8 - data []byte -} - -type Header struct { - Version uint8 - IsStartOfFrame bool - IsFinalOfFrame bool - HasExtensions bool - Handle uint16 - SequenceNumber uint16 - FrameNumber uint16 - Timestamp uint32 - ExtensionsSize uint16 - Extensions []Extension -} - -/* - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* 0 1 2 3 - ┆* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* |V |S|F|X| reserved | handle | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* | sequence number | frame number | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* | timestamp | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |* Extensions Size if X=1 | Extensions... | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - Each extension - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* 0 1 2 3 - ┆* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - ┆* | Extension ID | Extension size| Extension payload | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - End of all extensions - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |* padded to 4 byte boundary if aggregate of `Extensions Size` | - |* field and all extensions do not end on a 4 byte boundary | - ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -*/ - -func (h *Header) Unmarshal(buf []byte) (int, error) { - if len(buf) < headerLength { - return 0, fmt.Errorf("%w: %d < %d", errHeaderSizeInsufficient, len(buf), headerLength) - } - - hdrSize := headerLength - h.Version = buf[0] >> versionShift & versionMask - h.IsStartOfFrame = (buf[0] >> startOfFrameShift & startOfFrameMask) > 0 - h.IsFinalOfFrame = (buf[0] >> finalOfFrameShift & finalOfFrameMask) > 0 - h.HasExtensions = (buf[0] >> extensionsShift & extensionsMask) > 0 - - h.Handle = binary.BigEndian.Uint16(buf[handleOffset : handleOffset+handleLength]) - h.SequenceNumber = binary.BigEndian.Uint16(buf[seqNumOffset : seqNumOffset+seqNumLength]) - h.FrameNumber = binary.BigEndian.Uint16(buf[frameNumOffset : frameNumOffset+frameNumLength]) - h.Timestamp = binary.BigEndian.Uint32(buf[timestampOffset : timestampOffset+timestampLength]) - - if h.HasExtensions { - if len(buf) < extensionsSizeOffset+extensionsSizeLength { - return 0, fmt.Errorf("%w: %d < %d", errHeaderSizeInsufficient, len(buf), extensionsSizeOffset+extensionsSizeLength) - } - extensionsSize := (int(binary.BigEndian.Uint16(buf[extensionsSizeOffset:extensionsSizeOffset+extensionsSizeLength]))+1)*4 - extensionsSizeLength - hdrSize += extensionsSizeLength - - extensionHeaderSize := extensionIDLength + extensionSizeLength - remainingSize := extensionsSize - idx := extensionsSizeOffset + extensionsSizeLength - for remainingSize != 0 { - // read extension header - if len(buf[idx:]) < extensionIDLength || remainingSize < extensionIDLength { - return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), extensionIDLength) - } - id := buf[idx] - if id == 0 { - // end of extensions, padding has started - if len(buf[idx:]) < remainingSize { - return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), remainingSize) - } - hdrSize += remainingSize - break - } - - if len(buf[idx+1:]) < extensionSizeLength || remainingSize < extensionSizeLength { - return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), extensionSizeLength) - } - size := int(buf[idx+1]) - - remainingSize -= extensionHeaderSize - idx += extensionHeaderSize - hdrSize += extensionHeaderSize - - // read extension data - if len(buf[idx:]) < size || remainingSize < size { - return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), size) - } - h.Extensions = append(h.Extensions, Extension{id: id, data: buf[idx : idx+size]}) - - remainingSize -= size - idx += size - hdrSize += size - } - h.ExtensionsSize = uint16(extensionsSize - remainingSize) - } - - return hdrSize, nil -} - -func (h *Header) MarshalSize() int { - extensionsSize := 0 - if h.HasExtensions { - extensionsSize += extensionsSizeLength - for _, ext := range h.Extensions { - extensionsSize += len(ext.data) + extensionIDLength + extensionSizeLength - } - } - - return headerLength + (extensionsSize+3)/4*4 -} - -func (h *Header) MarshalTo(buf []byte) (int, error) { - if len(buf) < headerLength { - return 0, fmt.Errorf("%w: %d < %d", errHeaderSizeInsufficient, len(buf), headerLength) - } - - hdrSize := headerLength - buf[0] = h.Version << versionShift - if h.IsStartOfFrame { - buf[0] |= (1 << startOfFrameShift) - } - if h.IsFinalOfFrame { - buf[0] |= (1 << finalOfFrameShift) - } - if h.HasExtensions { - buf[0] |= (1 << extensionsShift) - } - - binary.BigEndian.PutUint16(buf[handleOffset:handleOffset+handleLength], h.Handle) - binary.BigEndian.PutUint16(buf[seqNumOffset:seqNumOffset+seqNumLength], h.SequenceNumber) - binary.BigEndian.PutUint16(buf[frameNumOffset:frameNumOffset+frameNumLength], h.FrameNumber) - binary.BigEndian.PutUint32(buf[timestampOffset:timestampOffset+timestampLength], h.Timestamp) - - if h.HasExtensions { - extensionsSize := (extensionsSizeLength + h.ExtensionsSize + 3) / 4 * 4 - binary.BigEndian.PutUint16(buf[extensionsSizeOffset:extensionsSizeOffset+extensionsSizeLength], (extensionsSize/4)-1) - hdrSize += extensionsSizeLength - - addedSize := 0 - idx := extensionsSizeOffset + extensionsSizeLength - for _, ext := range h.Extensions { - buf[idx] = ext.id - if len(ext.data) > 255 { - return 0, fmt.Errorf("%w: %d > 255", errExtensionSizeTooBig, len(ext.data)) - } - buf[idx+extensionIDLength] = byte(len(ext.data)) - copy(buf[idx+extensionIDLength+extensionSizeLength:], ext.data) - - extSize := len(ext.data) + extensionIDLength + extensionSizeLength - idx += extSize - hdrSize += extSize - addedSize += extSize - } - - paddingSize := extensionsSize - extensionsSizeLength - uint16(addedSize) - for i := range paddingSize { - buf[idx+int(i)] = 0 - } - idx += int(paddingSize) - hdrSize += int(paddingSize) - } - - return hdrSize, nil -} - -func (h *Header) AddExtension(ext Extension) { - for i, existingExt := range h.Extensions { - if existingExt.id == ext.id { - h.ExtensionsSize -= uint16(len(existingExt.data) + extensionIDLength + extensionSizeLength) - h.Extensions[i].data = ext.data - h.ExtensionsSize += uint16(len(h.Extensions[i].data) + extensionIDLength + extensionSizeLength) - return - } - } - - h.Extensions = append(h.Extensions, ext) - h.ExtensionsSize += uint16(len(ext.data) + extensionIDLength + extensionSizeLength) - h.HasExtensions = true -} - -func (h *Header) GetExtension(id uint8) (Extension, error) { - for _, ext := range h.Extensions { - if ext.id == id { - return ext, nil - } - } - return Extension{}, fmt.Errorf("%w, id: %d", errExtensionNotFound, id) -} - -// ---------------------------------------------------- - -type Packet struct { - Header - Payload []byte -} - -func (p *Packet) Unmarshal(buf []byte) error { - hdrSize, err := p.Header.Unmarshal(buf) - if err != nil { - return err - } - if hdrSize > len(buf) { - return fmt.Errorf("%w: %d < %d", errBufferSizeInsufficient, len(buf), hdrSize) - } - - p.Payload = buf[hdrSize:] - return nil -} - -func (p *Packet) Marshal() ([]byte, error) { - buf := make([]byte, p.Header.MarshalSize()+len(p.Payload)) - if err := p.MarshalTo(buf); err != nil { - return nil, err - } - - return buf, nil -} - -func (p *Packet) MarshalTo(buf []byte) error { - size := p.Header.MarshalSize() + len(p.Payload) - if len(buf) < size { - return fmt.Errorf("%w: %d < %d", errBufferSizeInsufficient, len(buf), size) - } - - hdrSize, err := p.Header.MarshalTo(buf) - if err != nil { - return err - } - - copy(buf[hdrSize:], p.Payload) - return nil -} diff --git a/pkg/rtc/datatrack/packet_test.go b/pkg/rtc/datatrack/packet_test.go deleted file mode 100644 index 8f1bc672b..000000000 --- a/pkg/rtc/datatrack/packet_test.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package datatrack - -import ( - "testing" - - "github.com/livekit/protocol/livekit" - "github.com/stretchr/testify/require" -) - -func TestPacket(t *testing.T) { - t.Run("without extension", func(t *testing.T) { - payload := make([]byte, 6) - for i := range len(payload) { - payload[i] = byte(255 - i) - } - packet := &Packet{ - Header: Header{ - Version: 0, - IsStartOfFrame: true, - IsFinalOfFrame: true, - Handle: 3333, - SequenceNumber: 6666, - FrameNumber: 9999, - Timestamp: 0xdeadbeef, - }, - Payload: payload, - } - rawPacket, err := packet.Marshal() - require.NoError(t, err) - - expectedRawPacket := []byte{ - 0x18, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0xff, 0xfe, 0xfd, 0xfc, - 0xfb, 0xfa, - } - require.Equal(t, expectedRawPacket, rawPacket) - - var unmarshaled Packet - err = unmarshaled.Unmarshal(rawPacket) - require.NoError(t, err) - require.Equal(t, packet, &unmarshaled) - }) - - t.Run("with extension", func(t *testing.T) { - payload := make([]byte, 4) - for i := range len(payload) { - payload[i] = byte(255 - i) - } - packet := &Packet{ - Header: Header{ - Version: 0, - IsStartOfFrame: true, - IsFinalOfFrame: false, - Handle: 3333, - SequenceNumber: 6666, - FrameNumber: 9999, - Timestamp: 0xdeadbeef, - }, - Payload: payload, - } - if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { - if ext, err := extParticipantSid.Marshal(); err == nil { - packet.AddExtension(ext) - } - } - rawPacket, err := packet.Marshal() - require.NoError(t, err) - - expectedRawPacket := []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x04, 0x01, 0x10, - 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, - 0xff, 0xfe, 0xfd, 0xfc, - } - require.Equal(t, expectedRawPacket, rawPacket) - - var unmarshaled Packet - err = unmarshaled.Unmarshal(rawPacket) - require.NoError(t, err) - require.Equal(t, packet, &unmarshaled) - - ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) - require.NoError(t, err) - - var extParticipantSid ExtensionParticipantSid - require.NoError(t, extParticipantSid.Unmarshal(ext)) - require.Equal(t, livekit.ParticipantID("test_participant"), extParticipantSid.ParticipantID()) - }) - - t.Run("with extension padding", func(t *testing.T) { - payload := make([]byte, 4) - for i := range len(payload) { - payload[i] = byte(255 - i) - } - packet := &Packet{ - Header: Header{ - Version: 0, - IsStartOfFrame: true, - IsFinalOfFrame: false, - Handle: 3333, - SequenceNumber: 6666, - FrameNumber: 9999, - Timestamp: 0xdeadbeef, - }, - Payload: payload, - } - if extParticipantSid, err := NewExtensionParticipantSid("participant"); err == nil { - if ext, err := extParticipantSid.Marshal(); err == nil { - packet.AddExtension(ext) - } - } - rawPacket, err := packet.Marshal() - require.NoError(t, err) - - expectedRawPacket := []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0b, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - require.Equal(t, expectedRawPacket, rawPacket) - - var unmarshaled Packet - err = unmarshaled.Unmarshal(rawPacket) - require.NoError(t, err) - require.Equal(t, packet, &unmarshaled) - - ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) - require.NoError(t, err) - - var extParticipantSid ExtensionParticipantSid - require.NoError(t, extParticipantSid.Unmarshal(ext)) - require.Equal(t, livekit.ParticipantID("participant"), extParticipantSid.ParticipantID()) - }) - - t.Run("replace extension", func(t *testing.T) { - payload := make([]byte, 4) - for i := range len(payload) { - payload[i] = byte(255 - i) - } - packet := &Packet{ - Header: Header{ - Version: 0, - IsStartOfFrame: true, - IsFinalOfFrame: false, - Handle: 3333, - SequenceNumber: 6666, - FrameNumber: 9999, - Timestamp: 0xdeadbeef, - }, - Payload: payload, - } - if extParticipantSid, err := NewExtensionParticipantSid("participant"); err == nil { - if ext, err := extParticipantSid.Marshal(); err == nil { - packet.AddExtension(ext) - } - } - rawPacket, err := packet.Marshal() - require.NoError(t, err) - - expectedRawPacket := []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0b, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - require.Equal(t, expectedRawPacket, rawPacket) - - // replace existing extension ID and ensure that marshalled packet is updated - if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { - if ext, err := extParticipantSid.Marshal(); err == nil { - packet.AddExtension(ext) - } - } - rawPacket, err = packet.Marshal() - require.NoError(t, err) - - expectedRawPacket = []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x04, 0x01, 0x10, - 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, - 0xff, 0xfe, 0xfd, 0xfc, - } - require.Equal(t, expectedRawPacket, rawPacket) - - var unmarshaled Packet - err = unmarshaled.Unmarshal(rawPacket) - require.NoError(t, err) - require.Equal(t, packet, &unmarshaled) - - ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) - require.NoError(t, err) - - var extParticipantSid ExtensionParticipantSid - require.NoError(t, extParticipantSid.Unmarshal(ext)) - require.Equal(t, livekit.ParticipantID("test_participant"), extParticipantSid.ParticipantID()) - }) - - t.Run("bad packet", func(t *testing.T) { - var unmarshaled Packet - // extensions size too small - badPacket := []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x02, 0x01, 0x0b, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - err := unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - - // get an invalid extension id - badPacket = []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x02, 0x0b, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - err = unmarshaled.Unmarshal(badPacket) - require.NoError(t, err) - _, err = unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) - require.Error(t, err) - - // extension payload size bigger than payload - badPacket = []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0d, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - err = unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - - // extension payload size smaller than payload - badPacket = []byte{ - 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, - 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x07, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, - 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, - } - err = unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - }) - - t.Run("oversized extension padding does not panic", func(t *testing.T) { - var unmarshaled Packet - // HasExtensions set, extensionsSize describes more bytes than present, - // terminated by a 0x00 padding id -> hdrSize would exceed len(buf) - badPacket := []byte{ - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - } - err := unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - }) - - t.Run("extensions size wraparound does not panic", func(t *testing.T) { - var unmarshaled Packet - // 0xFFFF extensions-size field wraps (raw+1)*4 uint16 arithmetic to a huge - // remainingSize; the 0x00 padding id must not push hdrSize past len(buf) - badPacket := []byte{ - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, - } - err := unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - }) - - t.Run("truncated extensions size field does not panic", func(t *testing.T) { - var unmarshaled Packet - // HasExtensions set but buffer too short to hold the extensionsSize field - badPacket := []byte{ - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, - } - err := unmarshaled.Unmarshal(badPacket) - require.Error(t, err) - }) -} diff --git a/pkg/rtc/datatrack/testutils.go b/pkg/rtc/datatrack/testutils.go deleted file mode 100644 index e7df0bbd9..000000000 --- a/pkg/rtc/datatrack/testutils.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package datatrack - -import ( - "math/rand" - "time" -) - -func GenerateRawDataPackets(handle uint16, seqNum uint16, frameNum uint16, numFrames int, frameSize int, frameDuration time.Duration) [][]byte { - if seqNum == 0 { - seqNum = uint16(rand.Intn(256) + 1) - } - if frameNum == 0 { - frameNum = uint16(rand.Intn(256) + 1) - } - timestamp := uint32(rand.Intn(1024)) - - packetsPerFrame := (frameSize + 255) / 256 // using 256 bytes of payload per packet - if packetsPerFrame == 0 { - return nil - } - numPackets := packetsPerFrame * numFrames - rawPackets := make([][]byte, 0, numPackets) - for range numFrames { - remainingSize := frameSize - for packetIdx := range packetsPerFrame { - payloadSize := min(remainingSize, 256) - payload := make([]byte, payloadSize) - for i := range len(payload) { - payload[i] = byte(255 - i) - } - packet := &Packet{ - Header: Header{ - Version: 0, - IsStartOfFrame: packetIdx == 0, - IsFinalOfFrame: packetIdx == packetsPerFrame-1, - Handle: handle, - SequenceNumber: seqNum, - FrameNumber: frameNum, - Timestamp: timestamp, - }, - Payload: payload, - } - if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { - if ext, err := extParticipantSid.Marshal(); err == nil { - packet.AddExtension(ext) - } - } - rawPacket, err := packet.Marshal() - if err == nil { - rawPackets = append(rawPackets, rawPacket) - } - seqNum++ - remainingSize -= payloadSize - } - frameNum++ - timestamp += uint32(90000 * frameDuration.Seconds()) - } - - return rawPackets -} diff --git a/pkg/rtc/datatrack_stats.go b/pkg/rtc/datatrack_stats.go index 5bde6f57f..37ae15640 100644 --- a/pkg/rtc/datatrack_stats.go +++ b/pkg/rtc/datatrack_stats.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" ) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index d0fc9a9a4..3c167e0f2 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -46,7 +46,7 @@ var _ types.LocalMediaTrack = (*MediaTrack)(nil) // Implements MediaTrack and PublishedTrack interface type MediaTrack struct { params MediaTrackParams - buffer *buffer.Buffer + buffer atomic.Pointer[buffer.Buffer] everSubscribed atomic.Bool *MediaTrackReceiver @@ -54,26 +54,20 @@ type MediaTrack struct { dynacastManager dynacast.DynacastManager - lock sync.RWMutex - migrated bool - published bool + // serialises receiver creation in AddReceiver, i. e. the check-then-create + // of a receiver for a mime type. Held across receiver set up + addReceiverLock sync.Mutex + + regressionTargetCodecReceived atomic.Bool + + migrated atomic.Bool + published atomic.Bool rttFromXR atomic.Bool - backupCodecPolicy livekit.BackupCodecPolicy - regressionTargetCodec mime.MimeType - regressionTargetCodecReceived bool - - onSubscribedMaxQualityChange func( - trackID livekit.TrackID, - trackInfo *livekit.TrackInfo, - subscribedQualities []*livekit.SubscribedCodec, - maxSubscribedQualities []types.SubscribedCodecQuality, - ) error - onSubscribedAudioCodecChange func( - trackID livekit.TrackID, - codecs []*livekit.SubscribedAudioCodec, - ) error + backupCodecPolicy livekit.BackupCodecPolicy + // immutable after construction + regressionTargetCodec mime.MimeType } type MediaTrackParams struct { @@ -89,7 +83,7 @@ type MediaTrackParams struct { PLIThrottleConfig sfu.PLIThrottleConfig AudioConfig sfu.AudioConfig VideoConfig config.VideoConfig - TelemetryListener types.ParticipantTelemetryListener + TelemetryListener func() types.ParticipantTelemetryListener Logger logger.Logger Reporter roomobs.TrackReporter SimTracks map[uint32]interceptor.SimulcastTrackInfo @@ -101,6 +95,16 @@ type MediaTrackParams struct { EnableRTPStreamRestartDetection bool UpdateTrackInfoByVideoSizeChange bool ForceBackupCodecPolicySimulcast bool + OnSubscribedMaxQualityChange func( + trackID livekit.TrackID, + trackInfo *livekit.TrackInfo, + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, + ) error + OnSubscribedAudioCodecChange func( + trackID livekit.TrackID, + codecs []*livekit.SubscribedAudioCodec, + ) error } func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { @@ -138,8 +142,8 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { Logger: params.Logger, }) t.MediaLossProxy.OnMediaLossUpdate(func(fractionalLoss uint8) { - if t.buffer != nil { - t.buffer.SetLastFractionLostReport(fractionalLoss) + if buff := t.buffer.Load(); buff != nil { + buff.SetLastFractionLostReport(fractionalLoss) } }) t.MediaTrackReceiver.OnMediaLossFeedback(t.MediaLossProxy.HandleMaxLossFeedback) @@ -200,30 +204,6 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { return t } -func (t *MediaTrack) OnSubscribedMaxQualityChange( - f func( - trackID livekit.TrackID, - trackInfo *livekit.TrackInfo, - subscribedQualities []*livekit.SubscribedCodec, - maxSubscribedQualities []types.SubscribedCodecQuality, - ) error, -) { - t.lock.Lock() - t.onSubscribedMaxQualityChange = f - t.lock.Unlock() -} - -func (t *MediaTrack) OnSubscribedAudioCodecChange( - f func( - trackID livekit.TrackID, - codecs []*livekit.SubscribedAudioCodec, - ) error, -) { - t.lock.Lock() - t.onSubscribedAudioCodecChange = f - t.lock.Unlock() -} - func (t *MediaTrack) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) { if t.dynacastManager != nil { t.dynacastManager.NotifySubscriberNodeMaxQuality(nodeID, qualities) @@ -348,8 +328,11 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe }) ti := t.MediaTrackReceiver.TrackInfoClone() - t.lock.Lock() + var regressCodec bool + enableRegression := t.enableRegression() + + t.addReceiverLock.Lock() mimeType := mime.NormalizeMimeType(track.Codec().MimeType) layer := buffer.GetSpatialLayerForRid(mimeType, track.RID(), ti) if layer < 0 { @@ -361,7 +344,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe "codec", track.Codec(), "trackInfo", logger.Proto(ti), ) - t.lock.Unlock() + t.addReceiverLock.Unlock() return newCodec, false } @@ -405,7 +388,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe "mime", mimeType, "track", logger.Proto(ti), ) - t.lock.Unlock() + t.addReceiverLock.Unlock() return newCodec, false } @@ -448,11 +431,8 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { // send for only one codec, either primary (priority == 0) OR regressed codec - t.lock.RLock() - regressionTargetCodecReceived := t.regressionTargetCodecReceived - t.lock.RUnlock() - if priority == 0 || regressionTargetCodecReceived { - t.params.TelemetryListener.OnTrackStats(statsKey, stat) + if priority == 0 || t.regressionTargetCodecReceived.Load() { + t.params.TelemetryListener().OnTrackStats(statsKey, stat) if cs, ok := telemetry.CondenseStat(stat); ok { t.params.Reporter.Tx(func(tx roomobs.TrackTx) { @@ -477,10 +457,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe newWR.OnMaxLayerChange(func(mimeType mime.MimeType, maxLayer int32) { // send for only one codec, either primary (priority == 0) OR regressed codec - t.lock.RLock() - regressionTargetCodecReceived := t.regressionTargetCodecReceived - t.lock.RUnlock() - if priority == 0 || regressionTargetCodecReceived { + if priority == 0 || t.regressionTargetCodecReceived.Load() { t.MediaTrackReceiver.NotifyMaxLayerChange(mimeType, maxLayer) } }) @@ -505,7 +482,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } } - t.buffer = buff + t.buffer.Store(buff) t.MediaTrackReceiver.SetupReceiver(newWR, priority, mid) @@ -531,16 +508,16 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe }) } - if newCodec && t.enableRegression() { + if newCodec && enableRegression { if mimeType == t.regressionTargetCodec { t.params.Logger.Infow("regression target codec received", "codec", mimeType) - t.regressionTargetCodecReceived = true + t.regressionTargetCodecReceived.Store(true) regressCodec = true - } else if t.regressionTargetCodecReceived { + } else if t.regressionTargetCodecReceived.Load() { regressCodec = true } } - t.lock.Unlock() + t.addReceiverLock.Unlock() if err := wr.(*sfu.WebRTCReceiver).AddUpTrack(track, buff); err != nil { t.params.Logger.Warnw( @@ -597,7 +574,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe }) buff.OnFinalRtpStats(func(stats *livekit.RTPStats) { - t.params.TelemetryListener.OnTrackPublishRTPStats( + t.params.TelemetryListener().OnTrackPublishRTPStats( t.params.ParticipantID(), t.ID(), mimeType, @@ -678,12 +655,8 @@ func (t *MediaTrack) OnDynacastSubscribedMaxQualityChange( subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality, ) { - t.lock.RLock() - onSubscribedMaxQualityChange := t.onSubscribedMaxQualityChange - t.lock.RUnlock() - - if onSubscribedMaxQualityChange != nil && !t.IsMuted() { - _ = onSubscribedMaxQualityChange( + if t.params.OnSubscribedMaxQualityChange != nil && !t.IsMuted() { + _ = t.params.OnSubscribedMaxQualityChange( t.ID(), t.ToProto(), subscribedQualities, @@ -706,35 +679,23 @@ func (t *MediaTrack) OnDynacastSubscribedMaxQualityChange( } func (t *MediaTrack) OnDynacastSubscribedAudioCodecChange(codecs []*livekit.SubscribedAudioCodec) { - t.lock.RLock() - onSubscribedAudioCodecChange := t.onSubscribedAudioCodecChange - t.lock.RUnlock() - - if onSubscribedAudioCodecChange != nil { - _ = onSubscribedAudioCodecChange(t.ID(), codecs) + if t.params.OnSubscribedAudioCodecChange != nil { + _ = t.params.OnSubscribedAudioCodecChange(t.ID(), codecs) } } func (t *MediaTrack) SetMigrated(migrated bool) { - t.lock.Lock() - t.migrated = migrated - t.lock.Unlock() + t.migrated.Store(migrated) } func (t *MediaTrack) Migrated() bool { - t.lock.RLock() - defer t.lock.RUnlock() - return t.migrated + return t.migrated.Load() } func (t *MediaTrack) SetPublished(published bool) { - t.lock.Lock() - t.published = published - t.lock.Unlock() + t.published.Store(published) } func (t *MediaTrack) Published() bool { - t.lock.RLock() - defer t.lock.RUnlock() - return t.published + return t.published.Load() } diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index cffde83b5..c6130e352 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -125,7 +125,7 @@ type MediaTrackReceiverParams struct { ReceiverConfig ReceiverConfig SubscriberConfig DirectionConfig AudioConfig sfu.AudioConfig - TelemetryListener types.ParticipantTelemetryListener + TelemetryListener func() types.ParticipantTelemetryListener Logger logger.Logger RegressionTargetCodec mime.MimeType PreferVideoSizeFromMedia bool @@ -344,7 +344,7 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete } if !exist { receivers = append(receivers, &simulcastReceiver{ - TrackReceiver: NewDummyReceiver(t.ID(), string(t.PublisherID()), c, headers), + TrackReceiver: NewDummyReceiver(t.TrackInfo(), string(t.PublisherID()), c, headers), priority: i, }) } @@ -949,7 +949,7 @@ func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTr t.updateTrackInfoOfReceivers() - t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) + t.params.TelemetryListener().OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) t.params.Logger.Debugw("updated audio track", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) } @@ -973,7 +973,7 @@ func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTr t.updateTrackInfoOfReceivers() - t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) + t.params.TelemetryListener().OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) t.params.Logger.Debugw("updated video track", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) } @@ -1019,7 +1019,7 @@ func (t *MediaTrackReceiver) UpdateVideoSize(mimeType mime.MimeType, sizes []cod t.updateTrackInfoOfReceivers() - t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) + t.params.TelemetryListener().OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) t.params.Logger.Debugw("updated video sizes", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) } @@ -1050,7 +1050,7 @@ func (t *MediaTrackReceiver) NotifyMaxLayerChange(mimeType mime.MimeType, maxLay } } - t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), ti) + t.params.TelemetryListener().OnTrackPublishedUpdate(t.PublisherID(), ti) } // GetQualityForDimension finds the closest quality to use for desired dimensions diff --git a/pkg/rtc/migrationdatacache.go b/pkg/rtc/migrationdatacache.go index 8c44eac61..45a689b7f 100644 --- a/pkg/rtc/migrationdatacache.go +++ b/pkg/rtc/migrationdatacache.go @@ -14,9 +14,14 @@ const ( MigrationDataCacheStateDone ) +const ( + migrationDataCacheMaxSize = 4 << 20 // 4 MiB +) + type MigrationDataCache struct { lastSeq uint32 pkts []*livekit.DataPacket + size int state MigrationDataCacheState expiredAt time.Time } @@ -30,10 +35,10 @@ func NewMigrationDataCache(lastSeq uint32, expiredAt time.Time) *MigrationDataCa // Add adds a message to the cache if there is a gap between the last sequence number and cached messages then return the cache State: // - MigrationDataCacheStateWaiting: waiting for the next packet (lastSeq + 1) of last sequence from old node -// - MigrationDataCacheStateTimeout: the next packet is not received before the expiredAt, participant will -// continue to process the reliable messages, subscribers will see the gap after the publisher migration +// - MigrationDataCacheStateTimeout: the next packet is not received before the expiredAt or the cache is full, participant +// will continue to process the reliable messages, subscribers will see the gap after the publisher migration // - MigrationDataCacheStateDone: the next packet is received, participant can continue to process the reliable messages -func (c *MigrationDataCache) Add(pkt *livekit.DataPacket) MigrationDataCacheState { +func (c *MigrationDataCache) Add(pkt *livekit.DataPacket, size int) MigrationDataCacheState { if c.state == MigrationDataCacheStateDone || c.state == MigrationDataCacheStateTimeout { return c.state } @@ -48,7 +53,8 @@ func (c *MigrationDataCache) Add(pkt *livekit.DataPacket) MigrationDataCacheStat } c.pkts = append(c.pkts, pkt) - if time.Now().After(c.expiredAt) { + c.size += size + if c.size >= migrationDataCacheMaxSize || time.Now().After(c.expiredAt) { c.state = MigrationDataCacheStateTimeout } return c.state @@ -57,3 +63,7 @@ func (c *MigrationDataCache) Add(pkt *livekit.DataPacket) MigrationDataCacheStat func (c *MigrationDataCache) Get() []*livekit.DataPacket { return c.pkts } + +func (c *MigrationDataCache) Size() int { + return c.size +} diff --git a/pkg/rtc/migrationdatacache_test.go b/pkg/rtc/migrationdatacache_test.go index 046b2ddfc..a753c75f9 100644 --- a/pkg/rtc/migrationdatacache_test.go +++ b/pkg/rtc/migrationdatacache_test.go @@ -14,25 +14,44 @@ func TestMigrationDataCache_Add(t *testing.T) { cache := NewMigrationDataCache(10, expiredAt) pkt1 := &livekit.DataPacket{Sequence: 9} - state := cache.Add(pkt1) + state := cache.Add(pkt1, 0) require.Equal(t, MigrationDataCacheStateWaiting, state) require.Empty(t, cache.Get()) pkt2 := &livekit.DataPacket{Sequence: 11} - state = cache.Add(pkt2) + state = cache.Add(pkt2, 0) require.Equal(t, MigrationDataCacheStateDone, state) require.Empty(t, cache.Get()) pkt3 := &livekit.DataPacket{Sequence: 12} - state = cache.Add(pkt3) + state = cache.Add(pkt3, 0) require.Equal(t, MigrationDataCacheStateDone, state) require.Empty(t, cache.Get()) cache2 := NewMigrationDataCache(20, time.Now().Add(10*time.Millisecond)) pkt4 := &livekit.DataPacket{Sequence: 22} time.Sleep(20 * time.Millisecond) - state = cache2.Add(pkt4) + state = cache2.Add(pkt4, 0) require.Equal(t, MigrationDataCacheStateTimeout, state) require.Len(t, cache2.Get(), 1) require.Equal(t, uint32(22), cache2.Get()[0].Sequence) } + +func TestMigrationDataCache_MaxSize(t *testing.T) { + // the cache should not grow past the size budget even if the expiry is far in the future + cache := NewMigrationDataCache(10, time.Now().Add(time.Minute)) + + pktSize := 1000 + seq := uint32(12) + state := MigrationDataCacheStateWaiting + for ; state == MigrationDataCacheStateWaiting; seq++ { + state = cache.Add(&livekit.DataPacket{Sequence: seq}, pktSize) + } + + require.Equal(t, MigrationDataCacheStateTimeout, state) + require.LessOrEqual(t, cache.Size(), migrationDataCacheMaxSize+pktSize) + require.Len(t, cache.Get(), migrationDataCacheMaxSize/pktSize+1) + + // once full, further packets are dropped, including the continuous one + require.Equal(t, MigrationDataCacheStateTimeout, cache.Add(&livekit.DataPacket{Sequence: 11}, pktSize)) +} diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index f35228f33..01b14479d 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -81,6 +81,10 @@ const ( cMaxPendingTracks = 20 cMaxPendingQueuedTracks = 3 + // unsequenced reliable data (server API sends) cannot be recovered from the + // data message cache, so it is held here until the reliable data channel is writable + cMaxJoiningUnsequencedReliableBytes = 100_000 + PingIntervalSeconds = 5 PingTimeoutSeconds = 15 ) @@ -150,6 +154,9 @@ type reliableDataInfo struct { joiningMessageLock sync.Mutex joiningMessageFirstSeqs map[livekit.ParticipantID]uint32 joiningMessageLastWrittenSeqs map[livekit.ParticipantID]uint32 + joiningUnsequencedMessages [][]byte + joiningUnsequencedBytes int + joiningUnsequencedDropped int lastPubReliableSeq atomic.Uint32 stopReliableByMigrateOut atomic.Bool canWriteReliable bool @@ -244,6 +251,7 @@ type ParticipantImpl struct { params ParticipantParams participantListener atomic.Pointer[types.LocalParticipantListener] + telemetryListener atomic.Pointer[types.ParticipantTelemetryListener] participantHelper atomic.Value // types.LocalParticipantHelper id atomic.Value // types.ParticipantID @@ -253,6 +261,10 @@ type ParticipantImpl struct { state atomic.Value // livekit.ParticipantInfo_State disconnected chan struct{} + // a migrating in participant resumes on a reconnect response, the client takes it + // only as the first message on the resumed signal connection + reconnectResponseSent atomic.Bool + grants atomic.Pointer[auth.ClaimGrants] isPublisher atomic.Bool @@ -391,17 +403,8 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p.setupSignalling() p.id.Store(params.SID) - p.dataChannelStats = NewBytesTrackStats( - p.params.Country, - BytesTrackIDForParticipantID(BytesTrackTypeData, p.ID()), - p.ID(), - params.Grants.GetParticipantKind(), - params.Grants.GetKindDetails(), - params.TelemetryListener, - params.Reporter, - ) - p.reliableDataInfo.lastPubReliableSeq.Store(params.LastPubReliableSeq) p.setListener(params.ParticipantListener) + p.setTelemetryListener(params.TelemetryListener) p.participantHelper.Store(params.ParticipantHelper) if !params.DisableSupervisor { p.supervisor = supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}) @@ -410,6 +413,17 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p.version.Store(params.InitialVersion) p.timedVersion.Update(params.VersionGenerator.Next()) + p.dataChannelStats = NewBytesTrackStats( + p.params.Country, + BytesTrackIDForParticipantID(BytesTrackTypeData, p.ID()), + p.ID(), + params.Grants.GetParticipantKind(), + params.Grants.GetKindDetails(), + p.GetTelemetryListener, + params.Reporter, + ) + p.reliableDataInfo.lastPubReliableSeq.Store(params.LastPubReliableSeq) + p.migrateState.Store(types.MigrateStateInit) p.state.Store(livekit.ParticipantInfo_JOINING) @@ -493,6 +507,25 @@ func (p *ParticipantImpl) ClearParticipantListener() { p.setListener(nil) } +func (p *ParticipantImpl) setTelemetryListener(listener types.ParticipantTelemetryListener) { + if listener == nil { + p.telemetryListener.Store(nil) + return + } + p.telemetryListener.Store(&listener) +} + +func (p *ParticipantImpl) GetTelemetryListener() types.ParticipantTelemetryListener { + if l := p.telemetryListener.Load(); l != nil { + return *l + } + return &types.NullParticipantTelemetryListener{} +} + +func (p *ParticipantImpl) ClearTelemetryListener() { + p.setTelemetryListener(nil) +} + func (p *ParticipantImpl) GetCountry() string { return p.params.Country } @@ -583,9 +616,10 @@ func (p *ParticipantImpl) IsReady() bool { state := p.State() // when migrating, there is no JoinResponse, state transitions from JOINING -> ACTIVE -> DISCONNECTED - // so JOINING is considered ready. + // so JOINING is considered ready. The ReconnectResponse takes the place of the JoinResponse + // as the message the resumed signal connection opens with, so readiness waits for it. if p.params.Migration { - return state != livekit.ParticipantInfo_DISCONNECTED + return state != livekit.ParticipantInfo_DISCONNECTED && p.reconnectResponseSent.Load() } // when not migrating, there is a JoinResponse, state transitions from JOINING -> JOINED -> ACTIVE -> DISCONNECTED @@ -979,14 +1013,6 @@ func (p *ParticipantImpl) TelemetryGuard() *telemetry.ReferenceGuard { return p.telemetryGuard } -func (p *ParticipantImpl) GetTelemetryListener() types.ParticipantTelemetryListener { - if p.params.TelemetryListener == nil { - return &types.NullParticipantTelemetryListener{} - } - - return p.params.TelemetryListener -} - func (p *ParticipantImpl) AddOnClose(key string, callback func(types.LocalParticipant)) { if p.isClosed.Load() { if callback != nil { @@ -1013,7 +1039,7 @@ func (p *ParticipantImpl) OnClaimsChanged(callback func(types.LocalParticipant)) func (p *ParticipantImpl) HandleSignalSourceClose() { p.TransportManager.SetSignalSourceValid(false) - if !p.HasConnected() { + if !p.HasICEConnected() { _ = p.Close(false, types.ParticipantCloseReasonSignalSourceClose, false) } } @@ -1402,7 +1428,7 @@ func (p *ParticipantImpl) SetMigrateInfo( // for migrating in tracks, there is no AddTrack, so record a synthetic publish request for _, t := range mediaTracks { - p.params.TelemetryListener.OnTrackPublishRequested(p.ID(), p.Identity(), t.GetTrack(), false) + p.GetTelemetryListener().OnTrackPublishRequested(p.ID(), p.Identity(), t.GetTrack(), false) } for _, t := range dataTracks { @@ -1418,7 +1444,7 @@ func (p *ParticipantImpl) SetMigrateInfo( p.ID(), p.Kind(), p.KindDetails(), - p.params.TelemetryListener, + p.GetTelemetryListener, p.params.Reporter, ), }, @@ -1449,7 +1475,7 @@ func (p *ParticipantImpl) IsMigration() bool { } func (p *ParticipantImpl) recordRTCState(closeReason types.ParticipantCloseReason) { - if p.HasConnected() { + if p.HasICEConnected() { return } @@ -1461,7 +1487,8 @@ func (p *ParticipantImpl) recordRTCState(closeReason types.ParticipantCloseReaso } func (p *ParticipantImpl) IsConnectionCanceled(closeReason types.ParticipantCloseReason) bool { - return closeReason == types.ParticipantCloseReasonClientRequestLeave || + return closeReason == types.ParticipantCloseReasonJoinFailed || + closeReason == types.ParticipantCloseReasonClientRequestLeave || closeReason == types.ParticipantCloseReasonDuplicateIdentity || closeReason == types.ParticipantCloseReasonRoomClosed || closeReason == types.ParticipantCloseReasonMigrationRequested || @@ -2049,8 +2076,9 @@ func (p *ParticipantImpl) setupSignalling() { Participant: p, }) p.signaller = signalling.NewSignallerAsync(signalling.SignallerAsyncParams{ - Logger: p.params.Logger, - Participant: p, + Logger: p.params.Logger, + Participant: p, + OnHandshakeOpened: p.flushQueuedUpdates, }) } @@ -2169,7 +2197,6 @@ func (p *ParticipantImpl) setupSubscriptionManager() { DataTrackResolver: func(lp types.LocalParticipant, ti livekit.TrackID) types.DataResolverResult { return p.helper().ResolveDataTrack(lp, ti) }, - TelemetryListener: p.params.TelemetryListener, OnTrackSubscribed: p.onTrackSubscribed, OnTrackUnsubscribed: p.onTrackUnsubscribed, OnSubscriptionError: p.onSubscriptionError, @@ -2177,6 +2204,9 @@ func (p *ParticipantImpl) setupSubscriptionManager() { SubscriptionLimitAudio: p.params.SubscriptionLimitAudio, UseOneShotSignallingMode: p.params.UseOneShotSignallingMode, }) + p.SubscriptionManager.OnSubscribeStatusChanged(func(publisherID livekit.ParticipantID, subscribed bool) { + p.listener().OnSubscribeStatusChanged(p, publisherID, subscribed) + }) } func (p *ParticipantImpl) MetricsCollectorTimeToCollectMetrics() { @@ -2366,6 +2396,7 @@ func (p *ParticipantImpl) onMediaTrack(rtcTrack *webrtc.TrackRemote, rtpReceiver "ssrc", track.SSRC(), "rtxSsrc", track.RtxSSRC(), "mime", mime.NormalizeMimeType(codec.MimeType), + "isNewTrack", isNewTrack, "isReceiverAdded", isReceiverAdded, "sdpRids", logger.StringSlice(sdpRids[:]), ) @@ -2440,7 +2471,7 @@ func (p *ParticipantImpl) onReceivedDataMessage(kind livekit.DataPacket_Kind, da } if migrationCache := p.reliableDataInfo.migrateInPubDataCache.Load(); migrationCache != nil { - switch migrationCache.Add(dp) { + switch migrationCache.Add(dp, len(data)) { case MigrationDataCacheStateWaiting: // waiting for the reliable sequence to continue from last node return @@ -2459,6 +2490,8 @@ func (p *ParticipantImpl) onReceivedDataMessage(kind livekit.DataPacket_Kind, da "migration data cache timed out, handling cached messages", nil, "cachedFirstSeq", cachedMsgs[0].Sequence, "cachedLastSeq", cachedMsgs[len(cachedMsgs)-1].Sequence, + "cachedNum", len(cachedMsgs), + "cachedSize", migrationCache.Size(), "lastPubReliableSeq", p.params.LastPubReliableSeq, ) } @@ -2854,7 +2887,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange( break } } - p.params.TelemetryListener.OnTrackMaxSubscribedVideoQuality( + p.GetTelemetryListener().OnTrackMaxSubscribedVideoQuality( p.ID(), ti, maxSubscribedQuality.CodecMime, @@ -3157,7 +3190,7 @@ func (p *ParticipantImpl) addPendingTrack(req *livekit.AddTrackRequest) *livekit } p.pendingTracksLock.Unlock() - p.params.TelemetryListener.OnTrackPublishRequested(p.ID(), p.Identity(), utils.CloneProto(ti), true) + p.GetTelemetryListener().OnTrackPublishRequested(p.ID(), p.Identity(), utils.CloneProto(ti), true) return nil } @@ -3182,7 +3215,7 @@ func (p *ParticipantImpl) addPendingTrack(req *livekit.AddTrackRequest) *livekit } p.pendingTracksLock.Unlock() - p.params.TelemetryListener.OnTrackPublishRequested(p.ID(), p.Identity(), utils.CloneProto(ti), true) + p.GetTelemetryListener().OnTrackPublishRequested(p.ID(), p.Identity(), utils.CloneProto(ti), true) return ti } @@ -3199,6 +3232,10 @@ func (p *ParticipantImpl) GetPendingTrack(trackID livekit.TrackID) *livekit.Trac return nil } +func (p *ParticipantImpl) HasICEConnected() bool { + return p.TransportManager.HasSubscriberICEEverConnected() || p.TransportManager.HasPublisherICEEverConnected() +} + func (p *ParticipantImpl) HasConnected() bool { return p.TransportManager.HasSubscriberEverConnected() || p.TransportManager.HasPublisherEverConnected() } @@ -3244,9 +3281,9 @@ func (p *ParticipantImpl) setTrackMuted(mute *livekit.MuteTrackRequest, fromAdmi if trackInfo != nil && changed { if mute.Muted { - p.params.TelemetryListener.OnTrackMuted(p.ID(), trackInfo) + p.GetTelemetryListener().OnTrackMuted(p.ID(), trackInfo) } else { - p.params.TelemetryListener.OnTrackUnmuted(p.ID(), trackInfo) + p.GetTelemetryListener().OnTrackUnmuted(p.ID(), trackInfo) } } @@ -3267,7 +3304,6 @@ func (p *ParticipantImpl) mediaTrackReceived( rtpReceiver *webrtc.RTPReceiver, ) (*MediaTrack, bool, bool, buffer.VideoLayersRid) { p.pendingTracksLock.Lock() - newTrack := false mid := p.TransportManager.GetPublisherMid(rtpReceiver) p.pubLogger.Debugw( @@ -3291,10 +3327,13 @@ func (p *ParticipantImpl) mediaTrackReceived( } // use existing media track to handle simulcast - var createdAt time.Time - var isMigrated bool - var ridsFromSdp buffer.VideoLayersRid - var pubTime time.Duration + var ( + createdAt time.Time + isNewTrack bool + isMigrated bool + ridsFromSdp buffer.VideoLayersRid + pubTime time.Duration + ) mt, ok := p.getPublishedTrackBySdpCid(track.ID()).(*MediaTrack) if !ok { var ( @@ -3324,7 +3363,14 @@ func (p *ParticipantImpl) mediaTrackReceived( } } if codecFound != len(ti.Codecs) { - p.pubLogger.Warnw("migrated track codec mismatched", nil, "track", logger.Proto(ti), "webrtcCodec", parameters) + p.pubLogger.Warnw( + "migrated track codec mismatched", nil, + "trackID", ti.Sid, + "track", logger.Proto(ti), + "webrtcCodec", parameters, + "codecFound", codecFound, + "codecCount", len(ti.Codecs), + ) p.pendingTracksLock.Unlock() p.IssueFullReconnect(types.ParticipantCloseReasonMigrateCodecMismatch) return nil, false, false, ridsFromSdp @@ -3348,7 +3394,7 @@ func (p *ParticipantImpl) mediaTrackReceived( } mt = p.addMediaTrack(signalCid, ti) - newTrack = true + isNewTrack = true } // a track might have been set up in migrate-in path and won't show up as a new track here, @@ -3360,12 +3406,12 @@ func (p *ParticipantImpl) mediaTrackReceived( } } } - if !newTrack { - newTrack = !mt.Published() + if !isNewTrack { + isNewTrack = !mt.Published() } mt.SetPublished(true) - if newTrack { + if isNewTrack { // if the addTrackRequest is sent before publisher peer connection is established, then it means the client tries to publish // before fully connected, in this case we only record the time when publisher peer connection is established since // we want this metric to represent the time cost by publishing. @@ -3379,7 +3425,7 @@ func (p *ParticipantImpl) mediaTrackReceived( _, isReceiverAdded := mt.AddReceiver(rtpReceiver, track, mid) - if newTrack { + if isNewTrack { go func() { // TODO: remove this after we know where the high delay is coming from if pubTime > 3*time.Second { @@ -3410,11 +3456,12 @@ func (p *ParticipantImpl) mediaTrackReceived( p.GetClientInfo().GetSdk(), p.Kind(), ) + p.handleTrackPublished(mt, isMigrated, false) }() } - return mt, newTrack, isReceiverAdded, ridsFromSdp + return mt, isNewTrack, isReceiverAdded, ridsFromSdp } func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { @@ -3429,6 +3476,31 @@ func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *M return nil } + // check if the migrated track has correct codec + if len(ti.Codecs) > 0 { + parameters := rtpReceiver.GetParameters() + var codecFound int + for _, c := range ti.Codecs { + for _, nc := range parameters.Codecs { + if mime.IsMimeTypeStringEqual(nc.MimeType, c.MimeType) { + codecFound++ + break + } + } + } + if codecFound != len(ti.Codecs) { + p.pubLogger.Warnw( + "migrated track codec mismatched", nil, + "trackID", ti.Sid, + "track", logger.Proto(ti), + "webrtcCodec", parameters, + "codecFound", codecFound, + "codecCount", len(ti.Codecs), + ) + return nil + } + } + mt := p.addMediaTrack(cid, ti) mt.SetMigrated(true) @@ -3485,7 +3557,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo) ReceiverConfig: p.params.Config.Receiver, AudioConfig: p.params.AudioConfig, VideoConfig: p.params.VideoConfig, - TelemetryListener: p.params.TelemetryListener, + TelemetryListener: p.GetTelemetryListener, Logger: LoggerWithTrack(p.pubLogger, livekit.TrackID(ti.Sid), false), Reporter: p.params.Reporter.WithTrack(ti.Sid), SubscriberConfig: p.params.Config.Subscriber, @@ -3501,11 +3573,10 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo) EnableRTPStreamRestartDetection: p.params.EnableRTPStreamRestartDetection, UpdateTrackInfoByVideoSizeChange: p.params.UseOneShotSignallingMode, ForceBackupCodecPolicySimulcast: p.params.ForceBackupCodecPolicySimulcast, + OnSubscribedMaxQualityChange: p.onSubscribedMaxQualityChange, + OnSubscribedAudioCodecChange: p.onSubscribedAudioCodecChange, }, ti) - mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange) - mt.OnSubscribedAudioCodecChange(p.onSubscribedAudioCodecChange) - // add to published and clean up pending if p.supervisor != nil { p.supervisor.SetPublishedTrack(livekit.TrackID(ti.Sid), mt) @@ -3537,7 +3608,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo) p.supervisor.ClearPublishedTrack(trackID, mt) } - p.params.TelemetryListener.OnTrackUnpublished( + p.GetTelemetryListener().OnTrackUnpublished( p.ID(), p.Identity(), mt.ToProto(), @@ -3573,7 +3644,7 @@ func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack, isMigrate if !isSynthetic { // send webhook after callbacks are complete, persistence and state handling happens // in `onTrackPublished` cb - p.params.TelemetryListener.OnTrackPublished( + p.GetTelemetryListener().OnTrackPublished( p.ID(), p.Identity(), track.ToProto(), @@ -3952,7 +4023,37 @@ func (p *ParticipantImpl) SupportsTransceiverReuse(mt types.MediaTrack) bool { } func (p *ParticipantImpl) SendDataMessage(kind livekit.DataPacket_Kind, data []byte, sender livekit.ParticipantID, seq uint32) error { - if sender == "" || kind != livekit.DataPacket_RELIABLE || seq == 0 { + if kind != livekit.DataPacket_RELIABLE { + if p.State() != livekit.ParticipantInfo_ACTIVE { + return ErrDataChannelUnavailable + } + return p.TransportManager.SendDataMessage(kind, data) + } + + if sender == "" || seq == 0 { + // Unsequenced reliable data, i. e. not published by a participant, room service + // SendData for example. Such a message cannot be recovered by + // replayJoiningReliableMessages as the data message cache is keyed on + // sender/sequence number, so hold on to the message itself here till the + // reliable data channel is writable. + p.reliableDataInfo.joiningMessageLock.Lock() + if !p.reliableDataInfo.canWriteReliable { + if p.reliableDataInfo.joiningUnsequencedBytes+len(data) > cMaxJoiningUnsequencedReliableBytes { + p.reliableDataInfo.joiningUnsequencedDropped++ + p.reliableDataInfo.joiningMessageLock.Unlock() + return ErrDataChannelUnavailable + } + + p.reliableDataInfo.joiningUnsequencedMessages = append( + p.reliableDataInfo.joiningUnsequencedMessages, + slices.Clone(data), + ) + p.reliableDataInfo.joiningUnsequencedBytes += len(data) + p.reliableDataInfo.joiningMessageLock.Unlock() + return nil + } + p.reliableDataInfo.joiningMessageLock.Unlock() + if p.State() != livekit.ParticipantInfo_ACTIVE { return ErrDataChannelUnavailable } @@ -4067,6 +4168,20 @@ func (p *ParticipantImpl) replayJoiningReliableMessages() { p.TransportManager.SendDataMessage(livekit.DataPacket_RELIABLE, msgCache.Data) } + for _, msg := range p.reliableDataInfo.joiningUnsequencedMessages { + p.TransportManager.SendDataMessage(livekit.DataPacket_RELIABLE, msg) + } + if p.reliableDataInfo.joiningUnsequencedDropped != 0 { + p.params.Logger.Warnw( + "dropped unsequenced reliable data messages while joining", nil, + "numDropped", p.reliableDataInfo.joiningUnsequencedDropped, + "numReplayed", len(p.reliableDataInfo.joiningUnsequencedMessages), + ) + } + p.reliableDataInfo.joiningUnsequencedMessages = nil + p.reliableDataInfo.joiningUnsequencedBytes = 0 + p.reliableDataInfo.joiningUnsequencedDropped = 0 + p.reliableDataInfo.joiningMessageFirstSeqs = make(map[livekit.ParticipantID]uint32) p.reliableDataInfo.canWriteReliable = true p.reliableDataInfo.joiningMessageLock.Unlock() @@ -4207,7 +4322,7 @@ func (p *ParticipantImpl) MoveToRoom(params types.MoveToRoomParams) { track.(types.LocalMediaTrack).ClearSubscriberNodes() trackInfo := track.ToProto() - p.params.TelemetryListener.OnTrackUnpublished( + p.GetTelemetryListener().OnTrackUnpublished( p.ID(), p.Identity(), trackInfo, @@ -4216,6 +4331,9 @@ func (p *ParticipantImpl) MoveToRoom(params types.MoveToRoomParams) { ) } + p.params.Reporter.ReportEndTime(time.Now()) + p.SubscriptionManager.ClearAllSubscriptions() + // fire onClose callback for original room p.lock.Lock() onClose := p.onClose @@ -4231,13 +4349,15 @@ func (p *ParticipantImpl) MoveToRoom(params types.MoveToRoomParams) { p.telemetryGuard = &telemetry.ReferenceGuard{} p.lock.Unlock() - p.params.Reporter.ReportEndTime(time.Now()) p.params.LoggerResolver.Reset() p.params.ReporterResolver.Reset() + p.setListener(params.Listener) + p.setTelemetryListener(params.TelemetryListener) p.participantHelper.Store(params.Helper) - p.SubscriptionManager.ClearAllSubscriptions() + p.id.Store(params.ParticipantID) + grants := p.grants.Load().Clone() grants.Video.Room = string(params.RoomName) p.grants.Store(grants) diff --git a/pkg/rtc/participant_data_track.go b/pkg/rtc/participant_data_track.go index df9854744..23a34134f 100644 --- a/pkg/rtc/participant_data_track.go +++ b/pkg/rtc/participant_data_track.go @@ -15,8 +15,8 @@ package rtc import ( - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -121,7 +121,7 @@ func (p *ParticipantImpl) HandlePublishDataTrackRequest(req *livekit.PublishData p.ID(), p.Kind(), p.KindDetails(), - p.params.TelemetryListener, + p.GetTelemetryListener, p.params.Reporter, ), }, diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 33263a360..3a32808ac 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -783,6 +783,7 @@ type participantOpts struct { publisher bool clientConf *livekit.ClientConfiguration clientInfo *livekit.ClientInfo + migration bool } func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *participantOpts) *ParticipantImpl { @@ -837,6 +838,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p VersionGenerator: utils.NewDefaultTimedVersionGenerator(), ParticipantListener: &typesfakes.FakeLocalParticipantListener{}, ParticipantHelper: &typesfakes.FakeLocalParticipantHelper{}, + Migration: opts.migration, }) p.isPublisher.Store(opts.publisher) p.updateState(livekit.ParticipantInfo_ACTIVE) @@ -847,3 +849,201 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p func newParticipantForTest(identity livekit.ParticipantIdentity) *ParticipantImpl { return newParticipantForTestWithOpts(identity, nil) } + +func TestUnsequencedReliableDataBufferedWhileJoining(t *testing.T) { + // unsequenced reliable data, i. e. room service SendData, arriving before the + // reliable data channel is writable should be held and replayed, not dropped + t.Run("buffers and replays", func(t *testing.T) { + p := newParticipantForTest("test") + require.False(t, p.reliableDataInfo.canWriteReliable) + + require.NoError(t, p.SendDataMessage(livekit.DataPacket_RELIABLE, []byte("one"), "", 0)) + require.NoError(t, p.SendDataMessage(livekit.DataPacket_RELIABLE, []byte("two"), "", 0)) + + require.Equal(t, [][]byte{[]byte("one"), []byte("two")}, p.reliableDataInfo.joiningUnsequencedMessages) + require.Equal(t, 6, p.reliableDataInfo.joiningUnsequencedBytes) + + p.replayJoiningReliableMessages() + + require.True(t, p.reliableDataInfo.canWriteReliable) + require.Empty(t, p.reliableDataInfo.joiningUnsequencedMessages) + require.Zero(t, p.reliableDataInfo.joiningUnsequencedBytes) + }) + + t.Run("does not buffer once writable", func(t *testing.T) { + p := newParticipantForTest("test") + p.replayJoiningReliableMessages() + + // no data channel in test, so the write through fails rather than getting buffered + require.Error(t, p.SendDataMessage(livekit.DataPacket_RELIABLE, []byte("one"), "", 0)) + require.Empty(t, p.reliableDataInfo.joiningUnsequencedMessages) + }) + + t.Run("bounded buffer", func(t *testing.T) { + p := newParticipantForTest("test") + + data := make([]byte, cMaxJoiningUnsequencedReliableBytes) + require.NoError(t, p.SendDataMessage(livekit.DataPacket_RELIABLE, data, "", 0)) + require.Error(t, p.SendDataMessage(livekit.DataPacket_RELIABLE, []byte("overflow"), "", 0)) + + require.Len(t, p.reliableDataInfo.joiningUnsequencedMessages, 1) + require.Equal(t, 1, p.reliableDataInfo.joiningUnsequencedDropped) + }) + + t.Run("lossy is not buffered", func(t *testing.T) { + p := newParticipantForTest("test") + + require.Error(t, p.SendDataMessage(livekit.DataPacket_LOSSY, []byte("one"), "", 0)) + require.Empty(t, p.reliableDataInfo.joiningUnsequencedMessages) + }) +} + +func TestMigratingInParticipantWaitsForReconnectResponse(t *testing.T) { + // a migrating in participant resumes on a ReconnectResponse, the client takes it only + // as the first message on the resumed signal connection, so everything else waits + newMigratingParticipant := func() (*ParticipantImpl, *routingfakes.FakeMessageSink) { + p := newParticipantForTestWithOpts("test", &participantOpts{ + migration: true, + protocolVersion: 17, + clientInfo: &livekit.ClientInfo{Sdk: livekit.ClientInfo_JS, Version: "2.15.2"}, + }) + return p, p.params.Sink.(*routingfakes.FakeMessageSink) + } + + t.Run("not ready until the response is sent", func(t *testing.T) { + p, sink := newMigratingParticipant() + require.False(t, p.IsReady()) + + require.NoError(t, p.HandleReconnectAndSendResponse( + livekit.ReconnectReason_RR_UNKNOWN, + &livekit.ReconnectResponse{LastMessageSeq: 21}, + )) + require.True(t, p.IsReady()) + + require.Equal(t, 1, sink.WriteMessageCallCount()) + res := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse) + require.NotNil(t, res.GetReconnect()) + require.EqualValues(t, 21, res.GetReconnect().LastMessageSeq) + }) + + t.Run("the response goes out first", func(t *testing.T) { + p, sink := newMigratingParticipant() + + // a room update while waiting is dropped, it is re-sent after the migration + require.NoError(t, p.SendRoomUpdate(&livekit.Room{Name: "test"})) + // participant updates while waiting are queued + require.NoError(t, p.SendParticipantUpdate([]*livekit.ParticipantInfo{{ + Sid: "PA_other", + Identity: "other", + Version: 1, + }})) + require.Zero(t, sink.WriteMessageCallCount()) + + require.NoError(t, p.HandleReconnectAndSendResponse( + livekit.ReconnectReason_RR_UNKNOWN, + &livekit.ReconnectResponse{}, + )) + + require.Equal(t, 2, sink.WriteMessageCallCount()) + first := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse) + require.NotNil(t, first.GetReconnect()) + second := sink.WriteMessageArgsForCall(1).(*livekit.SignalResponse) + require.NotNil(t, second.GetUpdate()) + require.Len(t, second.GetUpdate().Participants, 1) + require.EqualValues(t, "PA_other", second.GetUpdate().Participants[0].Sid) + }) + + t.Run("a client that cannot handle the response is ready right away", func(t *testing.T) { + p := newParticipantForTestWithOpts("test", &participantOpts{ + migration: true, + protocolVersion: 17, + clientInfo: &livekit.ClientInfo{Sdk: livekit.ClientInfo_JS, Version: "1.6.2"}, + }) + sink := p.params.Sink.(*routingfakes.FakeMessageSink) + + require.NoError(t, p.HandleReconnectAndSendResponse( + livekit.ReconnectReason_RR_UNKNOWN, + &livekit.ReconnectResponse{}, + )) + require.True(t, p.IsReady()) + require.Zero(t, sink.WriteMessageCallCount()) + + require.NoError(t, p.SendRoomUpdate(&livekit.Room{Name: "test"})) + require.Equal(t, 1, sink.WriteMessageCallCount()) + }) +} + +func TestResumedParticipantWaitsForReconnectResponse(t *testing.T) { + // a resumed connection has to open with the ReconnectResponse too, and the + // participant is ready throughout, so the signaller holds messages back + newResumedParticipant := func(version string) (*ParticipantImpl, *routingfakes.FakeMessageSink) { + p := newParticipantForTestWithOpts("test", &participantOpts{ + protocolVersion: 17, + clientInfo: &livekit.ClientInfo{Sdk: livekit.ClientInfo_JS, Version: version}, + }) + require.True(t, p.IsReady()) + + sink := &routingfakes.FakeMessageSink{} + p.SwapResponseSink(sink, types.SignallingCloseReasonResume) + return p, sink + } + + t.Run("the response goes out first", func(t *testing.T) { + p, sink := newResumedParticipant("2.15.2") + + // a room update in this window is dropped, the resume path re-sends room state + require.NoError(t, p.SendRoomUpdate(&livekit.Room{Name: "test"})) + // participant updates are queued + require.NoError(t, p.SendParticipantUpdate([]*livekit.ParticipantInfo{{ + Sid: "PA_other", + Identity: "other", + Version: 1, + }})) + require.Zero(t, sink.WriteMessageCallCount()) + + require.NoError(t, p.HandleReconnectAndSendResponse( + livekit.ReconnectReason_RR_SIGNAL_DISCONNECTED, + &livekit.ReconnectResponse{LastMessageSeq: 7}, + )) + + require.Equal(t, 2, sink.WriteMessageCallCount()) + first := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse) + require.NotNil(t, first.GetReconnect()) + require.EqualValues(t, 7, first.GetReconnect().LastMessageSeq) + second := sink.WriteMessageArgsForCall(1).(*livekit.SignalResponse) + require.NotNil(t, second.GetUpdate()) + require.EqualValues(t, "PA_other", second.GetUpdate().Participants[0].Sid) + }) + + t.Run("a connection that opens without a response still delivers the queue", func(t *testing.T) { + p, sink := newResumedParticipant("2.15.2") + + require.NoError(t, p.SendParticipantUpdate([]*livekit.ParticipantInfo{{ + Sid: "PA_other", + Identity: "other", + Version: 1, + }})) + require.Zero(t, sink.WriteMessageCallCount()) + + // no ReconnectResponse written, this is what the handshake window does on expiry + p.signaller.OpenHandshake() + + require.Equal(t, 1, sink.WriteMessageCallCount()) + res := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse) + require.NotNil(t, res.GetUpdate()) + require.EqualValues(t, "PA_other", res.GetUpdate().Participants[0].Sid) + }) + + t.Run("a client that cannot handle the response is not held back", func(t *testing.T) { + p, sink := newResumedParticipant("1.6.2") + + require.NoError(t, p.HandleReconnectAndSendResponse( + livekit.ReconnectReason_RR_SIGNAL_DISCONNECTED, + &livekit.ReconnectResponse{}, + )) + require.Zero(t, sink.WriteMessageCallCount()) + + require.NoError(t, p.SendRoomUpdate(&livekit.Room{Name: "test"})) + require.Equal(t, 1, sink.WriteMessageCallCount()) + }) +} diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 8f5c3dd2b..57f03f305 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -81,7 +81,9 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. return nil } - if !p.IsReady() { + // read under the lock the flush takes, so an update either queues before the flush + // or goes out after the connection has opened + if !p.IsReady() || p.signaller.HandshakePending() { // queue up updates p.queuedUpdates = append(p.queuedUpdates, participantsToUpdate...) p.updateLock.Unlock() @@ -177,11 +179,23 @@ func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit p.TransportManager.HandleClientReconnect(reconnectReason) if !p.params.ClientInfo.CanHandleReconnectResponse() { - return nil + // no ReconnectResponse opens this connection, so nothing to hold back + return p.reconnectResponseSentAndFlush() } - if err := p.signaller.WriteMessage(p.signalling.SignalReconnectResponse(reconnectResponse)); err != nil { + + // send reconnect response + err := p.signaller.WriteMessage(p.signalling.SignalReconnectResponse(reconnectResponse)) + + // mark sent after sending the message, so that nothing could slip through before + // ReconnectResponse is sent. Marked on a failed write too: the sink is gone in that + // case, and leaving it unmarked would hold back every message after it. + flushErr := p.reconnectResponseSentAndFlush() + if err != nil { return err } + if flushErr != nil { + return flushErr + } if p.params.ProtocolVersion.SupportsDisconnectedUpdate() { return p.sendDisconnectUpdatesForReconnect() @@ -190,6 +204,44 @@ func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit return nil } +// reconnectResponseSentAndFlush makes a migrating in participant ready and sends what was +// queued up while the connection was waiting for its ReconnectResponse. +func (p *ParticipantImpl) reconnectResponseSentAndFlush() error { + // a successful write opens the connection by itself, this covers the paths that do + // not write one, i. e. a client that cannot handle it and a failed write + p.signaller.OpenHandshake() + + p.updateLock.Lock() + p.reconnectResponseSent.Store(true) + queuedUpdates := p.queuedUpdates + p.queuedUpdates = nil + p.updateLock.Unlock() + + if len(queuedUpdates) > 0 { + return p.SendParticipantUpdate(queuedUpdates) + } + + return nil +} + +// flushQueuedUpdates sends the updates queued while the connection was closed for the +// handshake. Called when the connection opens, including when it opens without a +// ReconnectResponse having been written. +func (p *ParticipantImpl) flushQueuedUpdates() { + p.updateLock.Lock() + queuedUpdates := p.queuedUpdates + p.queuedUpdates = nil + p.updateLock.Unlock() + + if len(queuedUpdates) == 0 { + return + } + + if err := p.SendParticipantUpdate(queuedUpdates); err != nil { + p.params.Logger.Warnw("could not send queued participant updates", err) + } +} + func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { lastSignalAt := p.TransportManager.LastSeenSignalAt() var disconnectedParticipants []*livekit.ParticipantInfo diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 74f7f3059..76224e731 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -40,7 +40,6 @@ import ( "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -48,6 +47,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/datatrack" ) const ( @@ -139,7 +139,7 @@ type Room struct { onParticipantChanged func(p types.Participant) onRoomUpdated func() - onClose func() + onClose func(types.RoomCloseReason) simulationLock sync.Mutex disconnectSignalOnResumeParticipants map[livekit.ParticipantIdentity]time.Time @@ -802,12 +802,12 @@ func (r *Room) CloseIfEmpty() { r.lock.Unlock() if elapsed >= int64(timeout) { - r.Close(types.ParticipantCloseReasonRoomClosed) + r.Close(types.RoomCloseReasonIdleTimeout) r.logger.Infow("closing idle room", "reason", reason) } } -func (r *Room) Close(reason types.ParticipantCloseReason) { +func (r *Room) Close(reason types.RoomCloseReason) { r.lock.Lock() select { case <-r.closed: @@ -819,19 +819,20 @@ func (r *Room) Close(reason types.ParticipantCloseReason) { close(r.closed) r.lock.Unlock() - r.logger.Infow("closing room") + r.logger.Infow("closing room", "reason", reason) + participantCloseReason := reason.ToParticipantCloseReason() for _, p := range r.GetParticipants() { - _ = p.Close(true, reason, false) + _ = p.Close(true, participantCloseReason, false) } r.protoProxy.Stop() if r.onClose != nil { - r.onClose() + r.onClose(reason) } } -func (r *Room) OnClose(f func()) { +func (r *Room) OnClose(f func(types.RoomCloseReason)) { r.onClose = f } diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 0d69710d3..4f76d0705 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -377,8 +377,10 @@ func TestRoomClosure(t *testing.T) { t.Run("room closes after participant leaves", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) isClosed := false - rm.OnClose(func() { + var closeReason types.RoomCloseReason + rm.OnClose(func(reason types.RoomCloseReason) { isClosed = true + closeReason = reason }) p := rm.GetParticipants()[0] rm.lock.Lock() @@ -392,6 +394,7 @@ func TestRoomClosure(t *testing.T) { rm.CloseIfEmpty() require.Len(t, rm.GetParticipants(), 0) require.True(t, isClosed) + require.Equal(t, types.RoomCloseReasonIdleTimeout, closeReason) require.Equal(t, ErrRoomClosed, rm.Join(p, nil, nil, iceServersForRoom)) }) @@ -399,19 +402,24 @@ func TestRoomClosure(t *testing.T) { t.Run("room does not close before empty timeout", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 0}) isClosed := false - rm.OnClose(func() { + var closeReason types.RoomCloseReason + rm.OnClose(func(reason types.RoomCloseReason) { isClosed = true + closeReason = reason }) require.NotZero(t, rm.protoRoom.EmptyTimeout) rm.CloseIfEmpty() require.False(t, isClosed) + require.Equal(t, types.RoomCloseReasonUnknown, closeReason) }) t.Run("room closes after empty timeout", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 0}) isClosed := false - rm.OnClose(func() { + var closeReason types.RoomCloseReason + rm.OnClose(func(reason types.RoomCloseReason) { isClosed = true + closeReason = reason }) rm.lock.Lock() rm.protoRoom.EmptyTimeout = 1 @@ -420,6 +428,7 @@ func TestRoomClosure(t *testing.T) { time.Sleep(1010 * time.Millisecond) rm.CloseIfEmpty() require.True(t, isClosed) + require.Equal(t, types.RoomCloseReasonIdleTimeout, closeReason) }) } @@ -461,7 +470,7 @@ func TestActiveSpeakers(t *testing.T) { audioUpdateDuration := (audioUpdateInterval + 10) * time.Millisecond t.Run("participant should not be getting audio updates (protocol 2)", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1, protocol: 2}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) p := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) require.Empty(t, rm.GetActiveSpeakers()) @@ -473,7 +482,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("speakers should be sorted by loudness", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p2 := participants[1].(*typesfakes.FakeLocalParticipant) @@ -488,7 +497,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("participants are getting audio updates (protocol 3+)", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) time.Sleep(time.Millisecond) // let the first update cycle run @@ -527,7 +536,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("audio level is smoothed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3, audioSmoothIntervals: 3}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) @@ -621,7 +630,7 @@ func TestDataChannel(t *testing.T) { mode := mode t.Run(modeNames[mode], func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) lpl := rm.LocalParticipantListener() @@ -667,7 +676,7 @@ func TestDataChannel(t *testing.T) { mode := mode t.Run(modeNames[mode], func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 4}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) lpl := rm.LocalParticipantListener() @@ -713,7 +722,7 @@ func TestDataChannel(t *testing.T) { t.Run("publishing disallowed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) @@ -743,7 +752,7 @@ func TestDataChannel(t *testing.T) { func TestHiddenParticipants(t *testing.T) { t.Run("other participants don't receive hidden updates", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, numHidden: 1}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) pNew := NewMockParticipant("new", types.CurrentProtocol, false, false, rm.LocalParticipantListener()) rm.Join(pNew, nil, nil, iceServersForRoom) @@ -775,7 +784,7 @@ func TestHiddenParticipants(t *testing.T) { func TestRoomUpdate(t *testing.T) { t.Run("updates are sent when participant joined", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) require.Equal(t, 0, p1.SendRoomUpdateCallCount()) @@ -791,7 +800,7 @@ func TestRoomUpdate(t *testing.T) { t.Run("participants should receive metadata update", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close(types.ParticipantCloseReasonNone) + defer rm.Close(types.RoomCloseReasonUnknown) rm.SetMetadata("test metadata...") diff --git a/pkg/rtc/signalanddatastats.go b/pkg/rtc/signalanddatastats.go index fe83bd9f4..fd7227d6e 100644 --- a/pkg/rtc/signalanddatastats.go +++ b/pkg/rtc/signalanddatastats.go @@ -60,7 +60,7 @@ type BytesTrackStats struct { sendMessages, recvMessages atomic.Uint32 totalSendBytes, totalRecvBytes atomic.Uint64 totalSendMessages, totalRecvMessages atomic.Uint32 - telemetryListener types.ParticipantTelemetryListener + telemetryListener func() types.ParticipantTelemetryListener reporter roomobs.TrackReporter done core.Fuse } @@ -71,7 +71,7 @@ func NewBytesTrackStats( pID livekit.ParticipantID, kind livekit.ParticipantInfo_Kind, kindDetails []livekit.ParticipantInfo_KindDetail, - telemetryListener types.ParticipantTelemetryListener, + telemetryListener func() types.ParticipantTelemetryListener, participantReporter roomobs.ParticipantSessionReporter, ) *BytesTrackStats { s := &BytesTrackStats{ @@ -134,7 +134,7 @@ func (s *BytesTrackStats) Stop() { func (s *BytesTrackStats) report() { if recv := s.recv.Swap(0); recv > 0 { packets := s.recvMessages.Swap(0) - s.telemetryListener.OnTrackStats( + s.telemetryListener().OnTrackStats( telemetry.StatsKeyForData(s.country, livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -149,7 +149,7 @@ func (s *BytesTrackStats) report() { if send := s.send.Swap(0); send > 0 { packets := s.sendMessages.Swap(0) - s.telemetryListener.OnTrackStats( + s.telemetryListener().OnTrackStats( telemetry.StatsKeyForData(s.country, livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -217,7 +217,7 @@ func NewBytesSignalStats( trackResolver: trackReporterResolver, } b.BytesTrackStats = BytesTrackStats{ - telemetryListener: b, + telemetryListener: b.telemetryListener, reporter: trackReporter, } return b @@ -295,6 +295,10 @@ func (s *BytesSignalStats) worker() { close(s.stopped) } +func (s *BytesSignalStats) telemetryListener() types.ParticipantTelemetryListener { + return s +} + func (s *BytesSignalStats) OnTrackStats(key telemetry.StatsKey, stat *livekit.AnalyticsStat) { stat.RoomId, stat.RoomName = s.ri.Sid, s.ri.Name s.telemetry.TrackStats(livekit.RoomID(s.ri.Sid), livekit.RoomName(s.ri.Name), key, stat) diff --git a/pkg/rtc/signalling/interfaces.go b/pkg/rtc/signalling/interfaces.go index 99c3a0acc..2f8f59138 100644 --- a/pkg/rtc/signalling/interfaces.go +++ b/pkg/rtc/signalling/interfaces.go @@ -32,6 +32,13 @@ type ParticipantSignaller interface { GetResponseSink() routing.MessageSink CloseSignalConnection(reason types.SignallingCloseReason) + // HandshakePending reports whether a resumed connection is still waiting for the + // ReconnectResponse it has to open with + HandshakePending() bool + // OpenHandshake lets messages flow on a resumed connection that does not open with + // a ReconnectResponse + OpenHandshake() + WriteMessage(msg proto.Message) error } diff --git a/pkg/rtc/signalling/signallerasync.go b/pkg/rtc/signalling/signallerasync.go index 0dc0853ee..c82dbc582 100644 --- a/pkg/rtc/signalling/signallerasync.go +++ b/pkg/rtc/signalling/signallerasync.go @@ -31,8 +31,9 @@ import ( var _ ParticipantSignaller = (*signallerAsync)(nil) type SignallerAsyncParams struct { - Logger logger.Logger - Participant types.LocalParticipant + Logger logger.Logger + Participant types.LocalParticipant + OnHandshakeOpened func() } type signallerAsync struct { @@ -43,8 +44,11 @@ type signallerAsync struct { func NewSignallerAsync(params SignallerAsyncParams) ParticipantSignaller { return &signallerAsync{ - params: params, - signallerAsyncBase: newSignallerAsyncBase(signallerAsyncBaseParams{Logger: params.Logger}), + params: params, + signallerAsyncBase: newSignallerAsyncBase(signallerAsyncBaseParams{ + Logger: params.Logger, + OnHandshakeOpened: params.OnHandshakeOpened, + }), } } @@ -69,17 +73,35 @@ func (s *signallerAsync) WriteMessage(msg proto.Message) error { return nil } - if !s.params.Participant.IsReady() { - if typed, ok := msg.(*livekit.SignalResponse); !ok { - s.params.Logger.Warnw( - "unknown message type", nil, - "messageType", fmt.Sprintf("%T", msg), - ) - } else { - if typed.GetJoin() == nil { - return nil + // a signal connection opens with a join response, or with a reconnect response when + // it is resumed or migrated in. The client reads that first message as the handshake, + // so nothing may go out ahead of it. + isHandshake := false + isSdp := false + if typed, ok := msg.(*livekit.SignalResponse); !ok { + s.params.Logger.Warnw( + "unknown message type", nil, + "messageType", fmt.Sprintf("%T", msg), + ) + } else { + isHandshake = typed.GetJoin() != nil || typed.GetReconnect() != nil + isSdp = typed.GetOffer() != nil || typed.GetAnswer() != nil + } + + if !isHandshake && (!s.params.Participant.IsReady() || s.HandshakePending()) { + logFunc := s.params.Logger.Debugw + if isSdp { + // a dropped SDP leaves the negotiation waiting for the peer until the + // negotiation state machine recovers it, so make it visible + logFunc = func(msg string, keysAndValues ...any) { + s.params.Logger.Infow(msg, keysAndValues...) } } + logFunc( + "dropping message, connection has not opened yet", + "messageType", getMessageType(msg), + ) + return nil } sink := s.GetResponseSink() @@ -108,6 +130,10 @@ func (s *signallerAsync) WriteMessage(msg proto.Message) error { return err } } else { + if isHandshake { + // the connection is open, hold nothing back any more + s.OpenHandshake() + } s.params.Logger.Debugw("sent signal response", "response", logger.Proto(msg)) } return nil diff --git a/pkg/rtc/signalling/signallerasyncbase.go b/pkg/rtc/signalling/signallerasyncbase.go index 0b4d2b665..28bfc968d 100644 --- a/pkg/rtc/signalling/signallerasyncbase.go +++ b/pkg/rtc/signalling/signallerasyncbase.go @@ -16,6 +16,7 @@ package signalling import ( "sync" + "time" "github.com/livekit/protocol/logger" @@ -25,8 +26,16 @@ import ( type signallerAsyncBaseParams struct { Logger logger.Logger + // called when the connection opens, i. e. when messages held back for the + // handshake may be sent + OnHandshakeOpened func() } +// how long a resumed connection holds messages back waiting for its ReconnectResponse. +// A path that resumes without sending one lets messages flow after this instead of +// holding them back for the rest of the session. +const handshakeWindow = 5 * time.Second + type signallerAsyncBase struct { signallerUnimplemented @@ -34,6 +43,13 @@ type signallerAsyncBase struct { resSinkMu sync.Mutex resSink routing.MessageSink + // set while a resumed connection holds messages back waiting for its + // ReconnectResponse, the timer opens the connection if none is written. + // handshakeGeneration tells a timeout whether it belongs to the current wait, a + // Stop cannot cancel a callback that has already started running. + handshakePending bool + handshakeGeneration uint32 + handshakeTimer *time.Timer } func newSignallerAsyncBase(params signallerAsyncBaseParams) *signallerAsyncBase { @@ -46,8 +62,24 @@ func (s *signallerAsyncBase) SwapResponseSink(sink routing.MessageSink, reason t s.resSinkMu.Lock() oldSink := s.resSink s.resSink = sink + // a resumed connection has to open with the ReconnectResponse, the client takes it + // only as the first message it reads + opened := false + switch { + case sink == nil: + // the connection is gone, keep anything queued for the next one + s.disarmHandshakeLocked() + case reason == types.SignallingCloseReasonResume: + s.armHandshakeLocked() + default: + opened = s.disarmHandshakeLocked() + } s.resSinkMu.Unlock() + if opened { + s.notifyHandshakeOpened() + } + if oldSink != nil { if sink != nil { s.params.Logger.Debugw( @@ -67,6 +99,80 @@ func (s *signallerAsyncBase) SwapResponseSink(sink routing.MessageSink, reason t } } +// HandshakePending is a plain read, so a caller can decide to hold a message back +// while holding its own lock +func (s *signallerAsyncBase) HandshakePending() bool { + s.resSinkMu.Lock() + defer s.resSinkMu.Unlock() + + return s.handshakePending +} + +func (s *signallerAsyncBase) OpenHandshake() { + s.resSinkMu.Lock() + opened := s.disarmHandshakeLocked() + s.resSinkMu.Unlock() + + if opened { + s.notifyHandshakeOpened() + } +} + +func (s *signallerAsyncBase) armHandshakeLocked() { + s.stopHandshakeTimerLocked() + + s.handshakePending = true + s.handshakeGeneration++ + generation := s.handshakeGeneration + // the connection opens on its own if nothing writes a ReconnectResponse, a path + // that resumes without one should not hold messages back for the whole session + s.handshakeTimer = time.AfterFunc(handshakeWindow, func() { s.onHandshakeTimeout(generation) }) +} + +// disarmHandshakeLocked reports whether it opened a connection that was holding +// messages back +func (s *signallerAsyncBase) disarmHandshakeLocked() bool { + s.stopHandshakeTimerLocked() + + wasPending := s.handshakePending + s.handshakePending = false + // a timeout of the wait that just ended is stale + s.handshakeGeneration++ + return wasPending +} + +func (s *signallerAsyncBase) stopHandshakeTimerLocked() { + if s.handshakeTimer != nil { + s.handshakeTimer.Stop() + s.handshakeTimer = nil + } +} + +func (s *signallerAsyncBase) onHandshakeTimeout(generation uint32) { + s.resSinkMu.Lock() + if generation != s.handshakeGeneration { + // the wait this timeout was armed for has ended, a later one may be in progress + s.resSinkMu.Unlock() + return + } + opened := s.disarmHandshakeLocked() + s.resSinkMu.Unlock() + + if !opened { + return + } + + s.params.Logger.Warnw("resumed connection did not open with a ReconnectResponse", nil) + s.notifyHandshakeOpened() +} + +// notifyHandshakeOpened runs without resSinkMu held, the callback sends messages +func (s *signallerAsyncBase) notifyHandshakeOpened() { + if s.params.OnHandshakeOpened != nil { + s.params.OnHandshakeOpened() + } +} + func (s *signallerAsyncBase) GetResponseSink() routing.MessageSink { s.resSinkMu.Lock() defer s.resSinkMu.Unlock() diff --git a/pkg/rtc/signalling/signallerasyncbase_test.go b/pkg/rtc/signalling/signallerasyncbase_test.go new file mode 100644 index 000000000..1d91a4c7e --- /dev/null +++ b/pkg/rtc/signalling/signallerasyncbase_test.go @@ -0,0 +1,157 @@ +package signalling + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/routing/routingfakes" + "github.com/livekit/livekit-server/pkg/rtc/types" +) + +func newTestSignallerBase(onHandshakeOpened func()) *signallerAsyncBase { + return newSignallerAsyncBase(signallerAsyncBaseParams{ + Logger: logger.GetLogger(), + OnHandshakeOpened: onHandshakeOpened, + }) +} + +// currentHandshakeGeneration is what a timeout armed right now would be tagged with +func currentHandshakeGeneration(s *signallerAsyncBase) uint32 { + s.resSinkMu.Lock() + defer s.resSinkMu.Unlock() + + return s.handshakeGeneration +} + +func TestHandshakeGate(t *testing.T) { + t.Run("a resumed connection holds messages back", func(t *testing.T) { + s := newTestSignallerBase(nil) + require.False(t, s.HandshakePending()) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.True(t, s.HandshakePending()) + + s.OpenHandshake() + require.False(t, s.HandshakePending()) + }) + + t.Run("other sink swaps do not hold messages back", func(t *testing.T) { + s := newTestSignallerBase(nil) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonUnknown) + require.False(t, s.HandshakePending()) + }) + + t.Run("closing the connection clears the gate", func(t *testing.T) { + s := newTestSignallerBase(nil) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.True(t, s.HandshakePending()) + + s.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) + require.False(t, s.HandshakePending()) + }) + + t.Run("the handshake window opens the gate", func(t *testing.T) { + s := newTestSignallerBase(nil) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.True(t, s.HandshakePending()) + + s.onHandshakeTimeout(currentHandshakeGeneration(s)) + require.False(t, s.HandshakePending()) + }) + + t.Run("a swap to a fresh connection opens the gate", func(t *testing.T) { + s := newTestSignallerBase(nil) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.True(t, s.HandshakePending()) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonUnknown) + require.False(t, s.HandshakePending()) + }) + + t.Run("a timeout of a wait that has ended does nothing", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + stale := currentHandshakeGeneration(s) + + // the client resumes again while the first timeout is running, Stop cannot + // cancel a callback that has already started + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + s.onHandshakeTimeout(stale) + + require.True(t, s.HandshakePending()) + require.Zero(t, opened.Load()) + + // and the wait in progress still opens on its own timeout + s.onHandshakeTimeout(currentHandshakeGeneration(s)) + require.False(t, s.HandshakePending()) + require.EqualValues(t, 1, opened.Load()) + }) +} + +func TestHandshakeGateNotifiesOnOpen(t *testing.T) { + t.Run("on an explicit open", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.Zero(t, opened.Load()) + + s.OpenHandshake() + require.EqualValues(t, 1, opened.Load()) + + // only the transition notifies + s.OpenHandshake() + require.EqualValues(t, 1, opened.Load()) + }) + + t.Run("on the handshake window expiring", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + s.onHandshakeTimeout(currentHandshakeGeneration(s)) + + require.False(t, s.HandshakePending()) + require.EqualValues(t, 1, opened.Load()) + }) + + t.Run("the window fires without anything else touching the gate", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + require.Eventually(t, func() bool { + return !s.HandshakePending() && opened.Load() == 1 + }, 2*handshakeWindow, 100*time.Millisecond) + }) + + t.Run("not on a connection close", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.SwapResponseSink(&routingfakes.FakeMessageSink{}, types.SignallingCloseReasonResume) + s.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) + + require.False(t, s.HandshakePending()) + require.Zero(t, opened.Load()) + }) + + t.Run("not when the gate was never armed", func(t *testing.T) { + var opened atomic.Int32 + s := newTestSignallerBase(func() { opened.Inc() }) + + s.OpenHandshake() + require.Zero(t, opened.Load()) + }) +} diff --git a/pkg/rtc/signalling/signallerunimplemented.go b/pkg/rtc/signalling/signallerunimplemented.go index 3592fd857..ae160a74f 100644 --- a/pkg/rtc/signalling/signallerunimplemented.go +++ b/pkg/rtc/signalling/signallerunimplemented.go @@ -34,6 +34,12 @@ func (u *signallerUnimplemented) GetResponseSink() routing.MessageSink { func (u *signallerUnimplemented) CloseSignalConnection(reason types.SignallingCloseReason) {} +func (u *signallerUnimplemented) HandshakePending() bool { + return false +} + +func (u *signallerUnimplemented) OpenHandshake() {} + func (u *signallerUnimplemented) WriteMessage(msg proto.Message) error { return nil } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 7e06a8f68..38d939f22 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -57,7 +57,6 @@ type SubscriptionManagerParams struct { OnTrackSubscribed func(subTrack types.SubscribedTrack) OnTrackUnsubscribed func(subTrack types.SubscribedTrack) OnSubscriptionError func(trackID livekit.TrackID, fatal bool, err error) - TelemetryListener types.ParticipantTelemetryListener SubscriptionLimitVideo, SubscriptionLimitAudio int32 @@ -308,6 +307,19 @@ func (m *SubscriptionManager) GetSubscribedTracks() []types.SubscribedTrack { return tracks } +func (m *SubscriptionManager) GetSubscribedDataTracks() []types.DataDownTrack { + m.lock.RLock() + defer m.lock.RUnlock() + + tracks := make([]types.DataDownTrack, 0, len(m.dataTrackSubscriptions)) + for _, s := range m.dataTrackSubscriptions { + if dt := s.getDataDownTrack(); dt != nil { + tracks = append(tracks, dt) + } + } + return tracks +} + func (m *SubscriptionManager) IsTrackNameSubscribed(publisherIdentity livekit.ParticipantIdentity, trackName string) bool { m.lock.RLock() defer m.lock.RUnlock() @@ -503,7 +515,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *mediaTrackSubscription) { numAttempts := s.getNumAttempts() if numAttempts == 0 { - m.params.TelemetryListener.OnTrackSubscribeRequested( + m.params.Participant.GetTelemetryListener().OnTrackSubscribeRequested( s.subscriberID, &livekit.TrackInfo{ Sid: string(s.trackID), @@ -523,14 +535,14 @@ func (m *SubscriptionManager) reconcileSubscription(s *mediaTrackSubscription) { // - ErrSubscriptionLimitExceeded: the participant have reached the limit of subscriptions, wait for the other subscription to be unsubscribed // We'll still log an event to reflect this in telemetry since it's been too long if s.durationSinceStart() > subscriptionTimeout { - s.maybeRecordError(m.params.TelemetryListener, err, true) + s.maybeRecordError(m.params.Participant.GetTelemetryListener(), err, true) } case ErrTrackNotFound: // source track was never published or closed // if after timeout we'd unsubscribe from it. // this is the *only* case we'd change desired state if s.durationSinceStart() > notFoundTimeout { - s.maybeRecordError(m.params.TelemetryListener, err, true) + s.maybeRecordError(m.params.Participant.GetTelemetryListener(), err, true) s.logger.Infow("unsubscribing from track after notFoundTimeout", "error", err) s.setDesired(false) m.queueReconcile(s.trackID) @@ -543,7 +555,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *mediaTrackSubscription) { "failed to subscribe, triggering error handler", err, "attempt", s.getNumAttempts(), ) - s.maybeRecordError(m.params.TelemetryListener, err, false) + s.maybeRecordError(m.params.Participant.GetTelemetryListener(), err, false) m.params.OnSubscriptionError(s.trackID, true, err) } else { s.logger.Debugw( @@ -582,7 +594,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *mediaTrackSubscription) { wait := min(time.Since(activeAt), s.durationSinceStart()) if wait > subscriptionTimeout { s.logger.Warnw("track not bound after timeout", nil) - s.maybeRecordError(m.params.TelemetryListener, ErrTrackNotBound, false) + s.maybeRecordError(m.params.Participant.GetTelemetryListener(), ErrTrackNotBound, false) m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) } } @@ -600,7 +612,7 @@ func (m *SubscriptionManager) reconcileDataTrackSubscriptions() { var needsToReconcile []*dataTrackSubscription m.lock.RLock() for _, sub := range m.dataTrackSubscriptions { - if sub.needsSubscribe() || sub.needsUnsubscribe() { + if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsCleanup() { needsToReconcile = append(needsToReconcile, sub) } } @@ -631,7 +643,7 @@ func (m *SubscriptionManager) reconcileDataTrackSubscription(s *dataTrackSubscri if s.durationSinceStart() > notFoundTimeout { s.logger.Infow("unsubscribing from data track after notFoundTimeout", "error", err) s.setDesired(false) - m.queueReconcile(s.trackID) + m.queueReconcileDataTrack(s.trackID) } default: // all other errors @@ -669,12 +681,16 @@ func (m *SubscriptionManager) reconcileDataTrackSubscription(s *dataTrackSubscri } m.lock.Lock() - if s.needsCleanup() { + cleanedUp := s.needsCleanup() + if cleanedUp { s.logger.Debugw("cleanup removing data track subscription") delete(m.dataTrackSubscriptions, s.trackID) - m.notifyDataTrackSubscriberHandles() } m.lock.Unlock() + + if cleanedUp { + m.notifyDataTrackSubscriberHandles() + } } // trigger an immediate reconciliation, when trackID is empty, will reconcile all subscriptions @@ -873,13 +889,13 @@ func (m *SubscriptionManager) addSubscriber(sub *mediaTrackSubscription, track t subTrack.AddOnBind(func(err error) { if err != nil { sub.logger.Infow("failed to bind track", "err", err) - sub.maybeRecordError(m.params.TelemetryListener, err, true) + sub.maybeRecordError(m.params.Participant.GetTelemetryListener(), err, true) m.UnsubscribeFromTrack(trackID) m.params.OnSubscriptionError(trackID, false, err) return } sub.setBound() - sub.maybeRecordSuccess(m.params.TelemetryListener) + sub.maybeRecordSuccess(m.params.Participant.GetTelemetryListener()) }) sub.setSubscribedTrack(subTrack) @@ -1003,7 +1019,7 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *mediaTrackSubscripti // * the participant isn't closing // * it's not a migration if wasBound { - m.params.TelemetryListener.OnTrackUnsubscribed( + m.params.Participant.GetTelemetryListener().OnTrackUnsubscribed( s.subscriberID, &livekit.TrackInfo{Sid: string(s.trackID), Type: subTrack.MediaTrack().Kind()}, !isExpectedToResume, @@ -1013,7 +1029,7 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *mediaTrackSubscripti if dt != nil { stats := dt.GetTrackStats() if stats != nil { - m.params.TelemetryListener.OnTrackSubscribeRTPStats( + m.params.Participant.GetTelemetryListener().OnTrackSubscribeRTPStats( s.subscriberID, s.trackID, dt.Mime(), @@ -1223,7 +1239,7 @@ func (m *SubscriptionManager) unmarkSubscribedTo(publisherID livekit.Participant } m.lock.Unlock() if changedCB != nil && lastSubscription { - go changedCB(publisherID, false) + changedCB(publisherID, false) } } diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 6c4618927..15ecc1096 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -82,7 +82,7 @@ func TestSubscribe(t *testing.T) { require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent - tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + tl := sm.params.Participant.GetTelemetryListener().(*typesfakes.FakeParticipantTelemetryListener) require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) // ensure bound @@ -113,7 +113,9 @@ func TestSubscribe(t *testing.T) { require.Eventually(t, func() bool { return numParticipantSubscribed.Load() == 2 }, subSettleTimeout, subCheckInterval, "participant subscribe status was not updated twice") - require.Equal(t, int32(1), numParticipantUnsubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantUnsubscribed.Load() == 1 + }, subSettleTimeout, subCheckInterval, "participant unsubscribe status was not updated") }) t.Run("no track permission", func(t *testing.T) { @@ -141,7 +143,7 @@ func TestSubscribe(t *testing.T) { require.Len(t, sm.GetSubscribedTracks(), 0) // trackSubscribed telemetry not sent - tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + tl := sm.params.Participant.GetTelemetryListener().(*typesfakes.FakeParticipantTelemetryListener) require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) require.Equal(t, 0, tl.OnTrackSubscribedCallCount()) @@ -248,9 +250,12 @@ func TestUnsubscribe(t *testing.T) { // no traces should be left require.Len(t, sm.GetSubscribedTracks(), 0) - require.False(t, res.TrackChangedNotifier.HasObservers()) + // the observer is dropped on a goroutine of its own + require.Eventually(t, func() bool { + return !res.TrackChangedNotifier.HasObservers() + }, subSettleTimeout, subCheckInterval, "observer was not removed") - tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + tl := sm.params.Participant.GetTelemetryListener().(*typesfakes.FakeParticipantTelemetryListener) require.Equal(t, 1, tl.OnTrackUnsubscribedCallCount()) } @@ -390,7 +395,7 @@ func TestSubscriptionLimits(t *testing.T) { require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent - tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + tl := sm.params.Participant.GetTelemetryListener().(*typesfakes.FakeParticipantTelemetryListener) require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) // ensure bound @@ -521,6 +526,33 @@ func TestSubscribeDataTrack(t *testing.T) { }, subSettleTimeout, subCheckInterval, "should be resubscribed") require.Equal(t, 2, resolver.dataTrack.AddSubscriberCallCount()) }) + + t.Run("unsubscribe before data track resolves", func(t *testing.T) { + sm := newTestSubscriptionManager() + defer sm.Close(false) + // no track available, subscribe attempts fail with ErrTrackNotFound + resolver := newTestDataTrackResolver(true, false, "pub", "pubID") + sm.params.DataTrackResolver = resolver.Resolve + + sm.SubscribeToDataTrack("track") + sm.lock.RLock() + s := sm.dataTrackSubscriptions["track"] + sm.lock.RUnlock() + require.NotNil(t, s) + + // let the worker attempt (and fail) the subscribe + require.Eventually(t, func() bool { + return s.getNumAttempts() > 0 + }, subSettleTimeout, subCheckInterval, "no subscribe attempt was made") + + sm.UnsubscribeFromDataTrack("track") + require.Eventually(t, func() bool { + sm.lock.RLock() + _, ok := sm.dataTrackSubscriptions["track"] + sm.lock.RUnlock() + return !ok + }, subSettleTimeout, subCheckInterval, "data track subscription was not cleaned up") + }) } type testSubscriptionParams struct { @@ -538,6 +570,10 @@ func newTestSubscriptionManagerWithParams(params testSubscriptionParams) *Subscr p.IDReturns("subID") p.IdentityReturns("sub") p.KindReturns(livekit.ParticipantInfo_STANDARD) + + tl := &typesfakes.FakeParticipantTelemetryListener{} + p.GetTelemetryListenerReturns(tl) + return NewSubscriptionManager(SubscriptionManagerParams{ Participant: p, Logger: logger.GetLogger(), @@ -547,7 +583,6 @@ func newTestSubscriptionManagerWithParams(params testSubscriptionParams) *Subscr TrackResolver: func(sub types.LocalParticipant, trackID livekit.TrackID) types.MediaResolverResult { return types.MediaResolverResult{} }, - TelemetryListener: &typesfakes.FakeParticipantTelemetryListener{}, SubscriptionLimitAudio: params.SubscriptionLimitAudio, SubscriptionLimitVideo: params.SubscriptionLimitVideo, }) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index cdd3eb6e1..2f9f7d440 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -223,14 +223,14 @@ type PCTransport struct { dataTrackDC *datachannel.DataChannelWriter[*webrtc.DataChannel] unlabeledDataChannels []*datachannel.DataChannelWriter[*webrtc.DataChannel] - iceStartedAt time.Time - iceConnectedAt time.Time - firstConnectedAt time.Time - connectedAt time.Time - tcpICETimer *time.Timer - connectAfterICETimer *time.Timer // timer to wait for pc to connect after ice connected - resetShortConnOnICERestart atomic.Bool - signalingRTT atomic.Uint32 // milliseconds + iceFirstStartedAt time.Time + iceFirstConnectedAt time.Time + peerConnectionFirstConnectedAt time.Time + peerConnectionLastconnectedAt time.Time + tcpICETimer *time.Timer + connectAfterICETimer *time.Timer // timer to wait for pc to connect after ice connected + resetShortConnOnICERestart atomic.Bool + signalingRTT atomic.Uint32 // milliseconds hasFullyEstablishedRecorded bool @@ -384,6 +384,7 @@ func newPeerConnection( } if params.EnableWarp { + params.Logger.Debugw("enable warp") se.EnableSped(true) se.EnableSctpSnap(true) } @@ -530,6 +531,8 @@ func newPeerConnection( params.Logger.Debugw("rtx pair found from extension", "repair", repair, "base", base, "rsid", rsid) params.Config.BufferFactory.SetRTXPair(repair, base, rsid) }, + params.Config.BufferFactory, + params.SimTracks, params.Logger, ) // put rtx interceptor behind unhandle simulcast interceptor so it can get the correct mid & rid @@ -670,8 +673,8 @@ func (t *PCTransport) SetSignalingRTT(rtt uint32) { func (t *PCTransport) setICEStartedAt(at time.Time) { t.lock.Lock() - if t.iceStartedAt.IsZero() { - t.iceStartedAt = at + if t.iceFirstStartedAt.IsZero() { + t.iceFirstStartedAt = at // checklist of ice agent will be cleared on ice failed, get stats before that t.mayFailedICEStatsTimer = time.AfterFunc(iceFailedTimeoutTotal-time.Second, t.logMayFailedICEStats) @@ -702,15 +705,15 @@ func (t *PCTransport) setICEStartedAt(at time.Time) { func (t *PCTransport) setICEConnectedAt(at time.Time) { t.lock.Lock() - if t.iceConnectedAt.IsZero() { + if t.iceFirstConnectedAt.IsZero() { // // Record initial connection time. - // This prevents reset of connected at time if ICE goes `Connected` -> `Disconnected` -> `Connected`. + // This prevents reset of iceFirstConnectedAt if ICE goes `Connected` -> `Disconnected` -> `Connected`. // - t.iceConnectedAt = at + t.iceFirstConnectedAt = at // set failure timer for dtls handshake - iceDuration := at.Sub(t.iceStartedAt) + iceDuration := at.Sub(t.iceFirstStartedAt) connTimeoutAfterICE := min(max(minConnectTimeoutAfterICE, 3*iceDuration), maxConnectTimeoutAfterICE) t.params.Logger.Debugw("setting connection timer after ICE connected", "timeout", connTimeoutAfterICE, "iceDuration", iceDuration) t.connectAfterICETimer = time.AfterFunc(connTimeoutAfterICE, func() { @@ -777,9 +780,9 @@ func (t *PCTransport) logMayFailedICEStats() { func (t *PCTransport) resetShortConn() { t.params.Logger.Infow("resetting short connection on ICE restart") t.lock.Lock() - t.iceStartedAt = time.Time{} - t.iceConnectedAt = time.Time{} - t.connectedAt = time.Time{} + t.iceFirstStartedAt = time.Time{} + t.iceFirstConnectedAt = time.Time{} + t.peerConnectionLastconnectedAt = time.Time{} if t.connectAfterICETimer != nil { t.connectAfterICETimer.Stop() t.connectAfterICETimer = nil @@ -795,23 +798,23 @@ func (t *PCTransport) IsShortConnection(at time.Time) (bool, time.Duration) { t.lock.RLock() defer t.lock.RUnlock() - if t.iceConnectedAt.IsZero() { + if t.iceFirstConnectedAt.IsZero() { return false, 0 } - duration := at.Sub(t.iceConnectedAt) + duration := at.Sub(t.iceFirstConnectedAt) return duration < shortConnectionThreshold, duration } -func (t *PCTransport) setConnectedAt(at time.Time) bool { +func (t *PCTransport) setPeerConnectionConnectedAt(at time.Time) bool { t.lock.Lock() - t.connectedAt = at - if !t.firstConnectedAt.IsZero() { + t.peerConnectionLastconnectedAt = at + if !t.peerConnectionFirstConnectedAt.IsZero() { t.lock.Unlock() return false } - t.firstConnectedAt = at + t.peerConnectionFirstConnectedAt = at prometheus.RecordServiceOperationSuccess("peer_connection") prometheus.RecordPeerConnectionState(t.params.Transport, "connected") t.lock.Unlock() @@ -865,7 +868,7 @@ func (t *PCTransport) onPeerConnectionStateChange(state webrtc.PeerConnectionSta switch state { case webrtc.PeerConnectionStateConnected: t.clearConnTimer() - isInitialConnection := t.setConnectedAt(time.Now()) + isInitialConnection := t.setPeerConnectionConnectedAt(time.Now()) if isInitialConnection { t.params.Handler.OnInitialConnected() @@ -993,7 +996,7 @@ func (t *PCTransport) isFullyEstablished() bool { dataChannelReady := t.params.UseOneShotSignallingMode || t.firstOfferNoDataChannel || (t.reliableDCOpened && t.lossyDCOpened) - return dataChannelReady && !t.connectedAt.IsZero() + return dataChannelReady && !t.peerConnectionLastconnectedAt.IsZero() } func (t *PCTransport) SetPreferTCP(preferTCP bool) { @@ -1440,18 +1443,25 @@ func (t *PCTransport) IsEstablished() bool { return t.pc.ConnectionState() != webrtc.PeerConnectionStateNew } -func (t *PCTransport) HasEverConnected() bool { +func (t *PCTransport) ICEHasEverConnected() bool { t.lock.RLock() defer t.lock.RUnlock() - return !t.firstConnectedAt.IsZero() + return !t.iceFirstConnectedAt.IsZero() } -func (t *PCTransport) FirstConnectedAt() time.Time { +func (t *PCTransport) PeerConnectionHasEverConnected() bool { t.lock.RLock() defer t.lock.RUnlock() - return t.firstConnectedAt + return !t.peerConnectionFirstConnectedAt.IsZero() +} + +func (t *PCTransport) PeerConnectionFirstConnectedAt() time.Time { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.peerConnectionFirstConnectedAt } func (t *PCTransport) GetICEConnectionInfo() *types.ICEConnectionInfo { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 9e8cf51b1..108651f21 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -232,12 +232,16 @@ func (t *TransportManager) SubscriberClose() { t.subscriber.Close() } +func (t *TransportManager) HasPublisherICEEverConnected() bool { + return t.publisher.ICEHasEverConnected() +} + func (t *TransportManager) HasPublisherEverConnected() bool { - return t.publisher.HasEverConnected() + return t.publisher.PeerConnectionHasEverConnected() } func (t *TransportManager) PublisherFirstConnectedAt() time.Time { - return t.publisher.FirstConnectedAt() + return t.publisher.PeerConnectionFirstConnectedAt() } func (t *TransportManager) IsPublisherEstablished() bool { @@ -272,19 +276,27 @@ func (t *TransportManager) GetSubscriberRTT() (float64, bool) { } } +func (t *TransportManager) HasSubscriberICEEverConnected() bool { + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.ICEHasEverConnected() + } else { + return t.subscriber.ICEHasEverConnected() + } +} + func (t *TransportManager) HasSubscriberEverConnected() bool { if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { - return t.publisher.HasEverConnected() + return t.publisher.PeerConnectionHasEverConnected() } else { - return t.subscriber.HasEverConnected() + return t.subscriber.PeerConnectionHasEverConnected() } } func (t *TransportManager) SubscriberFirstConnectedAt() time.Time { if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { - return t.publisher.FirstConnectedAt() + return t.publisher.PeerConnectionFirstConnectedAt() } else { - return t.subscriber.FirstConnectedAt() + return t.subscriber.PeerConnectionFirstConnectedAt() } } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index ddc462e1f..fa8e260b8 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -30,11 +30,11 @@ import ( "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/datatrack" "google.golang.org/protobuf/proto" ) @@ -227,6 +227,8 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { } } +// --------------------------------------------- + // IsIntentionalDisconnect reports whether a disconnect reason represents an // intentional/expected closure (client leaving, admin action, room teardown, // migration, etc.) as opposed to a connection failure. @@ -292,6 +294,7 @@ func (s SignallingCloseReason) String() string { } // --------------------------------------------- + const ( ParticipantCloseKeyNormal = "normal" ParticipantCloseKeyWHIP = "whip" @@ -299,6 +302,70 @@ const ( // --------------------------------------------- +type RoomCloseReason int + +const ( + RoomCloseReasonUnknown RoomCloseReason = iota + RoomCloseReasonAPIDelete + RoomCloseReasonIdleTimeout + RoomCloseReasonServerShutdown + RoomCloseReasonSuperseded + RoomCloseReasonOpenFailed +) + +func (r RoomCloseReason) String() string { + switch r { + case RoomCloseReasonUnknown: + return "UNKNOWN" + case RoomCloseReasonAPIDelete: + return "API_DELETE" + case RoomCloseReasonIdleTimeout: + return "IDLE_TIMEOUT" + case RoomCloseReasonServerShutdown: + return "SERVER_SHUTDOWN" + case RoomCloseReasonSuperseded: + return "SUPERSEDED" + case RoomCloseReasonOpenFailed: + return "OPEN_FAILED" + default: + return fmt.Sprintf("%d", int(r)) + } +} + +func (r RoomCloseReason) ToProto() livekit.RoomEndReason { + switch r { + case RoomCloseReasonAPIDelete: + return livekit.RoomEndReason_ROOM_END_API_DELETE + case RoomCloseReasonIdleTimeout: + return livekit.RoomEndReason_ROOM_END_IDLE_TIMEOUT + case RoomCloseReasonServerShutdown: + return livekit.RoomEndReason_ROOM_END_SERVER_SHUTDOWN + case RoomCloseReasonSuperseded: + return livekit.RoomEndReason_ROOM_END_SUPERSEDED + case RoomCloseReasonOpenFailed: + return livekit.RoomEndReason_ROOM_END_OPEN_FAILED + default: + return livekit.RoomEndReason_ROOM_END_UNKNOWN + } +} + +// ToParticipantCloseReason gives the reason participants are closed with when the +// room closes for this reason, so the two can never disagree. +func (r RoomCloseReason) ToParticipantCloseReason() ParticipantCloseReason { + switch r { + case RoomCloseReasonAPIDelete: + return ParticipantCloseReasonServiceRequestDeleteRoom + case RoomCloseReasonIdleTimeout, RoomCloseReasonSuperseded: + return ParticipantCloseReasonRoomClosed + case RoomCloseReasonServerShutdown: + return ParticipantCloseReasonRoomManagerStop + default: + return ParticipantCloseReasonNone + } +} + +// --------------------------------------------- + //counterfeiter:generate . Participant type Participant interface { ID() livekit.ParticipantID @@ -370,10 +437,11 @@ type AddTrackParams struct { } type MoveToRoomParams struct { - RoomName livekit.RoomName - ParticipantID livekit.ParticipantID - Listener LocalParticipantListener - Helper LocalParticipantHelper + RoomName livekit.RoomName + ParticipantID livekit.ParticipantID + Listener LocalParticipantListener + TelemetryListener ParticipantTelemetryListener + Helper LocalParticipantHelper } type DataMessageCache struct { @@ -425,6 +493,7 @@ type LocalParticipant interface { GetPlayoutDelayConfig() *livekit.PlayoutDelay GetPendingTrack(trackID livekit.TrackID) *livekit.TrackInfo GetICEConnectionInfo() []*ICEConnectionInfo + HasICEConnected() bool HasConnected() bool GetEnabledPublishCodecs() []*livekit.Codec GetPublisherICESessionUfrag() (string, error) @@ -479,6 +548,7 @@ type LocalParticipant interface { UnsubscribeFromTrack(trackID livekit.TrackID) UpdateSubscribedTrackSettings(trackID livekit.TrackID, settings *livekit.UpdateTrackSettings) GetSubscribedTracks() []SubscribedTrack + GetSubscribedDataTracks() []DataDownTrack IsTrackNameSubscribed(publisherIdentity livekit.ParticipantIdentity, trackName string) bool SubscribeToDataTrack(trackID livekit.TrackID) UnsubscribeFromDataTrack(trackID livekit.TrackID) diff --git a/pkg/rtc/types/typesfakes/fake_data_track.go b/pkg/rtc/types/typesfakes/fake_data_track.go index 07a49f9b1..ddd512d92 100644 --- a/pkg/rtc/types/typesfakes/fake_data_track.go +++ b/pkg/rtc/types/typesfakes/fake_data_track.go @@ -4,8 +4,8 @@ package typesfakes import ( "sync" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/types/typesfakes/fake_data_track_sender.go b/pkg/rtc/types/typesfakes/fake_data_track_sender.go index a9a69486a..f45c28e82 100644 --- a/pkg/rtc/types/typesfakes/fake_data_track_sender.go +++ b/pkg/rtc/types/typesfakes/fake_data_track_sender.go @@ -4,8 +4,8 @@ package typesfakes import ( "sync" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index d4fcfd21b..b6f349e70 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -6,13 +6,13 @@ import ( "time" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/observability/roomobs" @@ -557,6 +557,16 @@ type FakeLocalParticipant struct { getResponseSinkReturnsOnCall map[int]struct { result1 routing.MessageSink } + GetSubscribedDataTracksStub func() []types.DataDownTrack + getSubscribedDataTracksMutex sync.RWMutex + getSubscribedDataTracksArgsForCall []struct { + } + getSubscribedDataTracksReturns struct { + result1 []types.DataDownTrack + } + getSubscribedDataTracksReturnsOnCall map[int]struct { + result1 []types.DataDownTrack + } GetSubscribedParticipantsStub func() []livekit.ParticipantID getSubscribedParticipantsMutex sync.RWMutex getSubscribedParticipantsArgsForCall []struct { @@ -774,6 +784,16 @@ type FakeLocalParticipant struct { hasConnectedReturnsOnCall map[int]struct { result1 bool } + HasICEConnectedStub func() bool + hasICEConnectedMutex sync.RWMutex + hasICEConnectedArgsForCall []struct { + } + hasICEConnectedReturns struct { + result1 bool + } + hasICEConnectedReturnsOnCall map[int]struct { + result1 bool + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -4403,6 +4423,59 @@ func (fake *FakeLocalParticipant) GetResponseSinkReturnsOnCall(i int, result1 ro }{result1} } +func (fake *FakeLocalParticipant) GetSubscribedDataTracks() []types.DataDownTrack { + fake.getSubscribedDataTracksMutex.Lock() + ret, specificReturn := fake.getSubscribedDataTracksReturnsOnCall[len(fake.getSubscribedDataTracksArgsForCall)] + fake.getSubscribedDataTracksArgsForCall = append(fake.getSubscribedDataTracksArgsForCall, struct { + }{}) + stub := fake.GetSubscribedDataTracksStub + fakeReturns := fake.getSubscribedDataTracksReturns + fake.recordInvocation("GetSubscribedDataTracks", []interface{}{}) + fake.getSubscribedDataTracksMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetSubscribedDataTracksCallCount() int { + fake.getSubscribedDataTracksMutex.RLock() + defer fake.getSubscribedDataTracksMutex.RUnlock() + return len(fake.getSubscribedDataTracksArgsForCall) +} + +func (fake *FakeLocalParticipant) GetSubscribedDataTracksCalls(stub func() []types.DataDownTrack) { + fake.getSubscribedDataTracksMutex.Lock() + defer fake.getSubscribedDataTracksMutex.Unlock() + fake.GetSubscribedDataTracksStub = stub +} + +func (fake *FakeLocalParticipant) GetSubscribedDataTracksReturns(result1 []types.DataDownTrack) { + fake.getSubscribedDataTracksMutex.Lock() + defer fake.getSubscribedDataTracksMutex.Unlock() + fake.GetSubscribedDataTracksStub = nil + fake.getSubscribedDataTracksReturns = struct { + result1 []types.DataDownTrack + }{result1} +} + +func (fake *FakeLocalParticipant) GetSubscribedDataTracksReturnsOnCall(i int, result1 []types.DataDownTrack) { + fake.getSubscribedDataTracksMutex.Lock() + defer fake.getSubscribedDataTracksMutex.Unlock() + fake.GetSubscribedDataTracksStub = nil + if fake.getSubscribedDataTracksReturnsOnCall == nil { + fake.getSubscribedDataTracksReturnsOnCall = make(map[int]struct { + result1 []types.DataDownTrack + }) + } + fake.getSubscribedDataTracksReturnsOnCall[i] = struct { + result1 []types.DataDownTrack + }{result1} +} + func (fake *FakeLocalParticipant) GetSubscribedParticipants() []livekit.ParticipantID { fake.getSubscribedParticipantsMutex.Lock() ret, specificReturn := fake.getSubscribedParticipantsReturnsOnCall[len(fake.getSubscribedParticipantsArgsForCall)] @@ -5618,6 +5691,59 @@ func (fake *FakeLocalParticipant) HasConnectedReturnsOnCall(i int, result1 bool) }{result1} } +func (fake *FakeLocalParticipant) HasICEConnected() bool { + fake.hasICEConnectedMutex.Lock() + ret, specificReturn := fake.hasICEConnectedReturnsOnCall[len(fake.hasICEConnectedArgsForCall)] + fake.hasICEConnectedArgsForCall = append(fake.hasICEConnectedArgsForCall, struct { + }{}) + stub := fake.HasICEConnectedStub + fakeReturns := fake.hasICEConnectedReturns + fake.recordInvocation("HasICEConnected", []interface{}{}) + fake.hasICEConnectedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HasICEConnectedCallCount() int { + fake.hasICEConnectedMutex.RLock() + defer fake.hasICEConnectedMutex.RUnlock() + return len(fake.hasICEConnectedArgsForCall) +} + +func (fake *FakeLocalParticipant) HasICEConnectedCalls(stub func() bool) { + fake.hasICEConnectedMutex.Lock() + defer fake.hasICEConnectedMutex.Unlock() + fake.HasICEConnectedStub = stub +} + +func (fake *FakeLocalParticipant) HasICEConnectedReturns(result1 bool) { + fake.hasICEConnectedMutex.Lock() + defer fake.hasICEConnectedMutex.Unlock() + fake.HasICEConnectedStub = nil + fake.hasICEConnectedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) HasICEConnectedReturnsOnCall(i int, result1 bool) { + fake.hasICEConnectedMutex.Lock() + defer fake.hasICEConnectedMutex.Unlock() + fake.HasICEConnectedStub = nil + if fake.hasICEConnectedReturnsOnCall == nil { + fake.hasICEConnectedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.hasICEConnectedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] diff --git a/pkg/rtc/types/typesfakes/fake_local_participant_listener.go b/pkg/rtc/types/typesfakes/fake_local_participant_listener.go index 4c0a3a07a..7f6912626 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant_listener.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant_listener.go @@ -4,8 +4,8 @@ package typesfakes import ( "sync" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 652c8bf14..df9f2255d 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -5,8 +5,8 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" diff --git a/pkg/rtc/types/typesfakes/fake_participant_listener.go b/pkg/rtc/types/typesfakes/fake_participant_listener.go index a05a3e0f5..ac1b05bea 100644 --- a/pkg/rtc/types/typesfakes/fake_participant_listener.go +++ b/pkg/rtc/types/typesfakes/fake_participant_listener.go @@ -4,8 +4,8 @@ package typesfakes import ( "sync" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/updatatrackmanager.go b/pkg/rtc/updatatrackmanager.go index c3af8dc50..dbf6c5a32 100644 --- a/pkg/rtc/updatatrackmanager.go +++ b/pkg/rtc/updatatrackmanager.go @@ -19,8 +19,8 @@ import ( "slices" "sync" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 4ff4da723..c9b9f4c30 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -184,7 +184,7 @@ func (r *WrappedReceiver) AddOnReady(f func()) { type DummyReceiver struct { receiver atomic.Value - trackID livekit.TrackID + trackInfo *livekit.TrackInfo streamId string codec webrtc.RTPCodecParameters headerExtensions []webrtc.RTPHeaderExtensionParameter @@ -201,9 +201,14 @@ type DummyReceiver struct { redReceiver, primaryReceiver *DummyRedReceiver } -func NewDummyReceiver(trackID livekit.TrackID, streamId string, codec webrtc.RTPCodecParameters, headerExtensions []webrtc.RTPHeaderExtensionParameter) *DummyReceiver { +func NewDummyReceiver( + trackInfo *livekit.TrackInfo, + streamId string, + codec webrtc.RTPCodecParameters, + headerExtensions []webrtc.RTPHeaderExtensionParameter, +) *DummyReceiver { return &DummyReceiver{ - trackID: trackID, + trackInfo: trackInfo, streamId: streamId, codec: codec, headerExtensions: headerExtensions, @@ -262,7 +267,7 @@ func (d *DummyReceiver) Upgrade(receiver sfu.TrackReceiver) { } func (d *DummyReceiver) TrackID() livekit.TrackID { - return d.trackID + return livekit.TrackID(d.trackInfo.Sid) } func (d *DummyReceiver) StreamID() string { @@ -391,7 +396,7 @@ func (d *DummyReceiver) TrackInfo() *livekit.TrackInfo { if receiver := d.getReceiver(); receiver != nil { return receiver.TrackInfo() } - return nil + return d.trackInfo } func (d *DummyReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { diff --git a/pkg/service/docker_test.go b/pkg/service/docker_test.go index 9fc5b90bb..df25e4322 100644 --- a/pkg/service/docker_test.go +++ b/pkg/service/docker_test.go @@ -16,6 +16,7 @@ package service_test import ( "context" + "flag" "fmt" "log" "net" @@ -31,24 +32,43 @@ import ( var Docker dockertest.ClosablePool -func TestMain(m *testing.M) { - ctx := context.Background() - pool, err := dockertest.NewPool(ctx, "") - if err != nil { - log.Fatalf("Could not construct pool: %s", err) - } +// go test -docker=false ./pkg/service skips the tests that need a docker +// daemon, for a checkout without one. Running them is the default: a run that +// quietly covers less than the last one is worse than a run that stops, so a +// daemon that should be there and is not still fails the whole package. +var useDocker = flag.Bool("docker", true, "run the tests that need a docker daemon") - // uses pool to try to connect to Docker - _, err = pool.Client().Ping(ctx, mobyclient.PingOptions{}) - if err != nil { - log.Fatalf("Could not connect to Docker: %s", err) +func TestMain(m *testing.M) { + // m.Run would parse them, but the flag is read before that + flag.Parse() + + if *useDocker { + ctx := context.Background() + pool, err := dockertest.NewPool(ctx, "") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + // uses pool to try to connect to Docker + _, err = pool.Client().Ping(ctx, mobyclient.PingOptions{}) + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + Docker = pool } - Docker = pool code := m.Run() os.Exit(code) } +func requireDocker(t testing.TB) { + t.Helper() + + if !*useDocker { + t.Skip("this test needs a docker daemon, and -docker=false says there is none") + } +} + func waitTCPPort(t testing.TB, addr string) { if err := Docker.Retry(t.Context(), 30*time.Second, func() error { conn, err := net.Dial("tcp", addr) @@ -66,6 +86,8 @@ func waitTCPPort(t testing.TB, addr string) { var redisLast atomic.Uint32 func runRedis(t testing.TB) string { + requireDocker(t) + c, err := Docker.Run(t.Context(), "redis", dockertest.WithName(fmt.Sprintf("lktest-redis-%d", redisLast.Inc())), diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 4036b0e1e..528309c68 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -19,8 +19,6 @@ import ( "fmt" "net/url" - "github.com/livekit/livekit-server/pkg/config" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -28,8 +26,12 @@ import ( "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" "github.com/livekit/psrpc" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/telemetry" ) +//counterfeiter:generate . IngressLauncher type IngressLauncher interface { LaunchPullIngress(ctx context.Context, info *livekit.IngressInfo) (*livekit.IngressInfo, error) } @@ -133,20 +135,27 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string if err != nil { return nil, psrpc.NewError(psrpc.InvalidArgument, err) } - if urlObj.Scheme != "http" && urlObj.Scheme != "https" && urlObj.Scheme != "srt" { + switch urlObj.Scheme { + case "http", "https", "srt": + case "udp": + if !s.conf.EnableUDPURLPull { + return nil, ingress.ErrInvalidIngress("udp url pull is not enabled") + } + default: return nil, ingress.ErrInvalidIngress(fmt.Sprintf("invalid url scheme %s", urlObj.Scheme)) } // Marshall the URL again for sanitization urlStr = urlObj.String() } + reqID := RequestID(ctx) var sk string if req.InputType != livekit.IngressInput_URL_INPUT { sk = guid.New("") } info := &livekit.IngressInfo{ - IngressId: guid.New(utils.IngressPrefix), + IngressId: DeterministicID(utils.IngressPrefix, reqID), Name: req.Name, StreamKey: sk, Url: urlStr, @@ -192,11 +201,13 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string } // The Ingress instance will create the ingress object when handling the URL pull ingress } else { - // TODO-jie: ingress retry idempotency: generate ingress key by request-id, and return the ingress object from CreateIngress. - _, err = s.io.CreateIngress(ctx, info) + var resp *rpc.CreateIngressResponse + resp, err = s.io.CreateIngress(ctx, info) switch err { case nil: - break + if resp.GetInfo() != nil { + info = resp.GetInfo() + } case ingress.ErrIngressOutOfDate: // Error returned if the ingress was already created by the ingress service err = nil diff --git a/pkg/service/ingress_test.go b/pkg/service/ingress_test.go new file mode 100644 index 000000000..86d29d2e2 --- /dev/null +++ b/pkg/service/ingress_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/livekit" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/livekit-server/pkg/service/servicefakes" +) + +func TestCreateURLPullIngressScheme(t *testing.T) { + newService := func(enableUDP bool) (*service.IngressService, *servicefakes.FakeIngressLauncher) { + launcher := &servicefakes.FakeIngressLauncher{} + launcher.LaunchPullIngressCalls(func(_ context.Context, info *livekit.IngressInfo) (*livekit.IngressInfo, error) { + return info, nil + }) + + svc := service.NewIngressServiceWithIngressLauncher( + &config.IngressConfig{EnableUDPURLPull: enableUDP}, + "nodeID", + nil, + nil, + &servicefakes.FakeIngressStore{}, + nil, + nil, + launcher, + ) + return svc, launcher + } + + adminCtx := func() context.Context { + return service.WithGrants(context.Background(), &auth.ClaimGrants{Video: &auth.VideoGrant{IngressAdmin: true}}, "") + } + + createReq := func(url string) *livekit.CreateIngressRequest { + return &livekit.CreateIngressRequest{ + InputType: livekit.IngressInput_URL_INPUT, + Url: url, + RoomName: "testroom", + ParticipantIdentity: "ingress", + } + } + + t.Run("udp rejected when disabled", func(t *testing.T) { + svc, launcher := newService(false) + + _, err := svc.CreateIngress(adminCtx(), createReq("udp://1.2.3.4:1234")) + require.Error(t, err) + require.Contains(t, err.Error(), "udp url pull is not enabled") + require.Zero(t, launcher.LaunchPullIngressCallCount()) + }) + + t.Run("udp accepted when enabled", func(t *testing.T) { + svc, launcher := newService(true) + + info, err := svc.CreateIngress(adminCtx(), createReq("udp://1.2.3.4:1234")) + require.NoError(t, err) + require.Equal(t, "udp://1.2.3.4:1234", info.Url) + require.Equal(t, 1, launcher.LaunchPullIngressCallCount()) + }) + + t.Run("other schemes unaffected by the udp option", func(t *testing.T) { + for _, url := range []string{"http://example.com/live", "https://example.com/live", "srt://1.2.3.4:1234"} { + svc, _ := newService(false) + + info, err := svc.CreateIngress(adminCtx(), createReq(url)) + require.NoError(t, err, url) + require.Equal(t, url, info.Url) + } + + svc, _ := newService(true) + _, err := svc.CreateIngress(adminCtx(), createReq("rtsp://1.2.3.4/live")) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid url scheme rtsp") + }) +} diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 57282b3c3..e25fae883 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -246,7 +246,7 @@ func (r *RoomManager) Stop() { r.lock.RUnlock() for _, room := range rooms { - room.Close(types.ParticipantCloseReasonRoomManagerStop) + room.Close(types.RoomCloseReasonServerShutdown) } r.roomManagerServer.Kill() @@ -692,12 +692,12 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, createRoom *livekit.C return nil, err } - newRoom.OnClose(func() { + newRoom.OnClose(func(reason types.RoomCloseReason) { killRoomServer() killDispServer() roomInfo := newRoom.ToProto() - r.telemetry.RoomEnded(ctx, roomInfo) + r.telemetry.RoomEnded(ctx, roomInfo, reason.ToProto()) prometheus.RoomEnded(time.Unix(roomInfo.CreationTime, 0)) if err := r.deleteRoom(ctx, roomName); err != nil { newRoom.Logger().Errorw("could not delete room", err) @@ -943,7 +943,7 @@ func (r *RoomManager) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq } } else { room.Logger().Infow("deleting room") - room.Close(types.ParticipantCloseReasonServiceRequestDeleteRoom) + room.Close(types.RoomCloseReasonAPIDelete) } return &livekit.DeleteRoomResponse{}, nil } diff --git a/pkg/service/roommanager_service.go b/pkg/service/roommanager_service.go index d8ea05433..d6ead74ad 100644 --- a/pkg/service/roommanager_service.go +++ b/pkg/service/roommanager_service.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -281,13 +282,25 @@ type whipParticipantService struct { *RoomManager } +func getWHIPParticipant( + room *rtc.Room, + identity livekit.ParticipantIdentity, + participantID livekit.ParticipantID, +) types.LocalParticipant { + lp := room.GetParticipant(identity) + if lp == nil || lp.ID() != participantID { + return nil + } + return lp +} + func (r whipParticipantService) ICETrickle(ctx context.Context, req *rpc.WHIPParticipantICETrickleRequest) (*emptypb.Empty, error) { room := r.RoomManager.GetRoom(ctx, livekit.RoomName(req.Room)) if room == nil { return nil, ErrRoomNotFound } - lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + lp := getWHIPParticipant(room, livekit.ParticipantIdentity(req.GetParticipantIdentity()), livekit.ParticipantID(req.GetParticipantId())) if lp == nil { return nil, ErrParticipantNotFound } @@ -318,7 +331,7 @@ func (r whipParticipantService) ICERestart(ctx context.Context, req *rpc.WHIPPar return nil, ErrRoomNotFound } - lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + lp := getWHIPParticipant(room, livekit.ParticipantIdentity(req.GetParticipantIdentity()), livekit.ParticipantID(req.GetParticipantId())) if lp == nil { return nil, ErrParticipantNotFound } @@ -346,7 +359,7 @@ func (r whipParticipantService) DeleteSession(ctx context.Context, req *rpc.WHIP return nil, ErrRoomNotFound } - lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + lp := getWHIPParticipant(room, livekit.ParticipantIdentity(req.GetParticipantIdentity()), livekit.ParticipantID(req.GetParticipantId())) if lp != nil { room.RemoveParticipant( lp.Identity(), diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 520bec369..b111e4108 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -44,6 +44,17 @@ import ( "github.com/livekit/livekit-server/pkg/utils" ) +const ( + // how long the response source is drained after the request direction is gone, + // applies only when the source is not closed by the relay, i. e. when there is + // no way to tell that everything pending has been read + responseFlushTimeout = 250 * time.Millisecond + + // how long the response pump is given to stop, a bit more than the drain deadline + // so that it can finish the write it is in when that deadline expires + responsePumpDoneTimeout = 2 * responseFlushTimeout +) + type RTCService struct { router routing.MessageRouter roomAllocator RoomAllocator @@ -381,7 +392,7 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ var cr connectionResult var initialResponse *livekit.SignalResponse for attempt := 0; attempt < s.config.SignalRelay.ConnectAttempts; attempt++ { - connectionTimeout := 3 * time.Second * time.Duration(attempt+1) + connectionTimeout := time.Duration(3+attempt) * time.Second ctx := utils.ContextWithAttempt(r.Context(), attempt) cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) if err == nil || errors.Is(err, context.Canceled) { @@ -428,13 +439,34 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ closedByClient := atomic.NewBool(false) done := make(chan struct{}) + // closed by the response pump when it has stopped writing to the web socket + responsePumpDone := make(chan struct{}) + responsePumpStarted := false + var sigConn *WSSignalConnection // function exits when websocket terminates, it'll close the event reading off of request sink and response source as well defer func() { resolveLogger(true) pLogger.Debugw("finishing WS connection", "closedByClient", closedByClient.Load()) - cr.ResponseSource.Close() - cr.RequestSink.Close() + + // signal the response pump before anything else so that it can flush responses + // the participant queued on its way out, a leave request sent just before the + // signalling connection was closed (on migration for example) is dropped otherwise close(done) + cr.RequestSink.Close() + if responsePumpStarted { + select { + case <-responsePumpDone: + case <-time.After(responsePumpDoneTimeout): + pLogger.Debugw("timed out waiting for response pump to finish") + } + } + cr.ResponseSource.Close() + + // close the web socket on all paths, even when the response pump is wedged + // writing to an unresponsive client + if sigConn != nil { + sigConn.CloseWithReason("") + } signalStats.Stop() }() @@ -466,8 +498,7 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ }() // websocket established - sigConn := NewWSSignalConnection(conn, s.limits.SignalMessageSizeLimit) - defer sigConn.CloseWithReason("") + sigConn = NewWSSignalConnection(conn, s.limits.SignalMessageSizeLimit) pLogger.Debugw("sending initial response", "response", logger.Proto(initialResponse)) count, err := sigConn.WriteResponse(initialResponse) if err != nil { @@ -491,9 +522,79 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ "nodeSelectionReason", cr.NodeSelectionReason, ) + // writes one response from the response source to the web socket, + // returns false if the response pump should stop + writeResponse := func(msg proto.Message) bool { + res, ok := msg.(*livekit.SignalResponse) + if !ok { + pLogger.Errorw( + "unexpected message type", nil, + "type", fmt.Sprintf("%T", msg), + ) + return true + } + + switch m := res.Message.(type) { + case *livekit.SignalResponse_Offer: + pLogger.Debugw("sending offer", "offer", logger.Proto(res)) + + case *livekit.SignalResponse_Answer: + pLogger.Debugw("sending answer", "answer", logger.Proto(res)) + + case *livekit.SignalResponse_Join: + pLogger.Debugw("sending join", "join", logger.Proto(res)) + signalStats.ResolveRoom(m.Join.GetRoom()) + signalStats.ResolveParticipant(m.Join.GetParticipant()) + + case *livekit.SignalResponse_RoomUpdate: + updateRoomID := livekit.RoomID(m.RoomUpdate.GetRoom().GetSid()) + if updateRoomID != "" { + roomID = updateRoomID + resolveLogger(false) + } + pLogger.Debugw("sending room update", "roomUpdate", logger.Proto(res)) + signalStats.ResolveRoom(m.RoomUpdate.GetRoom()) + + case *livekit.SignalResponse_Update: + pLogger.Debugw("sending participant update", "participantUpdate", logger.Proto(res)) + + case *livekit.SignalResponse_RoomMoved: + resetLogger() + signalStats.Reset() + + roomName = livekit.RoomName(m.RoomMoved.GetRoom().GetName()) + moveRoomID := livekit.RoomID(m.RoomMoved.GetRoom().GetSid()) + if moveRoomID != "" { + roomID = moveRoomID + } + participantIdentity = livekit.ParticipantIdentity(m.RoomMoved.GetParticipant().GetIdentity()) + pID = livekit.ParticipantID(m.RoomMoved.GetParticipant().GetSid()) + resolveLogger(false) + + signalStats.ResolveRoom(m.RoomMoved.GetRoom()) + signalStats.ResolveParticipant(m.RoomMoved.GetParticipant()) + pLogger.Debugw("sending room moved", "roomMoved", logger.Proto(res)) + + default: + pLogger.Debugw("sending signal response", "response", logger.Proto(res)) + } + + if count, err := sigConn.WriteResponse(res); err != nil { + pLogger.Warnw("error writing to websocket", err) + return false + } else { + signalStats.AddBytes(uint64(count), true) + } + + return true + } + // handle responses + responsePumpStarted = true go func() { defer func() { + close(responsePumpDone) + // when the source is terminated, this means Participant.Close had been called and RTC connection is done // we would terminate the signal connection as well sigConn.CloseWithReason("") @@ -506,72 +607,23 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ for { select { case <-done: + // the request direction is gone, flush what the participant queued on its + // way out unless the client is the one that went away + if !closedByClient.Load() { + if !drainMessageSource(cr.ResponseSource, responseFlushTimeout, writeResponse) { + pLogger.Debugw("could not drain response source fully") + } + } return + case msg := <-cr.ResponseSource.ReadChan(): if msg == nil { resolveLogger(true) pLogger.Debugw("nothing to read from response source") return } - res, ok := msg.(*livekit.SignalResponse) - if !ok { - pLogger.Errorw( - "unexpected message type", nil, - "type", fmt.Sprintf("%T", msg), - ) - continue - } - - switch m := res.Message.(type) { - case *livekit.SignalResponse_Offer: - pLogger.Debugw("sending offer", "offer", logger.Proto(res)) - - case *livekit.SignalResponse_Answer: - pLogger.Debugw("sending answer", "answer", logger.Proto(res)) - - case *livekit.SignalResponse_Join: - pLogger.Debugw("sending join", "join", logger.Proto(res)) - signalStats.ResolveRoom(m.Join.GetRoom()) - signalStats.ResolveParticipant(m.Join.GetParticipant()) - - case *livekit.SignalResponse_RoomUpdate: - updateRoomID := livekit.RoomID(m.RoomUpdate.GetRoom().GetSid()) - if updateRoomID != "" { - roomID = updateRoomID - resolveLogger(false) - } - pLogger.Debugw("sending room update", "roomUpdate", logger.Proto(res)) - signalStats.ResolveRoom(m.RoomUpdate.GetRoom()) - - case *livekit.SignalResponse_Update: - pLogger.Debugw("sending participant update", "participantUpdate", logger.Proto(res)) - - case *livekit.SignalResponse_RoomMoved: - resetLogger() - signalStats.Reset() - - roomName = livekit.RoomName(m.RoomMoved.GetRoom().GetName()) - moveRoomID := livekit.RoomID(m.RoomMoved.GetRoom().GetSid()) - if moveRoomID != "" { - roomID = moveRoomID - } - participantIdentity = livekit.ParticipantIdentity(m.RoomMoved.GetParticipant().GetIdentity()) - pID = livekit.ParticipantID(m.RoomMoved.GetParticipant().GetSid()) - resolveLogger(false) - - signalStats.ResolveRoom(m.RoomMoved.GetRoom()) - signalStats.ResolveParticipant(m.RoomMoved.GetParticipant()) - pLogger.Debugw("sending room moved", "roomMoved", logger.Proto(res)) - - default: - pLogger.Debugw("sending signal response", "response", logger.Proto(res)) - } - - if count, err := sigConn.WriteResponse(res); err != nil { - pLogger.Warnw("error writing to websocket", err) + if !writeResponse(msg) { return - } else { - signalStats.AddBytes(uint64(count), true) } } } @@ -634,6 +686,34 @@ func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequ } } +// drainMessageSource writes messages that are still queued in source using write. +// It is used when tearing a signalling connection down, responses the participant queued +// on its way out, a leave request on migration for example, would be dropped otherwise. +// +// The producer writes all pending messages into the source before closing it, so draining +// till the source is closed is a complete flush. The deadline is a backstop for the cases +// where the source is not closed, i. e. when there is no way to tell that everything +// pending has been read. Returns true if the source was drained fully. +func drainMessageSource(source routing.MessageSource, timeout time.Duration, write func(proto.Message) bool) bool { + deadline := time.NewTimer(timeout) + defer deadline.Stop() + + for { + select { + case msg := <-source.ReadChan(): + if msg == nil { + return true + } + if !write(msg) { + return false + } + + case <-deadline.C: + return false + } + } +} + func (s *RTCService) DrainConnections(interval time.Duration, force bool) { s.mu.Lock() conns := maps.Clone(s.connections) diff --git a/pkg/service/rtcservice_test.go b/pkg/service/rtcservice_test.go new file mode 100644 index 000000000..f3cfebe81 --- /dev/null +++ b/pkg/service/rtcservice_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/livekit/protocol/livekit" + + "github.com/livekit/livekit-server/pkg/routing" +) + +func leaveResponse(action livekit.LeaveRequest_Action) *livekit.SignalResponse { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: &livekit.LeaveRequest{ + Action: action, + Reason: livekit.DisconnectReason_MIGRATION, + }, + }, + } +} + +func TestDrainMessageSource(t *testing.T) { + collect := func(msgs *[]proto.Message) func(proto.Message) bool { + return func(msg proto.Message) bool { + *msgs = append(*msgs, msg) + return true + } + } + + t.Run("drains queued messages of a closed source", func(t *testing.T) { + source := routing.NewDefaultMessageChannel("CO_test") + require.NoError(t, source.WriteMessage(leaveResponse(livekit.LeaveRequest_RESUME))) + require.NoError(t, source.WriteMessage(leaveResponse(livekit.LeaveRequest_RECONNECT))) + source.Close() + + var got []proto.Message + require.True(t, drainMessageSource(source, time.Second, collect(&got))) + require.Len(t, got, 2) + require.Equal( + t, + livekit.LeaveRequest_RESUME, + got[0].(*livekit.SignalResponse).GetLeave().GetAction(), + ) + require.Equal( + t, + livekit.LeaveRequest_RECONNECT, + got[1].(*livekit.SignalResponse).GetLeave().GetAction(), + ) + }) + + t.Run("drains messages written while draining", func(t *testing.T) { + source := routing.NewDefaultMessageChannel("CO_test") + // mimics the relay pushing a message that was still in flight when the + // request direction went away, and closing the source right after + go func() { + time.Sleep(20 * time.Millisecond) + _ = source.WriteMessage(leaveResponse(livekit.LeaveRequest_RESUME)) + source.Close() + }() + + var got []proto.Message + require.True(t, drainMessageSource(source, time.Second, collect(&got))) + require.Len(t, got, 1) + }) + + t.Run("gives up on the deadline when the source stays open", func(t *testing.T) { + source := routing.NewDefaultMessageChannel("CO_test") + require.NoError(t, source.WriteMessage(leaveResponse(livekit.LeaveRequest_RESUME))) + + var got []proto.Message + start := time.Now() + require.False(t, drainMessageSource(source, 50*time.Millisecond, collect(&got))) + require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) + // what was queued is still flushed + require.Len(t, got, 1) + }) + + t.Run("stops when the write fails", func(t *testing.T) { + source := routing.NewDefaultMessageChannel("CO_test") + require.NoError(t, source.WriteMessage(leaveResponse(livekit.LeaveRequest_RESUME))) + require.NoError(t, source.WriteMessage(leaveResponse(livekit.LeaveRequest_RECONNECT))) + source.Close() + + var got []proto.Message + require.False(t, drainMessageSource(source, time.Second, func(msg proto.Message) bool { + got = append(got, msg) + return false + })) + require.Len(t, got, 1) + }) +} diff --git a/pkg/service/servicefakes/fake_ingress_launcher.go b/pkg/service/servicefakes/fake_ingress_launcher.go new file mode 100644 index 000000000..06ed69e6b --- /dev/null +++ b/pkg/service/servicefakes/fake_ingress_launcher.go @@ -0,0 +1,118 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package servicefakes + +import ( + "context" + "sync" + + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/protocol/livekit" +) + +type FakeIngressLauncher struct { + LaunchPullIngressStub func(context.Context, *livekit.IngressInfo) (*livekit.IngressInfo, error) + launchPullIngressMutex sync.RWMutex + launchPullIngressArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + launchPullIngressReturns struct { + result1 *livekit.IngressInfo + result2 error + } + launchPullIngressReturnsOnCall map[int]struct { + result1 *livekit.IngressInfo + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeIngressLauncher) LaunchPullIngress(arg1 context.Context, arg2 *livekit.IngressInfo) (*livekit.IngressInfo, error) { + fake.launchPullIngressMutex.Lock() + ret, specificReturn := fake.launchPullIngressReturnsOnCall[len(fake.launchPullIngressArgsForCall)] + fake.launchPullIngressArgsForCall = append(fake.launchPullIngressArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.LaunchPullIngressStub + fakeReturns := fake.launchPullIngressReturns + fake.recordInvocation("LaunchPullIngress", []interface{}{arg1, arg2}) + fake.launchPullIngressMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeIngressLauncher) LaunchPullIngressCallCount() int { + fake.launchPullIngressMutex.RLock() + defer fake.launchPullIngressMutex.RUnlock() + return len(fake.launchPullIngressArgsForCall) +} + +func (fake *FakeIngressLauncher) LaunchPullIngressCalls(stub func(context.Context, *livekit.IngressInfo) (*livekit.IngressInfo, error)) { + fake.launchPullIngressMutex.Lock() + defer fake.launchPullIngressMutex.Unlock() + fake.LaunchPullIngressStub = stub +} + +func (fake *FakeIngressLauncher) LaunchPullIngressArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.launchPullIngressMutex.RLock() + defer fake.launchPullIngressMutex.RUnlock() + argsForCall := fake.launchPullIngressArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeIngressLauncher) LaunchPullIngressReturns(result1 *livekit.IngressInfo, result2 error) { + fake.launchPullIngressMutex.Lock() + defer fake.launchPullIngressMutex.Unlock() + fake.LaunchPullIngressStub = nil + fake.launchPullIngressReturns = struct { + result1 *livekit.IngressInfo + result2 error + }{result1, result2} +} + +func (fake *FakeIngressLauncher) LaunchPullIngressReturnsOnCall(i int, result1 *livekit.IngressInfo, result2 error) { + fake.launchPullIngressMutex.Lock() + defer fake.launchPullIngressMutex.Unlock() + fake.LaunchPullIngressStub = nil + if fake.launchPullIngressReturnsOnCall == nil { + fake.launchPullIngressReturnsOnCall = make(map[int]struct { + result1 *livekit.IngressInfo + result2 error + }) + } + fake.launchPullIngressReturnsOnCall[i] = struct { + result1 *livekit.IngressInfo + result2 error + }{result1, result2} +} + +func (fake *FakeIngressLauncher) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeIngressLauncher) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ service.IngressLauncher = new(FakeIngressLauncher) diff --git a/pkg/service/sip.go b/pkg/service/sip.go index a1283f9de..7dc6b60ef 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -22,7 +22,6 @@ import ( "github.com/dennwc/iters" "github.com/twitchtv/twirp" "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -681,7 +680,7 @@ func (s *SIPService) CreateSIPParticipantRequest(ctx context.Context, req *livek return internalReq, nil } -func (s *SIPService) TransferSIPParticipant(ctx context.Context, req *livekit.TransferSIPParticipantRequest) (*emptypb.Empty, error) { +func (s *SIPService) TransferSIPParticipant(ctx context.Context, req *livekit.TransferSIPParticipantRequest) (*livekit.TransferSIPParticipantResponse, error) { AppendLogFields(ctx, "room", req.RoomName, "participant", req.ParticipantIdentity, @@ -722,13 +721,18 @@ func (s *SIPService) TransferSIPParticipant(ctx context.Context, req *livekit.Tr // own default (which could outlive us). ireq.RingingTimeout = durationpb.New(timeout) - _, err = s.psrpcClient.TransferSIPParticipant(ctx, ireq.SipCallId, ireq, psrpc.WithRequestTimeout(timeout)) + iresp, err := s.psrpcClient.TransferSIPParticipant(ctx, ireq.SipCallId, ireq, psrpc.WithRequestTimeout(timeout)) if err != nil { log.Errorw("cannot transfer sip participant", err) return nil, wrapSIPContextError(err) } - return &emptypb.Empty{}, nil + return &livekit.TransferSIPParticipantResponse{ + TransferId: iresp.GetTransferId(), + Status: iresp.GetStatus(), + Reason: iresp.GetReason(), + SipStatus: iresp.GetSipStatus(), + }, nil } func (s *SIPService) transferSIPParticipantRequest(ctx context.Context, req *livekit.TransferSIPParticipantRequest, log logger.UnlikelyLogger) (*rpc.InternalTransferSIPParticipantRequest, error) { diff --git a/pkg/service/turn.go b/pkg/service/turn.go index 0b3834c91..6e8da2e0e 100644 --- a/pkg/service/turn.go +++ b/pkg/service/turn.go @@ -26,6 +26,7 @@ import ( "github.com/jxskiss/base62" "github.com/pion/stun/v3" "github.com/pion/turn/v5" + "github.com/pires/go-proxyproto" "github.com/pkg/errors" "github.com/livekit/protocol/auth" @@ -162,26 +163,9 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone } if turnConf.TLSPort > 0 { - var listener net.Listener - var listenerErr error - - if turnConf.ExternalTLS { - listener, listenerErr = net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(turnConf.TLSPort))) - } else { - cert, err := tls.LoadX509KeyPair(turnConf.CertFile, turnConf.KeyFile) - if err != nil { - return nil, errors.Wrap(err, "TURN tls cert required") - } - - listener, listenerErr = tls.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(turnConf.TLSPort)), - &tls.Config{ - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{cert}, - }) - } - - if listenerErr != nil { - return nil, errors.Wrap(listenerErr, "could not listen on TURN TCP port") + listener, err := newTURNTCPListener(turnConf, net.JoinHostPort(addr, strconv.Itoa(turnConf.TLSPort))) + if err != nil { + return nil, err } if standalone { listener = telemetry.NewListener(listener) @@ -194,7 +178,7 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone } serverConfig.ListenerConfigs = append(serverConfig.ListenerConfigs, listenerConfig) - logValues = append(logValues, "turn.portTLS", turnConf.TLSPort, "turn.externalTLS", turnConf.ExternalTLS) + logValues = append(logValues, "turn.portTLS", turnConf.TLSPort, "turn.externalTLS", turnConf.ExternalTLS, "turn.proxyProtocol", turnConf.ProxyProtocol) } if turnConf.UDPPort > 0 { @@ -221,6 +205,65 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone return turn.NewServer(serverConfig) } +// newTURNTCPListener returns the TCP listener for TURN/TLS. The PROXY protocol +// header, when enabled, is read before TLS so the client address is known to +// the TLS layer and to TURN regardless of who terminates TLS. +func newTURNTCPListener(turnConf config.TURNConfig, address string) (net.Listener, error) { + var tlsConfig *tls.Config + if !turnConf.ExternalTLS { + cert, err := tls.LoadX509KeyPair(turnConf.CertFile, turnConf.KeyFile) + if err != nil { + return nil, errors.Wrap(err, "TURN tls cert required") + } + tlsConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + } + } + + var proxyPolicy proxyproto.ConnPolicyFunc + if turnConf.ProxyProtocol { + trusted, err := parsePeerCIDRs("turn.proxy_protocol_trusted_cidrs", turnConf.ProxyProtocolTrustedCIDRs) + if err != nil { + return nil, err + } + if len(trusted) == 0 { + return nil, errors.New("turn.proxy_protocol requires at least one entry in turn.proxy_protocol_trusted_cidrs") + } + proxyPolicy = proxyProtocolPolicy(trusted) + } + + listener, err := net.Listen("tcp", address) + if err != nil { + return nil, errors.Wrap(err, "could not listen on TURN TCP port") + } + if proxyPolicy != nil { + listener = &proxyproto.Listener{Listener: listener, ConnPolicy: proxyPolicy} + } + if tlsConfig != nil { + listener = tls.NewListener(listener, tlsConfig) + } + return listener, nil +} + +// proxyProtocolPolicy requires the PROXY header from trusted proxies and closes +// every other connection, so the header cannot be forged by a direct client. +func proxyProtocolPolicy(trusted []*net.IPNet) proxyproto.ConnPolicyFunc { + return func(opts proxyproto.ConnPolicyOptions) (proxyproto.Policy, error) { + tcpAddr, ok := opts.Upstream.(*net.TCPAddr) + if !ok { + return proxyproto.REJECT, fmt.Errorf("%w: unexpected address %v", proxyproto.ErrInvalidUpstream, opts.Upstream) + } + for _, ipnet := range trusted { + if ipnet.Contains(tcpAddr.IP) { + return proxyproto.REQUIRE, nil + } + } + // wrapping ErrInvalidUpstream closes this connection and keeps the listener accepting + return proxyproto.REJECT, fmt.Errorf("%w: %s is not a trusted proxy", proxyproto.ErrInvalidUpstream, tcpAddr.IP) + } +} + func getTURNAuthHandlerFunc(handler *TURNAuthHandler) turn.AuthHandler { return handler.HandleAuth } diff --git a/pkg/service/turn_test.go b/pkg/service/turn_test.go index ab6b2d9ad..c310df2b4 100644 --- a/pkg/service/turn_test.go +++ b/pkg/service/turn_test.go @@ -278,3 +278,139 @@ func TestTURNAuthHandler_CreateUsername_TTLClamped(t *testing.T) { _, negativeExpiry := h.CreateUsername(turnTestAPIKey, pID, -1<<40) require.InDelta(t, time.Now().Unix()+int64(config.DefaultTURNTTLSeconds), negativeExpiry, 2) } + +func proxyProtocolTURNConfig(trustedCIDRs ...string) config.TURNConfig { + return config.TURNConfig{ExternalTLS: true, ProxyProtocol: true, ProxyProtocolTrustedCIDRs: trustedCIDRs} +} + +func TestNewTURNTCPListener_ProxyProtocol(t *testing.T) { + listener, err := newTURNTCPListener(proxyProtocolTURNConfig("127.0.0.0/8"), "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + accepted := make(chan net.Addr, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + accepted <- nil + return + } + defer conn.Close() + // the PROXY header is consumed lazily, on the first read + buf := make([]byte, 1) + _, _ = conn.Read(buf) + accepted <- conn.RemoteAddr() + }() + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + _, err = conn.Write([]byte("PROXY TCP4 203.0.113.9 127.0.0.1 40123 443\r\nx")) + require.NoError(t, err) + + select { + case addr := <-accepted: + require.NotNil(t, addr) + require.Equal(t, "203.0.113.9:40123", addr.String()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the accepted connection") + } +} + +func TestNewTURNTCPListener_ProxyProtocolRejectsBareConnection(t *testing.T) { + listener, err := newTURNTCPListener(proxyProtocolTURNConfig("127.0.0.0/8"), "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + result := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + result <- err + return + } + defer conn.Close() + buf := make([]byte, 1) + _, err = conn.Read(buf) + result <- err + }() + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + _, err = conn.Write([]byte("not a proxy header\r\n")) + require.NoError(t, err) + + select { + case err := <-result: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the rejected connection") + } +} + +func TestNewTURNTCPListener_WithoutProxyProtocol(t *testing.T) { + listener, err := newTURNTCPListener(config.TURNConfig{ExternalTLS: true}, "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + accepted := make(chan net.Addr, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + accepted <- nil + return + } + defer conn.Close() + accepted <- conn.RemoteAddr() + }() + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + + select { + case addr := <-accepted: + require.NotNil(t, addr) + require.Equal(t, conn.LocalAddr().String(), addr.String()) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the accepted connection") + } +} + +func TestNewTURNTCPListener_ProxyProtocolClosesUntrustedProxy(t *testing.T) { + listener, err := newTURNTCPListener(proxyProtocolTURNConfig("203.0.113.0/24"), "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + accepted := make(chan struct{}, 1) + go func() { + if conn, err := listener.Accept(); err == nil { + conn.Close() + accepted <- struct{}{} + } + }() + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + _, _ = conn.Write([]byte("PROXY TCP4 203.0.113.9 127.0.0.1 40123 443\r\nx")) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = conn.Read(make([]byte, 1)) + require.Error(t, err, "the listener should have closed the connection") + + select { + case <-accepted: + t.Fatal("connection from an untrusted proxy must not be accepted") + case <-time.After(200 * time.Millisecond): + } +} + +func TestNewTURNTCPListener_ProxyProtocolRequiresTrustedCIDRs(t *testing.T) { + _, err := newTURNTCPListener(proxyProtocolTURNConfig(), "127.0.0.1:0") + require.Error(t, err) + + _, err = newTURNTCPListener(proxyProtocolTURNConfig("not-a-cidr"), "127.0.0.1:0") + require.Error(t, err) +} diff --git a/pkg/service/whipservice.go b/pkg/service/whipservice.go index 7f5f20fd9..b2d5d7f7a 100644 --- a/pkg/service/whipservice.go +++ b/pkg/service/whipservice.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "net/http" - "net/url" "strings" "github.com/pion/webrtc/v4" @@ -269,7 +268,7 @@ func (s *WHIPService) handleCreate(w http.ResponseWriter, r *http.Request) { for _, iceServer := range res.IceServers { for _, iceURL := range iceServer.Urls { iceServerLink := &linkheader.Link{ - URL: url.PathEscape(iceURL), + URL: iceURL, Rel: "ice-server", Params: map[string]string{}, } diff --git a/pkg/service/wire.go b/pkg/service/wire.go index a2d60fd32..9d1842b4b 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -193,11 +193,12 @@ func createStore(rc redis.UniversalClient) ObjectStore { return NewLocalStore() } -func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { +func getMessageBus(rc redis.UniversalClient, psrpcConf rpc.PSRPCConfig) psrpc.MessageBus { + opts := psrpcConf.BusOptions() if rc == nil { - return psrpc.NewLocalMessageBus() + return psrpc.NewLocalMessageBus(opts...) } - return psrpc.NewRedisMessageBus(rc) + return psrpc.NewRedisMessageBus(rc, opts...) } func getEgressStore(s ObjectStore) EgressStore { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 0da441132..f7c5a6267 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -39,14 +39,14 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live return nil, err } nodeID := getNodeID(currentNode) - messageBus := getMessageBus(universalClient) + psrpcConfig := getPSRPCConfig(conf) + v := getMessageBus(universalClient, psrpcConfig) signalRelayConfig := getSignalRelayConfig(conf) - signalClient, err := routing.NewSignalClient(nodeID, messageBus, signalRelayConfig) + signalClient, err := routing.NewSignalClient(nodeID, v, signalRelayConfig) if err != nil { return nil, err } - psrpcConfig := getPSRPCConfig(conf) - clientParams := getPSRPCClientParams(psrpcConfig, messageBus) + clientParams := getPSRPCClientParams(psrpcConfig, v) roomConfig := getRoomConfig(conf) roomManagerClient, err := routing.NewRoomManagerClient(clientParams, roomConfig) if err != nil { @@ -80,57 +80,57 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live } analyticsService := telemetry.NewAnalyticsService(conf, currentNode) telemetryService := createTelemetryService(queuedNotifier, analyticsService) - ioInfoService, err := NewIOInfoService(messageBus, egressStore, ingressStore, sipStore, telemetryService) + ioInfoService, err := NewIOInfoService(v, egressStore, ingressStore, sipStore, telemetryService) if err != nil { return nil, err } rtcEgressLauncher := NewEgressLauncher(egressClient, ioInfoService, objectStore) topicFormatter := rpc.NewTopicFormatter() - v, err := rpc.NewTypedRoomClient(clientParams) + v2, err := rpc.NewTypedRoomClient(clientParams) if err != nil { return nil, err } - v2, err := rpc.NewTypedParticipantClient(clientParams) + v3, err := rpc.NewTypedParticipantClient(clientParams) if err != nil { return nil, err } - roomService, err := NewRoomService(limitConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher, topicFormatter, v, v2) + roomService, err := NewRoomService(limitConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher, topicFormatter, v2, v3) if err != nil { return nil, err } - v3, err := rpc.NewTypedAgentDispatchInternalClient(clientParams) + v4, err := rpc.NewTypedAgentDispatchInternalClient(clientParams) if err != nil { return nil, err } - agentDispatchService := NewAgentDispatchService(limitConfig, v3, topicFormatter, roomAllocator, router) + agentDispatchService := NewAgentDispatchService(limitConfig, v4, topicFormatter, roomAllocator, router) egressService := NewEgressService(egressClient, rtcEgressLauncher, ioInfoService, roomService) ingressConfig := getIngressConfig(conf) ingressClient, err := rpc.NewIngressClient(clientParams) if err != nil { return nil, err } - ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, ioInfoService, telemetryService) + ingressService := NewIngressService(ingressConfig, nodeID, v, ingressClient, ingressStore, ioInfoService, telemetryService) sipConfig := getSIPConfig(conf) sipClient, err := newSIPClient(clientParams) if err != nil { return nil, err } - sipService := NewSIPService(sipConfig, nodeID, messageBus, sipClient, sipStore, roomService, telemetryService) + sipService := NewSIPService(sipConfig, nodeID, v, sipClient, sipStore, roomService, telemetryService) rtcService := NewRTCService(conf, roomAllocator, router, telemetryService) - v4, err := rpc.NewTypedWHIPParticipantClient(clientParams) + v5, err := rpc.NewTypedWHIPParticipantClient(clientParams) if err != nil { return nil, err } - serviceWHIPService, err := NewWHIPService(conf, router, roomAllocator, clientParams, topicFormatter, v4) + serviceWHIPService, err := NewWHIPService(conf, router, roomAllocator, clientParams, topicFormatter, v5) if err != nil { return nil, err } - agentService, err := NewAgentService(conf, currentNode, messageBus, keyProvider) + agentService, err := NewAgentService(conf, currentNode, v, keyProvider) if err != nil { return nil, err } agentConfig := getAgentConfig(conf) - client, err := agent.NewAgentClient(messageBus, agentConfig) + client, err := agent.NewAgentClient(v, agentConfig) if err != nil { return nil, err } @@ -138,16 +138,16 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live timedVersionGenerator := utils.NewDefaultTimedVersionGenerator() turnAuthHandler := NewTURNAuthHandler(keyProvider) forwardStats := createForwardStats(conf) - roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, roomAllocator, telemetryService, client, agentStore, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus, forwardStats) + roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, roomAllocator, telemetryService, client, agentStore, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, v, forwardStats) if err != nil { return nil, err } - signalServer, err := NewDefaultSignalServer(currentNode, messageBus, signalRelayConfig, router, roomManager) + signalServer, err := NewDefaultSignalServer(currentNode, v, signalRelayConfig, router, roomManager) if err != nil { return nil, err } - v5 := getTURNAuthHandlerFunc(turnAuthHandler) - server, err := newInProcessTurnServer(conf, v5) + v6 := getTURNAuthHandlerFunc(turnAuthHandler) + server, err := newInProcessTurnServer(conf, v6) if err != nil { return nil, err } @@ -164,14 +164,14 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi return nil, err } nodeID := getNodeID(currentNode) - messageBus := getMessageBus(universalClient) + psrpcConfig := getPSRPCConfig(conf) + v := getMessageBus(universalClient, psrpcConfig) signalRelayConfig := getSignalRelayConfig(conf) - signalClient, err := routing.NewSignalClient(nodeID, messageBus, signalRelayConfig) + signalClient, err := routing.NewSignalClient(nodeID, v, signalRelayConfig) if err != nil { return nil, err } - psrpcConfig := getPSRPCConfig(conf) - clientParams := getPSRPCClientParams(psrpcConfig, messageBus) + clientParams := getPSRPCClientParams(psrpcConfig, v) roomConfig := getRoomConfig(conf) roomManagerClient, err := routing.NewRoomManagerClient(clientParams, roomConfig) if err != nil { @@ -254,11 +254,12 @@ func createStore(rc redis.UniversalClient) ObjectStore { return NewLocalStore() } -func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { +func getMessageBus(rc redis.UniversalClient, psrpcConf rpc.PSRPCConfig) psrpc.MessageBus { + opts := psrpcConf.BusOptions() if rc == nil { - return psrpc.NewLocalMessageBus() + return psrpc.NewLocalMessageBus(opts...) } - return psrpc.NewRedisMessageBus(rc) + return psrpc.NewRedisMessageBus(rc, opts...) } func getEgressStore(s ObjectStore) EgressStore { diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index c7383c7b4..5468b3b94 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -72,6 +72,24 @@ type Buffer struct { primaryBufferForRTX *Buffer rtxPktBuf []byte + + streamInfoProbe *StreamInfoProbe + warnedPendingOverflow bool +} + +// StreamInfoProbe identifies a stream from the mid/rid/rsid header extensions of its +// packets. It runs on the write path, i. e. as SRTP pushes into this buffer, because +// nothing reads remote streams through pion's interceptor chain. +type StreamInfoProbe struct { + MidExtID uint8 + RidExtID uint8 + RsidExtID uint8 + + // Tries bounds how many packets are inspected before giving up. + Tries int + + // OnFound is called at most once, in a goroutine, as it can re-enter this buffer. + OnFound func(ssrc uint32, mid, rid, rsid string) } func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer { @@ -166,6 +184,10 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } + if b.streamInfoProbe != nil { + b.probeStreamInfoLocked(&rtpPacket) + } + // handle RTX packet if pb := b.primaryBufferForRTX; pb != nil { b.Unlock() @@ -191,6 +213,17 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { overflow := len(b.pPackets) - max(b.BufferBase.MaxVideoPkts(), b.BufferBase.MaxAudioPkts()) if overflow > 0 { startIdx = overflow + + // a stream that keeps arriving but never binds drops every packet from here + // on; for an RTX stream it means the pairing was never established + if !b.warnedPendingOverflow { + b.warnedPendingOverflow = true + b.logger.Warnw( + "unbound buffer overflowing, dropping packets", nil, + "ssrc", b.BufferBase.SSRC(), + "pending", len(b.pPackets), + ) + } } b.pPackets = append(b.pPackets[startIdx:], pendingPacket{ packet: packet, @@ -213,6 +246,59 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } +// SetStreamInfoProbe installs probe and runs it over packets already queued. +func (b *Buffer) SetStreamInfoProbe(probe *StreamInfoProbe) { + b.Lock() + defer b.Unlock() + + b.streamInfoProbe = probe + + for _, pp := range b.pPackets { + if b.streamInfoProbe == nil { + return + } + + var rtpPacket rtp.Packet + if err := rtpPacket.Unmarshal(pp.packet); err != nil { + continue + } + b.probeStreamInfoLocked(&rtpPacket) + } +} + +// probeStreamInfoLocked inspects one packet, clearing the probe once the stream is +// identified or the try budget runs out. +func (b *Buffer) probeStreamInfoLocked(rtpPacket *rtp.Packet) { + probe := b.streamInfoProbe + + var mid, rid, rsid string + if ext := rtpPacket.GetExtension(probe.MidExtID); ext != nil { + mid = string(ext) + } + if ext := rtpPacket.GetExtension(probe.RidExtID); ext != nil { + rid = string(ext) + } + if ext := rtpPacket.GetExtension(probe.RsidExtID); ext != nil { + rsid = string(ext) + } + + if mid != "" && (rid != "" || rsid != "") { + b.streamInfoProbe = nil + b.logger.Debugw("stream found", "ssrc", rtpPacket.SSRC, "mid", mid, "rid", rid, "rsid", rsid) + go probe.OnFound(rtpPacket.SSRC, mid, rid, rsid) + return + } + + // ignore padding only packets for probe count + if rtpPacket.Padding && len(rtpPacket.Payload) == 0 { + return + } + + if probe.Tries--; probe.Tries <= 0 { + b.streamInfoProbe = nil + } +} + func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) { b.Lock() b.primaryBufferForRTX = primaryBuffer diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 618200aab..09a0d3b52 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -60,12 +60,18 @@ type DependencyDescriptorParser struct { onMaxLayerChanged func(int32, int32) decodeTargets []DependencyDescriptorDecodeTarget - seqWrapAround *utils.WrapAround[uint16, uint64] - frameWrapAround *utils.WrapAround[uint16, uint64] - structureExtFrameNum uint64 - activeDecodeTargetsExtSeq uint64 - activeDecodeTargetsMask uint32 - frameChecker *FrameIntegrityChecker + seqWrapAround *utils.WrapAround[uint16, uint64] + frameWrapAround *utils.WrapAround[uint16, uint64] + structureExtFrameNum uint64 + // drop threshold for frames belonging to a previous dependency structure. + // advanced only when the structure id actually changes: structureExtFrameNum + // advances on every structure-bearing key frame, so with frequent key frames + // repeating the same structure (e. g. screen content), valid late/retransmitted + // frames would be dropped as "earlier than current structure". + structureChangeExtFrameNum uint64 + activeDecodeTargetsExtSeq uint64 + activeDecodeTargetsMask uint32 + frameChecker *FrameIntegrityChecker ddNotFoundCount atomic.Uint32 @@ -153,12 +159,12 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart) extFN := unwrapped.ExtendedVal - if extFN < r.structureExtFrameNum { + if extFN < r.structureChangeExtFrameNum { r.logger.Debugw( "drop frame which is earlier than current structure", "fn", ddVal.FrameNumber, "extFN", extFN, - "structureExtFrameNum", r.structureExtFrameNum, + "structureChangeExtFrameNum", r.structureChangeExtFrameNum, "unwrappedFN", unwrapped, "frameWrapAround", r.frameWrapAround, ) @@ -188,7 +194,24 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket } + if extFN < r.structureExtFrameNum { + // out-of-order key frame repeating the current structure: accepting it + // would regress structureExtFrameNum (ExtKeyFrameNum) and replay a stale + // structure update, confusing the downtrack's dependency descriptor + // selector. + r.logger.Debugw( + "drop out-of-order key frame", + "extFN", extFN, + "structureExtFrameNum", r.structureExtFrameNum, + ) + ReleaseExtDependencyDescriptor(extDD) + return nil, videoLayer, ErrFrameEarlierThanKeyFrame + } + if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId { + // structure actually changed (or first structure): advance the drop threshold + // so that only frames preceding this structure are dropped. + r.structureChangeExtFrameNum = extFN r.logger.Debugw( "structure updated", "structureID", ddVal.AttachedStructure.StructureId, @@ -252,6 +275,7 @@ func (r *DependencyDescriptorParser) restart() { r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt) r.structure = nil r.structureExtFrameNum = 0 + r.structureChangeExtFrameNum = 0 r.activeDecodeTargetsExtSeq = 0 r.activeDecodeTargetsMask = 0 r.decodeTargets = r.decodeTargets[:0] diff --git a/pkg/sfu/buffer/dependencydescriptorparser_test.go b/pkg/sfu/buffer/dependencydescriptorparser_test.go new file mode 100644 index 000000000..652b0b637 --- /dev/null +++ b/pkg/sfu/buffer/dependencydescriptorparser_test.go @@ -0,0 +1,188 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "testing" + + "github.com/pion/rtp" + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/logger" + + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" +) + +const ddTestExtID = uint8(1) + +// single spatial layer, single temporal layer, one decode target, one chain +func newL1T1Structure(structureID int) *dd.FrameDependencyStructure { + return &dd.FrameDependencyStructure{ + StructureId: structureID, + NumDecodeTargets: 1, + NumChains: 1, + DecodeTargetProtectedByChain: []int{0}, + Templates: []*dd.FrameDependencyTemplate{ + { // key frame + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetSwitch}, + ChainDiffs: []int{0}, + }, + { // delta frame + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetRequired}, + FrameDiffs: []int{1}, + ChainDiffs: []int{1}, + }, + }, + } +} + +type ddTestFeeder struct { + t *testing.T + parser *DependencyDescriptorParser + // structure the writer marshals against, i. e. what the publisher last sent + structure *dd.FrameDependencyStructure + seq uint16 +} + +func newDDTestFeeder(t *testing.T) *ddTestFeeder { + return &ddTestFeeder{ + t: t, + parser: NewDependencyDescriptorParser(ddTestExtID, logger.GetLogger(), func(int32, int32) {}, false), + } +} + +// keyFrame sends a key frame carrying `structure`. Passing the same structure id as +// the previous key frame models a publisher that repeats an unchanged structure, +// which is what screen content with frequent key frames does. +func (f *ddTestFeeder) keyFrame(frameNumber uint16, structure *dd.FrameDependencyStructure) (*ExtDependencyDescriptor, error) { + f.structure = structure + return f.feed(frameNumber, structure.Templates[0], structure) +} + +func (f *ddTestFeeder) deltaFrame(frameNumber uint16) (*ExtDependencyDescriptor, error) { + return f.feed(frameNumber, f.structure.Templates[1], nil) +} + +func (f *ddTestFeeder) feed( + frameNumber uint16, + template *dd.FrameDependencyTemplate, + attachedStructure *dd.FrameDependencyStructure, +) (*ExtDependencyDescriptor, error) { + f.t.Helper() + + ddVal := &dd.DependencyDescriptor{ + FirstPacketInFrame: true, + LastPacketInFrame: true, + FrameNumber: frameNumber, + FrameDependencies: template, + AttachedStructure: attachedStructure, + } + buf, err := (&dd.DependencyDescriptorExtension{Descriptor: ddVal, Structure: f.structure}).Marshal() + require.NoError(f.t, err) + + f.seq++ + pkt := &rtp.Packet{Header: rtp.Header{SequenceNumber: f.seq}} + require.NoError(f.t, pkt.SetExtension(ddTestExtID, buf)) + + extDD, _, err := f.parser.Parse(pkt) + return extDD, err +} + +// A key frame that repeats the current structure must not start dropping frames that +// precede it. Upstream advanced the drop threshold on every structure bearing key +// frame, so with frequent key frames (screen content) a late or retransmitted frame +// arriving after one was discarded as "earlier than current structure". +func TestDependencyDescriptorParserLateFrameAfterRepeatedStructure(t *testing.T) { + f := newDDTestFeeder(t) + structure := newL1T1Structure(0) + + _, err := f.keyFrame(0, structure) + require.NoError(t, err) + _, err = f.deltaFrame(1) + require.NoError(t, err) + _, err = f.deltaFrame(3) + require.NoError(t, err) + + // key frame repeating the same structure id + _, err = f.keyFrame(4, structure) + require.NoError(t, err) + + // frame 2 finally shows up (reordered or retransmitted). The structure has not + // changed, so it is still decodable and must be forwarded. + extDD, err := f.deltaFrame(2) + require.NoError(t, err) + require.NotNil(t, extDD) + require.EqualValues(t, 2, extDD.ExtFrameNum) +} + +// A key frame carrying a *different* structure does invalidate everything before it: +// earlier frames reference templates that no longer exist. +func TestDependencyDescriptorParserLateFrameAfterStructureChange(t *testing.T) { + f := newDDTestFeeder(t) + + _, err := f.keyFrame(0, newL1T1Structure(0)) + require.NoError(t, err) + _, err = f.deltaFrame(1) + require.NoError(t, err) + _, err = f.deltaFrame(3) + require.NoError(t, err) + + _, err = f.keyFrame(4, newL1T1Structure(1)) + require.NoError(t, err) + + _, err = f.deltaFrame(2) + require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame) +} + +// An out-of-order key frame must still be dropped: accepting it would regress +// structureExtFrameNum (ExtKeyFrameNum) and replay a stale structure update. +func TestDependencyDescriptorParserOutOfOrderKeyFrame(t *testing.T) { + f := newDDTestFeeder(t) + structure := newL1T1Structure(0) + + _, err := f.keyFrame(0, structure) + require.NoError(t, err) + _, err = f.deltaFrame(1) + require.NoError(t, err) + _, err = f.keyFrame(4, structure) + require.NoError(t, err) + + _, err = f.keyFrame(2, structure) + require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame) +} + +// ExtKeyFrameNum keeps tracking every structure bearing key frame, unchanged by the +// split of the drop threshold into its own field. +func TestDependencyDescriptorParserExtKeyFrameNum(t *testing.T) { + f := newDDTestFeeder(t) + structure := newL1T1Structure(0) + + extDD, err := f.keyFrame(0, structure) + require.NoError(t, err) + require.EqualValues(t, 0, extDD.ExtKeyFrameNum) + + extDD, err = f.deltaFrame(1) + require.NoError(t, err) + require.EqualValues(t, 0, extDD.ExtKeyFrameNum) + + // repeated structure still advances ExtKeyFrameNum + extDD, err = f.keyFrame(4, structure) + require.NoError(t, err) + require.EqualValues(t, 4, extDD.ExtKeyFrameNum) + + extDD, err = f.deltaFrame(5) + require.NoError(t, err) + require.EqualValues(t, 4, extDD.ExtKeyFrameNum) +} diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go index 4a73ab057..8fb6d46dd 100644 --- a/pkg/sfu/buffer/factory.go +++ b/pkg/sfu/buffer/factory.go @@ -118,6 +118,21 @@ func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader { return f.rtcpReaders[ssrc] } +// SetStreamInfoProbe installs probe on the buffer of ssrc, reporting whether that +// buffer exists. False means the stream can never be identified. +func (f *Factory) SetStreamInfoProbe(ssrc uint32, probe *StreamInfoProbe) bool { + f.RLock() + buffer := f.rtpBuffers[ssrc] + f.RUnlock() + + if buffer == nil { + return false + } + + buffer.SetStreamInfoProbe(probe) + return true +} + func (f *Factory) SetRTXPair(repair, base uint32, rsid string) { f.Lock() repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base] diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index b0326f4a9..1b993158c 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1059,9 +1059,10 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) int32 { } payload = payload[:len(tp.codecBytes)+n] + trailerStripped := 0 if d.params.StripPacketTrailer { - if strip := packettrailer.StripTrailer(payload, tp.marker); strip > 0 { - payload = payload[:len(payload)-strip] + if trailerStripped = packettrailer.StripTrailer(payload, tp.marker || tp.isEndOfLayerFrame); trailerStripped > 0 { + payload = payload[:len(payload)-trailerStripped] } } @@ -1132,6 +1133,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) int32 { tp.incomingHeaderSize, tp.ddBytes, actBytes, + trailerStripped, ) } @@ -2243,9 +2245,17 @@ func (d *DownTrack) retransmitPacket(epm *extPacketMeta, sourcePkt []byte, isPro payload = payload[:rtxOffset+int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] } - if d.params.StripPacketTrailer { - if strip := packettrailer.StripTrailer(payload[rtxOffset:], epm.marker); strip > 0 { - payload = payload[:len(payload)-strip] + // replay the strip done on the original transmission to keep the retransmitted + // payload byte identical to it + if epm.trailerStripped != 0 { + if int(epm.trailerStripped) > len(payload)-rtxOffset { + d.params.Logger.Warnw( + "recorded packet trailer size overflows payload", errPayloadOverflow, + "trailerStripped", epm.trailerStripped, + "payloadSize", len(payload)-rtxOffset, + ) + } else { + payload = payload[:len(payload)-int(epm.trailerStripped)] } } diff --git a/pkg/sfu/downtrack_downstream_integration_test.go b/pkg/sfu/downtrack_downstream_integration_test.go index 2d52d35ca..1f946a531 100644 --- a/pkg/sfu/downtrack_downstream_integration_test.go +++ b/pkg/sfu/downtrack_downstream_integration_test.go @@ -45,12 +45,8 @@ import ( "testing" "time" - "github.com/pion/interceptor" - "github.com/pion/logging" "github.com/pion/rtcp" "github.com/pion/rtp" - "github.com/pion/sdp/v3" - "github.com/pion/transport/v4/vnet" "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" @@ -66,168 +62,13 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/packettrailer" "github.com/livekit/livekit-server/pkg/sfu/sfufakes" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/sfu/testutils" + "github.com/livekit/livekit-server/pkg/testutils/vnettest" ) -// ----------------------------------------------------------------------------- -// vnet harness -// ----------------------------------------------------------------------------- - -type vnetHarness struct { - wan *vnet.Router - offerNet *vnet.Net - answerNet *vnet.Net -} - -func buildVNet(t *testing.T) *vnetHarness { - t.Helper() - - wan, err := vnet.NewRouter(&vnet.RouterConfig{ - CIDR: "1.2.3.0/24", - LoggerFactory: logging.NewDefaultLoggerFactory(), - }) - require.NoError(t, err) - - offerNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"1.2.3.4"}}) - require.NoError(t, err) - require.NoError(t, wan.AddNet(offerNet)) - - answerNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"1.2.3.5"}}) - require.NoError(t, err) - require.NoError(t, wan.AddNet(answerNet)) - - require.NoError(t, wan.Start()) - t.Cleanup(func() { _ = wan.Stop() }) - - return &vnetHarness{wan: wan, offerNet: offerNet, answerNet: answerNet} -} - -// mediaEngineConfig describes what codecs / header extensions to register on a PC. -type mediaEngineConfig struct { - video bool // register VP8 (+ RTX); otherwise register opus - headerExtensions bool // register abs-send-time + transport-cc header extensions -} - -func newMediaPC(t *testing.T, net *vnet.Net, factory *buffer.Factory, cfg mediaEngineConfig) *webrtc.PeerConnection { - t.Helper() - - me := &webrtc.MediaEngine{} - if cfg.video { - require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback(), - }, - PayloadType: 96, - }, webrtc.RTPCodecTypeVideo)) - require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeRTX, ClockRate: 90000, SDPFmtpLine: "apt=96", - }, - PayloadType: 97, - }, webrtc.RTPCodecTypeVideo)) - } else { - require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, - }, - PayloadType: 111, - }, webrtc.RTPCodecTypeAudio)) - } - - if cfg.headerExtensions { - kind := webrtc.RTPCodecTypeAudio - if cfg.video { - kind = webrtc.RTPCodecTypeVideo - } - require.NoError(t, me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: sdp.ABSSendTimeURI}, kind)) - require.NoError(t, me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: sdp.TransportCCURI}, kind)) - } - - // no pion default interceptors: the DownTrack/pacer fill abs-send-time and - // transport-cc themselves, and the tests drive RTCP feedback synthetically, so - // pion's own feedback generators would only add nondeterminism. - ir := &interceptor.Registry{} - - se := webrtc.SettingEngine{} - se.SetNet(net) - se.SetICETimeouts(time.Second, time.Second, 200*time.Millisecond) - se.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) - if factory != nil { - se.BufferFactory = factory.GetOrNew - } - - api := webrtc.NewAPI( - webrtc.WithMediaEngine(me), - webrtc.WithInterceptorRegistry(ir), - webrtc.WithSettingEngine(se), - ) - pc, err := api.NewPeerConnection(webrtc.Configuration{}) - require.NoError(t, err) - t.Cleanup(func() { _ = pc.Close() }) - - return pc -} - -func videoRTCPFeedback() []webrtc.RTCPFeedback { - return []webrtc.RTCPFeedback{ - {Type: "nack"}, - {Type: "nack", Parameter: "pli"}, - {Type: webrtc.TypeRTCPFBTransportCC}, - {Type: webrtc.TypeRTCPFBGoogREMB}, - } -} - -// signalPair performs a full offer/answer exchange between two PCs (adapted from -// pion's own test helper) and waits for both to reach the connected state. -func signalPair(t *testing.T, offerer, answerer *webrtc.PeerConnection) { - t.Helper() - - connected := untilConnected(offerer, answerer) - - offer, err := offerer.CreateOffer(nil) - require.NoError(t, err) - gatherOffer := webrtc.GatheringCompletePromise(offerer) - require.NoError(t, offerer.SetLocalDescription(offer)) - <-gatherOffer - - require.NoError(t, answerer.SetRemoteDescription(*offerer.LocalDescription())) - - answer, err := answerer.CreateAnswer(nil) - require.NoError(t, err) - gatherAnswer := webrtc.GatheringCompletePromise(answerer) - require.NoError(t, answerer.SetLocalDescription(answer)) - <-gatherAnswer - - require.NoError(t, offerer.SetRemoteDescription(*answerer.LocalDescription())) - - select { - case <-connected: - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for peer connections to connect") - } -} - -func untilConnected(pcs ...*webrtc.PeerConnection) <-chan struct{} { - var wg sync.WaitGroup - wg.Add(len(pcs)) - for _, pc := range pcs { - var once sync.Once - pc.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { - if s == webrtc.PeerConnectionStateConnected { - once.Do(wg.Done) - } - }) - } - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - return done -} - // ----------------------------------------------------------------------------- // packet capture on the far side // ----------------------------------------------------------------------------- @@ -319,7 +160,7 @@ var ( PayloadType: 111, } vp8CodecParams = webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback()}, + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: vnettest.VideoRTCPFeedback()}, PayloadType: 96, } ) @@ -335,11 +176,11 @@ type downTrackHarness struct { // newBoundDownTrack builds a real DownTrack, attaches it to a real sender PC, // negotiates with a real subscriber PC over vnet, and makes it writable. -func newBoundDownTrack(t *testing.T, h *vnetHarness, factory *buffer.Factory, codecParams webrtc.RTPCodecParameters, p pacer.Pacer, meCfg mediaEngineConfig) *downTrackHarness { +func newBoundDownTrack(t *testing.T, h *vnettest.Hosts, factory *buffer.Factory, codecParams webrtc.RTPCodecParameters, p pacer.Pacer, meCfg vnettest.MediaEngineConfig) *downTrackHarness { t.Helper() - sender := newMediaPC(t, h.offerNet, factory, meCfg) - sub := newMediaPC(t, h.answerNet, factory, meCfg) + sender := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.OfferNet, MediaEngine: meCfg, BufferFactory: factory.GetOrNew}) + sub := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.AnswerNet, MediaEngine: meCfg, BufferFactory: factory.GetOrNew}) capture := captureTrack(sub) rcv := newFakeTrackReceiver(codecParams) @@ -360,7 +201,7 @@ func newBoundDownTrack(t *testing.T, h *vnetHarness, factory *buffer.Factory, co require.NoError(t, err) dt.SetTransceiver(tr) - signalPair(t, sender, sub) + vnettest.SignalPair(t, sender, sub) dt.SetConnected() require.Eventually(t, func() bool { @@ -392,10 +233,10 @@ func distinctivePayload(seed byte, n int) []byte { // ----------------------------------------------------------------------------- func TestPionVNetForwardingSpike(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) - sender := newMediaPC(t, h.offerNet, nil, mediaEngineConfig{video: true}) - receiver := newMediaPC(t, h.answerNet, nil, mediaEngineConfig{video: true}) + sender := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.OfferNet, MediaEngine: vnettest.MediaEngineConfig{Video: true}}) + receiver := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.AnswerNet, MediaEngine: vnettest.MediaEngineConfig{Video: true}}) track, err := webrtc.NewTrackLocalStaticRTP( webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000}, @@ -407,7 +248,7 @@ func TestPionVNetForwardingSpike(t *testing.T) { cap := captureTrack(receiver) - signalPair(t, sender, receiver) + vnettest.SignalPair(t, sender, receiver) // pump a few RTP packets from the sender track go func() { @@ -444,12 +285,12 @@ func TestPionVNetForwardingSpike(t *testing.T) { // contiguous sequence numbers, timestamps, payload bytes, and that the padding bit // is cleared regardless of the source packet's padding bit. func TestDownTrackForwardsMedia(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE()) t.Cleanup(p.Stop) - dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, mediaEngineConfig{video: false}) + dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, vnettest.MediaEngineConfig{}) const numPackets = 20 sn := uint16(23333) @@ -516,12 +357,12 @@ func TestDownTrackForwardsMedia(t *testing.T) { // type, sequence number, timestamp, payload bytes, and that the padding bit is // cleared regardless of the source packet's padding bit. func TestDownTrackRetransmitsPacketsAsIs(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE()) t.Cleanup(p.Stop) - dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, mediaEngineConfig{video: false}) + dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, vnettest.MediaEngineConfig{}) const numPackets = 10 targetSN := uint16(40000) @@ -557,6 +398,7 @@ func TestDownTrackRetransmitsPacketsAsIs(t *testing.T) { uint64(ts), 0, raw, + 0, ) require.NoError(t, err) targetSN++ @@ -595,13 +437,13 @@ func TestDownTrackRetransmitsPacketsAsIs(t *testing.T) { // fields are not observable on the far side. The packets are still handed to the real // pacer (and written over pion). func TestDownTrackRetransmitsPacketsViaRTX(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() cp := &capturingPacer{inner: pacer.NewPassThrough(logger.GetLogger(), newNullBWE())} t.Cleanup(cp.Stop) // VP8 registers an RTX codec, so the bound DownTrack has an RTX SSRC / payload type. - dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, mediaEngineConfig{video: true}) + dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{Video: true}) require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated") require.NotZero(t, dh.dt.PayloadTypeRTXForTest(), "RTX payload type should be negotiated") @@ -635,7 +477,7 @@ func TestDownTrackRetransmitsPacketsViaRTX(t *testing.T) { ts := uint32(700000 + i*3000) wants[targetSN] = want{payload: payload, ts: ts} - _, err = dh.dt.RetransmitForTest(uint64(src.SequenceNumber), targetSN, ts, uint64(ts), 0, raw) + _, err = dh.dt.RetransmitForTest(uint64(src.SequenceNumber), targetSN, ts, uint64(ts), 0, raw, 0) require.NoError(t, err) targetSN++ } @@ -673,6 +515,65 @@ func TestDownTrackRetransmitsPacketsViaRTX(t *testing.T) { } } +// TestDownTrackReplaysPacketTrailerStripOnRetransmit verifies that a packet whose +// packet trailer was stripped when it was forwarded is retransmitted with the same +// bytes removed, i. e. the retransmitted media payload is byte identical to the +// original transmission. +func TestDownTrackReplaysPacketTrailerStripOnRetransmit(t *testing.T) { + h := vnettest.NewHosts(t) + factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() + cp := &capturingPacer{inner: pacer.NewPassThrough(logger.GetLogger(), newNullBWE())} + t.Cleanup(cp.Stop) + + dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{Video: true}) + require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated") + + video := distinctivePayload(11, 40) + trailer := lktsTrailer() + src := &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + PayloadType: 96, + SequenceNumber: 3000, + Timestamp: 270000, + SSRC: 0x44444444, + }, + Payload: append(append([]byte{}, video...), trailer...), + } + raw, err := src.Marshal() + require.NoError(t, err) + + osn := uint16(60000) + _, err = dh.dt.RetransmitForTest( + uint64(src.SequenceNumber), + osn, + src.Timestamp, + uint64(src.Timestamp), + 0, + raw, + uint8(len(trailer)), + ) + require.NoError(t, err) + + require.Eventually(t, func() bool { + return len(cp.rtxPackets()) >= 1 + }, 5*time.Second, 20*time.Millisecond, "expected RTX packet to be emitted") + + pr := cp.rtxPackets()[0] + require.EqualValues(t, osn, binary.BigEndian.Uint16(pr.payload[0:2])) + require.Equal(t, video, pr.payload[2:], "retransmitted payload must not carry the packet trailer") +} + +// lktsTrailer builds a 15-byte LKTS packet trailer carrying a user timestamp TLV. +func lktsTrailer() []byte { + trailer := []byte{0x01 ^ 0xFF, 8 ^ 0xFF} + for i := 0; i < 8; i++ { + trailer = append(trailer, byte(i)^0xFF) + } + trailer = append(trailer, 15^0xFF) + return append(trailer, packettrailer.Magic[:]...) +} + // ----------------------------------------------------------------------------- // WritePaddingRTP: padding-only packets // ----------------------------------------------------------------------------- @@ -688,12 +589,12 @@ func TestDownTrackRetransmitsPacketsViaRTX(t *testing.T) { // and reports both the padding bit and the padding size, so the received payload // length equals the declared padding size (RTPPaddingMaxPayloadSize). func TestDownTrackSendsPaddingOnlyPackets(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE()) t.Cleanup(p.Stop) - dh := newBoundDownTrack(t, h, factory, vp8CodecParams, p, mediaEngineConfig{video: true}) + dh := newBoundDownTrack(t, h, factory, vp8CodecParams, p, vnettest.MediaEngineConfig{Video: true}) // force a valid target/current layer so the video forwarder will forward (test seam // also used by forwarder_test.go's disable()). @@ -910,7 +811,7 @@ func TestDownTrackSendsProbePackets(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - h := buildVNet(t) + h := vnettest.NewHosts(t) factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory() b := tc.makeBWE() @@ -920,9 +821,9 @@ func TestDownTrackSendsProbePackets(t *testing.T) { // header extensions (abs-send-time for remote BWE, transport-cc for send-side) // are needed for the DownTrack to pick up an ext id, which WriteProbePackets // requires. - dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, mediaEngineConfig{ - video: true, - headerExtensions: true, + dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{ + Video: true, + HeaderExtensions: true, }) require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated") diff --git a/pkg/sfu/export_test.go b/pkg/sfu/export_test.go index da31e2b75..449289e20 100644 --- a/pkg/sfu/export_test.go +++ b/pkg/sfu/export_test.go @@ -56,13 +56,15 @@ func (d *DownTrack) RetransmitForTest( extTimestamp uint64, layer int8, sourcePkt []byte, + trailerStripped uint8, ) (int, error) { epm := extPacketMeta{ packetMeta: packetMeta{ - sourceSeqNo: sourceSeqNo, - targetSeqNo: targetSeqNo, - timestamp: timestamp, - layer: layer, + sourceSeqNo: sourceSeqNo, + targetSeqNo: targetSeqNo, + timestamp: timestamp, + layer: layer, + trailerStripped: trailerStripped, }, extSequenceNumber: uint64(targetSeqNo), extTimestamp: extTimestamp, diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 527d5165f..6d9069151 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pion/rtp" + "github.com/pion/rtp/codecs" "github.com/pion/webrtc/v4" "go.uber.org/zap/zapcore" @@ -202,6 +203,8 @@ type TranslationParams struct { incomingHeaderSize int codecBytes []byte marker bool + // end of the svc spatial layer frame + isEndOfLayerFrame bool } // ------------------------------------------------------------------- @@ -2125,6 +2128,20 @@ func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer in return tp, err } +// should be called with lock held +func (f *Forwarder) isEndOfLayerFrame(extPkt *buffer.ExtPacket) bool { + if extPkt.DependencyDescriptor != nil { + return extPkt.DependencyDescriptor.Descriptor != nil && extPkt.DependencyDescriptor.Descriptor.LastPacketInFrame + } + + if f.mime == mime.MimeTypeVP9 { + vp9, ok := extPkt.Payload.(codecs.VP9Packet) + return ok && vp9.E + } + + return false +} + // should be called with lock held func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { tp := TranslationParams{} @@ -2169,6 +2186,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.isSwitching = result.IsSwitching tp.ddBytes = result.DependencyDescriptorExtension tp.marker = result.RTPMarker + tp.isEndOfLayerFrame = f.isEndOfLayerFrame(extPkt) starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp) tp.isStarting = starting diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index ff83d61a6..b9d8170f8 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -17,14 +17,17 @@ package sfu import ( "testing" + "github.com/pion/rtp/codecs" "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/testutils" ) @@ -2215,3 +2218,34 @@ func TestGetRefLayerRTPTimestampBounds(t *testing.T) { require.Error(t, err) // unavailable sender report, not invalid layer require.Contains(t, err.Error(), "unavailable") } + +// TestForwarderIsEndOfLayerFrame checks the end-of-layer-frame detection used to +// locate packet trailers, which VP9 SVC can carry at the end of any spatial layer +// frame and not just at the end of a picture. +func TestForwarderIsEndOfLayerFrame(t *testing.T) { + vp9Codec := webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP9.String(), ClockRate: 90000} + + f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) + + f = newForwarder(vp9Codec, webrtc.RTPCodecTypeVideo) + require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: true}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{Payload: codecs.VP9Packet{E: false}})) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{})) + + require.True(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: true}, + }, + Payload: codecs.VP9Packet{E: false}, + })) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{LastPacketInFrame: false}, + }, + Payload: codecs.VP9Packet{E: true}, + })) + require.False(t, f.isEndOfLayerFrame(&buffer.ExtPacket{ + DependencyDescriptor: &buffer.ExtDependencyDescriptor{}, + })) +} diff --git a/pkg/sfu/interceptor/rtx.go b/pkg/sfu/interceptor/rtx.go index 1451ffa29..e6f4f2c57 100644 --- a/pkg/sfu/interceptor/rtx.go +++ b/pkg/sfu/interceptor/rtx.go @@ -21,6 +21,7 @@ import ( "github.com/pion/sdp/v3" "github.com/pion/webrtc/v4" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" ) @@ -31,6 +32,12 @@ const ( rtxProbeCount = 10 ) +// StreamInfoProber installs a bounded probe that identifies a remote stream from the +// mid/rid/rsid header extensions of its packets. Implemented by buffer.Factory. +type StreamInfoProber interface { + SetStreamInfoProbe(ssrc uint32, probe *buffer.StreamInfoProbe) bool +} + type streamInfo struct { mid string rid string @@ -40,22 +47,51 @@ type streamInfo struct { type RTXInfoExtractorFactory struct { onStreamFound func(*interceptor.StreamInfo) onRTXPairFound func(repair, base uint32, rsid string) + prober StreamInfoProber lock sync.Mutex streams map[uint32]streamInfo + paired map[uint32]struct{} logger logger.Logger } func NewRTXInfoExtractorFactory( onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32, rsid string), + prober StreamInfoProber, + simTracks map[uint32]SimulcastTrackInfo, logger logger.Logger, ) *RTXInfoExtractorFactory { - return &RTXInfoExtractorFactory{ + f := &RTXInfoExtractorFactory{ onStreamFound: onStreamFound, onRTXPairFound: onRTXPairFound, + prober: prober, streams: make(map[uint32]streamInfo), + paired: make(map[uint32]struct{}), logger: logger, } + f.seedSimulcastTracks(simTracks) + return f +} + +// seedSimulcastTracks pairs migrated streams from the migration info. A migrated client +// is mid-stream and stops sending rid/rsid, so the extensions never appear on the wire +// and the pairing has to come from what is already known about the tracks. +func (f *RTXInfoExtractorFactory) seedSimulcastTracks(simTracks map[uint32]SimulcastTrackInfo) { + for ssrc, info := range simTracks { + if info.Mid == "" || info.StreamID == "" { + continue + } + + if info.IsRepairStream { + f.SetStreamInfo(ssrc, info.Mid, "", info.StreamID) + continue + } + + f.SetStreamInfo(ssrc, info.Mid, info.StreamID, "") + if info.RepairSSRC != 0 { + f.SetStreamInfo(info.RepairSSRC, info.Mid, "", info.StreamID) + } + } } func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) { @@ -75,6 +111,12 @@ func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid stri return } + // the same stream can be reported by both the packet probe and the migration info + if _, ok := f.paired[ssrc]; ok { + f.lock.Unlock() + return + } + if rsid != "" { // repair stream found, find base stream for base, info := range f.streams { @@ -106,13 +148,15 @@ func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid stri rid: rid, rsid: rsid, } + f.lock.Unlock() + return } + f.paired[repairSsrc] = struct{}{} + f.paired[baseSsrc] = struct{}{} f.lock.Unlock() - if repairSsrc != 0 && baseSsrc != 0 { - f.onRTXPairFound(repairSsrc, baseSsrc, repairSid) - } + f.onRTXPairFound(repairSsrc, baseSsrc, repairSid) } // ------------------------------------------ @@ -134,65 +178,25 @@ func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader return reader } - return &rtxInfoReader{ - tryTimes: rtxProbeCount, - reader: reader, - midExtID: uint8(midExtensionID), - ridExtID: uint8(streamIDExtensionID), - rsidExtID: uint8(repairStreamIDExtensionID), - factory: u.factory, - logger: u.logger, + // Probe on the buffer write path rather than by wrapping this reader. Remote streams + // are consumed through SettingEngine.BufferFactory, so nothing here reads the + // interceptor chain. pion used to drive the repair stream reader, but since + // pion/webrtc#3470 it only does so when the application reads the TrackRemote or + // when no BufferFactory is set, neither of which holds. + ok := u.factory.prober.SetStreamInfoProbe(info.SSRC, &buffer.StreamInfoProbe{ + MidExtID: uint8(midExtensionID), + RidExtID: uint8(streamIDExtensionID), + RsidExtID: uint8(repairStreamIDExtensionID), + Tries: rtxProbeCount, + OnFound: u.factory.SetStreamInfo, + }) + if !ok { + u.logger.Warnw( + "could not install stream info probe, rtx pairing will not work", nil, + "ssrc", info.SSRC, + "mime", info.MimeType, + ) } -} - -// ------------------------------------------ - -type rtxInfoReader struct { - tryTimes int - reader interceptor.RTPReader - midExtID uint8 - ridExtID uint8 - rsidExtID uint8 - factory *RTXInfoExtractorFactory - logger logger.Logger -} - -func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) { - n, a, err := r.reader.Read(b, a) - if r.tryTimes < 0 || err != nil { - return n, a, err - } - - if a == nil { - a = make(interceptor.Attributes) - } - header, err := a.GetRTPHeader(b[:n]) - if err != nil { - return n, a, nil - } - - var mid, rid, rsid string - if payload := header.GetExtension(r.midExtID); payload != nil { - mid = string(payload) - } - - if payload := header.GetExtension(r.ridExtID); payload != nil { - rid = string(payload) - } - - if payload := header.GetExtension(r.rsidExtID); payload != nil { - rsid = string(payload) - } - - if mid != "" && (rid != "" || rsid != "") { - r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC) - r.tryTimes = -1 - go r.factory.SetStreamInfo(header.SSRC, mid, rid, rsid) - } else { - // ignore padding only packet for probe count - if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 { - r.tryTimes-- - } - } - return n, a, nil + + return reader } diff --git a/pkg/sfu/packettrailer/packet_trailer.go b/pkg/sfu/packettrailer/packet_trailer.go index 3b58f6681..d86a4cddc 100644 --- a/pkg/sfu/packettrailer/packet_trailer.go +++ b/pkg/sfu/packettrailer/packet_trailer.go @@ -25,9 +25,11 @@ const ( // StripTrailer returns the number of bytes to strip from the end of an RTP // payload if it contains an LKTS trailer. The trailer is located by checking // for the "LKTS" magic suffix and then reading the XORed trailer_len byte -// immediately before it. Returns 0 if absent or ineligible. -func StripTrailer(payload []byte, marker bool) int { - if !marker || len(payload) < envelopeSize { +// immediately before it. isEndOfFrame must be set only for packets ending an +// encoded frame, i. e. where a trailer could have been appended. Returns 0 if +// absent or ineligible. +func StripTrailer(payload []byte, isEndOfFrame bool) int { + if !isEndOfFrame || len(payload) < envelopeSize { return 0 } diff --git a/pkg/sfu/receiver_base.go b/pkg/sfu/receiver_base.go index ecc08d04b..80874a893 100644 --- a/pkg/sfu/receiver_base.go +++ b/pkg/sfu/receiver_base.go @@ -948,7 +948,7 @@ func (r *ReceiverBase) forwardRTP( } spatialLayer := layer - if extPkt.Spatial >= 0 { + if extPkt.Spatial >= 0 && !sfuutils.IsSimulcastMode(r.videoLayerMode) { // svc packet, take spatial layer info from packet spatialLayer = extPkt.Spatial } diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 0507120c1..f737ccbb5 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -66,6 +66,8 @@ type packetMeta struct { ddBytesSlice []byte // abs-capture-time of packet actBytes []byte + // number of packet trailer bytes stripped when the packet was forwarded + trailerStripped uint8 } func (pm packetMeta) MarshalLogObject(e zapcore.ObjectEncoder) error { @@ -87,6 +89,9 @@ func (pm packetMeta) MarshalLogObject(e zapcore.ObjectEncoder) error { if len(pm.actBytes) != 0 { e.AddInt("actBytes", len(pm.actBytes)) } + if pm.trailerStripped != 0 { + e.AddUint8("trailerStripped", pm.trailerStripped) + } return nil } @@ -178,6 +183,7 @@ func (s *sequencer) push( numCodecBytesIn int, ddBytes []byte, actBytes []byte, + trailerStripped int, ) { s.Lock() defer s.Unlock() @@ -246,6 +252,7 @@ func (s *sequencer) push( marker: marker, layer: layer, numCodecBytesIn: uint8(numCodecBytesIn), + trailerStripped: uint8(trailerStripped), lastNack: s.getRefTime(packetTime), // delay retransmissions after the original transmission } pm := &s.meta[slot] diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index be91af8be..4d769e096 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -29,11 +29,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint64(1); i < 518; i++ { - seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil, 0) } // send the last two out-of-order - seq.push(time.Now().UnixNano(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil) - seq.push(time.Now().UnixNano(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil, 0) + seq.push(time.Now().UnixNano(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil, 0) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} res := seq.getExtPacketMetas(req) @@ -63,14 +63,14 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.extTimestamp, uint64(123)) } - seq.push(time.Now().UnixNano(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil, 0) m := seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) m = seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(time.Now().UnixNano(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil, 0) m = seq.getExtPacketMetas([]uint16{505 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) @@ -83,7 +83,7 @@ func Test_sequencer_flush(t *testing.T) { off := uint16(15) for i := uint64(1); i < 100; i++ { - seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil, 0) } preFlush := []uint16{57 + off, 58 + off} @@ -96,7 +96,7 @@ func Test_sequencer_flush(t *testing.T) { // the sequencer re-initializes on the next push and works normally for new packets for i := uint64(200); i < 210; i++ { - seq.push(time.Now().UnixNano(), i, i+uint64(off), 456, true, 3, nil, 0, nil, nil) + seq.push(time.Now().UnixNano(), i, i+uint64(off), 456, true, 3, nil, 0, nil, nil, 0) } postFlush := []uint16{205 + off} require.Equal(t, 0, len(seq.getExtPacketMetas(postFlush))) // not enough time elapsed yet @@ -200,6 +200,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { len(tt.fields.codecBytesOversized), tt.fields.ddBytesOversized, tt.fields.actBytesOdd, + 0, ) } else { if i.seqNo%2 == 0 { @@ -214,6 +215,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, tt.fields.actBytesEven, + 0, ) } else { n.push( @@ -227,6 +229,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, tt.fields.actBytesOdd, + 0, ) } } @@ -354,6 +357,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, tt.fields.actBytesEven, + 0, ) } else { n.push( @@ -367,6 +371,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, tt.fields.actBytesOdd, + 0, ) } } diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 989dd870e..657ea69e8 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -57,18 +57,20 @@ func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) }) } -func (t *telemetryService) RoomEnded(ctx context.Context, room *livekit.Room) { +func (t *telemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) { t.enqueue(func() { t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventRoomFinished, - Room: room, + Event: webhook.EventRoomFinished, + Room: room, + RoomEndReason: reason, }) t.SendEvent(ctx, &livekit.AnalyticsEvent{ - Type: livekit.AnalyticsEventType_ROOM_ENDED, - Timestamp: timestamppb.Now(), - RoomId: room.Sid, - Room: room, + Type: livekit.AnalyticsEventType_ROOM_ENDED, + Timestamp: timestamppb.Now(), + RoomId: room.Sid, + Room: room, + RoomEndReason: reason, }) }) } @@ -182,6 +184,18 @@ func (t *telemetryService) ParticipantResumed( }) } +// RoomIDChanged re-keys the room's stats workers. +// +// NOTE: this shares the queue with the stats and participant events it races with, so +// ops raised before the id changed (carrying `prevRoomID`) are applied before the +// re-key and ops raised after it (carrying the new id) are applied after. Callers +// should raise this as soon as the room starts reporting the new id. +func (t *telemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) { + t.enqueue(func() { + t.reKeyRoom(prevRoomID, livekit.RoomID(room.Sid), livekit.RoomName(room.Name)) + }) +} + func (t *telemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, @@ -202,6 +216,7 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context, "participant", participant.Identity, "participantID", participant.Sid, "worker", worker, + "guard", guard, ) } } diff --git a/pkg/telemetry/events_test.go b/pkg/telemetry/events_test.go index 03fca0e4a..ae275c89f 100644 --- a/pkg/telemetry/events_test.go +++ b/pkg/telemetry/events_test.go @@ -243,3 +243,18 @@ func Test_OnTrackSubscribed_EventIsSent(t *testing.T) { require.Equal(t, publisherInfo.Identity, eventTrackSubscribed.Publisher.Identity) } + +func Test_OnRoomEnded_ReasonIsSent(t *testing.T) { + fixture := createFixture() + + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} + fixture.sut.RoomEnded(context.Background(), room, livekit.RoomEndReason_ROOM_END_API_DELETE) + + fixture.flush() + + require.Equal(t, 1, fixture.analytics.SendEventCallCount()) + _, event := fixture.analytics.SendEventArgsForCall(0) + require.Equal(t, livekit.AnalyticsEventType_ROOM_ENDED, event.Type) + require.Equal(t, room.Sid, event.RoomId) + require.Equal(t, livekit.RoomEndReason_ROOM_END_API_DELETE, event.RoomEndReason) +} diff --git a/pkg/telemetry/prometheus/node_nonwindows.go b/pkg/telemetry/prometheus/node_nonwindows.go index b765ce271..9661e08fa 100644 --- a/pkg/telemetry/prometheus/node_nonwindows.go +++ b/pkg/telemetry/prometheus/node_nonwindows.go @@ -18,39 +18,8 @@ package prometheus -import ( - "runtime" - "sync" - - "github.com/mackerelio/go-osstat/cpu" - "github.com/mackerelio/go-osstat/loadavg" -) - -var ( - cpuStatsLock sync.RWMutex - lastCPUTotal, lastCPUIdle uint64 -) +import "github.com/mackerelio/go-osstat/loadavg" func getLoadAvg() (*loadavg.Stats, error) { return loadavg.Get() } - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - cpuInfo, err := cpu.Get() - if err != nil { - return - } - - cpuStatsLock.Lock() - if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { - cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) - } - - lastCPUTotal = cpuInfo.Total - lastCPUIdle = cpuInfo.Idle - cpuStatsLock.Unlock() - - numCPUs = uint32(runtime.NumCPU()) - - return -} diff --git a/pkg/telemetry/prometheus/node_windows.go b/pkg/telemetry/prometheus/node_windows.go index eb0ba48a9..507b80437 100644 --- a/pkg/telemetry/prometheus/node_windows.go +++ b/pkg/telemetry/prometheus/node_windows.go @@ -23,7 +23,3 @@ import "github.com/mackerelio/go-osstat/loadavg" func getLoadAvg() (*loadavg.Stats, error) { return &loadavg.Stats{}, nil } - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - return 1, 1, nil -} diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 4729d99dd..f4f9d3dbe 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -626,6 +626,133 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) { require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[1].Kind) } +func Test_RoomIDChangeReKeysStatsWorkers(t *testing.T) { + fixture := createFixture() + + // prepare + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} + partSID := livekit.ParticipantID("part1") + participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} + trackID := livekit.TrackID("trackID") + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) + + stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}} + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) + + // do - the room restarts and gets a new id + restartedRoom := &livekit.Room{Sid: "RestartedSid", Name: "RoomName"} + fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(room.Sid), restartedRoom) + + // stats reported with the new id reach the same worker + stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}} + fixture.sut.TrackStats(livekit.RoomID(restartedRoom.Sid), livekit.RoomName(restartedRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) + + fixture.flush() + + // one worker, one flush, but two stats - each attributed to the session it was collected in + require.Equal(t, 1, fixture.analytics.SendStatsCallCount()) + _, stats := fixture.analytics.SendStatsArgsForCall(0) + require.Equal(t, 2, len(stats)) + + byRoom := map[string]*livekit.AnalyticsStat{} + for _, stat := range stats { + require.Equal(t, string(partSID), stat.ParticipantId) + byRoom[stat.RoomId] = stat + } + require.Len(t, byRoom, 2) + require.Equal(t, uint64(33), byRoom[room.Sid].Streams[0].PrimaryBytes) + require.Equal(t, uint64(44), byRoom[restartedRoom.Sid].Streams[0].PrimaryBytes) + + // the worker moved rather than being duplicated, so closing it out drains everything + fixture.sut.ParticipantLeft(context.Background(), restartedRoom, participantInfo, true, guard) + fixture.flush() + require.Equal(t, 1, fixture.analytics.SendStatsCallCount()) +} + +// a forwarded participant is in more than one room at a time under the same participant +// id, so re-keying one of those rooms must leave the other alone +func Test_RoomIDChangeLeavesForwardedParticipantAlone(t *testing.T) { + fixture := createFixture() + + // prepare - the same participant id in a source room and a forwarding destination room + sourceRoom := &livekit.Room{Sid: "SourceSid", Name: "SourceRoom"} + destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"} + partSID := livekit.ParticipantID("part1") + participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} + trackID := livekit.TrackID("trackID") + fixture.sut.ParticipantJoined(context.Background(), sourceRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{}) + fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{}) + + // do - only the destination room restarts + restartedDest := &livekit.Room{Sid: "RestartedDestSid", Name: "DestRoom"} + fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(destRoom.Sid), restartedDest) + + stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}} + fixture.sut.TrackStats(livekit.RoomID(sourceRoom.Sid), livekit.RoomName(sourceRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) + stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}} + fixture.sut.TrackStats(livekit.RoomID(restartedDest.Sid), livekit.RoomName(restartedDest.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) + + fixture.flush() + + // the source room's worker is untouched, the destination room's worker moved + byRoom := map[string]*livekit.AnalyticsStat{} + for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ { + _, stats := fixture.analytics.SendStatsArgsForCall(i) + for _, stat := range stats { + byRoom[stat.RoomId] = stat + } + } + require.Len(t, byRoom, 2) + require.Equal(t, uint64(33), byRoom[sourceRoom.Sid].Streams[0].PrimaryBytes) + require.Equal(t, sourceRoom.Name, byRoom[sourceRoom.Sid].RoomName) + require.Equal(t, uint64(44), byRoom[restartedDest.Sid].Streams[0].PrimaryBytes) + require.Equal(t, restartedDest.Name, byRoom[restartedDest.Sid].RoomName) +} + +// a re-key should never land on a room that already has workers, but if it does only +// one worker can be keyed at (room, participant) and the superseded one must not be +// left unreachable in the flush list +func Test_RoomIDChangeParticipantCollision(t *testing.T) { + fixture := createFixture() + + // prepare - the same participant id in the room being re-keyed and in its destination + prevRoom := &livekit.Room{Sid: "PrevSid", Name: "PrevRoom"} + destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"} + partSID := livekit.ParticipantID("part1") + participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} + trackID := livekit.TrackID("trackID") + fixture.sut.ParticipantJoined(context.Background(), prevRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{}) + fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{}) + + stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}} + fixture.sut.TrackStats(livekit.RoomID(prevRoom.Sid), livekit.RoomName(prevRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) + + // do + fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(prevRoom.Sid), destRoom) + + // the superseded worker drains what it collected under the room it was in + fixture.flush() + byRoom := map[string]*livekit.AnalyticsStat{} + for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ { + _, stats := fixture.analytics.SendStatsArgsForCall(i) + for _, stat := range stats { + byRoom[stat.RoomId] = stat + } + } + require.Equal(t, uint64(33), byRoom[prevRoom.Sid].Streams[0].PrimaryBytes) + + // the worker already keyed at the destination wins and keeps receiving stats + stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}} + fixture.sut.TrackStats(livekit.RoomID(destRoom.Sid), livekit.RoomName(destRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) + + fixture.flush() + _, stats := fixture.analytics.SendStatsArgsForCall(fixture.analytics.SendStatsCallCount() - 1) + require.Equal(t, 1, len(stats)) + require.Equal(t, destRoom.Sid, stats[0].RoomId) + require.Equal(t, uint64(44), stats[0].Streams[0].PrimaryBytes) +} + func (f *telemetryServiceFixture) flush() { time.Sleep(time.Millisecond * 500) f.sut.FlushStats() diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index 59429a0d9..03ec21a04 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -18,6 +18,7 @@ import ( "context" "sync" "time" + "unsafe" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/timestamppb" @@ -32,6 +33,17 @@ type ReferenceGuard struct { activated, released bool } +func (r *ReferenceGuard) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r != nil { + e.AddUintptr("self", uintptr(unsafe.Pointer(r))) + e.AddBool("activated", r.activated) + e.AddBool("released", r.released) + } + return nil +} + +// ---------------------------------------- + type ReferenceCount struct { count int } @@ -52,6 +64,18 @@ func (s *ReferenceCount) Release(guard *ReferenceGuard) bool { return s.count == 0 } +// Take hands over every reference held, leaving none behind. +func (s *ReferenceCount) Take() int { + count := s.count + s.count = 0 + return count +} + +// Absorb takes on references handed over from elsewhere. +func (s *ReferenceCount) Absorb(count int) { + s.count += count +} + func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddInt("count", s.count) return nil @@ -59,19 +83,39 @@ func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error { // ---------------------------------------- +// statsBatch is stats collected while the worker was in one room +type statsBatch struct { + roomID livekit.RoomID + roomName livekit.RoomName + incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat + outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat +} + +func (b statsBatch) isEmpty() bool { + return len(b.incomingPerTrack) == 0 && len(b.outgoingPerTrack) == 0 +} + +// ---------------------------------------- + // StatsWorker handles participant stats type StatsWorker struct { next *StatsWorker ctx context.Context t TelemetryService - roomID livekit.RoomID - roomName livekit.RoomName participantID livekit.ParticipantID participantIdentity livekit.ParticipantIdentity - isConnected bool - lock sync.RWMutex + lock sync.RWMutex + // the room a worker belongs to can change mid-session, so it is mutable state + // guarded by `lock`. it is kept in sync with the key the worker is filed under in + // telemetryService.workers, see telemetryService.reKeyRoom. + roomID livekit.RoomID + roomName livekit.RoomName + // batches sealed off by a room change, they carry the room they were collected + // in and go out on the next flush + sealed []statsBatch + isConnected bool outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat refCount ReferenceCount @@ -115,7 +159,56 @@ func (s *StatsWorker) ParticipantID() livekit.ParticipantID { return s.participantID } +func (s *StatsWorker) RoomID() livekit.RoomID { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.roomID +} + +// SetRoom re-points the worker at a room. +// +// Stats collected so far are sealed off rather than re-stamped - a room id changes +// because the previous session ended, so what was collected under it belongs to it. +// Sealing keeps the re-key free of any sending, the sealed stats go out on the next +// flush like every other stat. +func (s *StatsWorker) SetRoom(roomID livekit.RoomID, roomName livekit.RoomName) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.roomID == roomID && s.roomName == roomName { + return + } + + if batch := s.sealStatsLocked(); !batch.isEmpty() { + s.sealed = append(s.sealed, batch) + } + + s.roomID = roomID + s.roomName = roomName +} + +// sealStatsLocked hands over everything collected since the last seal, stamped +// with the room it was collected in +func (s *StatsWorker) sealStatsLocked() statsBatch { + batch := statsBatch{ + roomID: s.roomID, + roomName: s.roomName, + incomingPerTrack: s.incomingPerTrack, + outgoingPerTrack: s.outgoingPerTrack, + } + + s.incomingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) + s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) + + return batch +} + func (s *StatsWorker) SetConnected() { + if s == nil { + return + } + s.lock.Lock() s.isConnected = true s.lock.Unlock() @@ -132,19 +225,24 @@ func (s *StatsWorker) Flush(now time.Time, closeWait time.Duration) bool { ts := timestamppb.New(now) s.lock.Lock() - stats := make([]*livekit.AnalyticsStat, 0, len(s.incomingPerTrack)+len(s.outgoingPerTrack)) - - incomingPerTrack := s.incomingPerTrack - s.incomingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) - - outgoingPerTrack := s.outgoingPerTrack - s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) + // anything sealed off by a room change goes out along with the current batch, + // each stamped with the room it was collected in + batches := append(s.sealed, s.sealStatsLocked()) + s.sealed = nil closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > closeWait s.lock.Unlock() - stats = s.collectStats(ts, livekit.StreamType_UPSTREAM, incomingPerTrack, stats) - stats = s.collectStats(ts, livekit.StreamType_DOWNSTREAM, outgoingPerTrack, stats) + numTracks := 0 + for _, batch := range batches { + numTracks += len(batch.incomingPerTrack) + len(batch.outgoingPerTrack) + } + + stats := make([]*livekit.AnalyticsStat, 0, numTracks) + for _, batch := range batches { + stats = s.collectStats(ts, batch, livekit.StreamType_UPSTREAM, stats) + stats = s.collectStats(ts, batch, livekit.StreamType_DOWNSTREAM, stats) + } if len(stats) > 0 { s.t.SendStats(s.ctx, stats) } @@ -167,6 +265,34 @@ func (s *StatsWorker) Close(guard *ReferenceGuard) bool { return ok } +// ForceClose closes the worker irrespective of outstanding references. Used when a +// worker can no longer be reached through the worker map, so that it drains and is +// reaped instead of lingering in the flush list forever. +// +// Its references are handed over to `successor`, the worker that can be reached in its +// place, so that whoever holds one still has a live worker to close. A ReferenceGuard +// records that it activated some worker, not which one, so leaving them behind would +// strand the successor with references it can never see released. +func (s *StatsWorker) ForceClose(successor *StatsWorker) bool { + s.lock.Lock() + if !s.closedAt.IsZero() { + s.lock.Unlock() + return false + } + + s.closedAt = time.Now() + count := s.refCount.Take() + s.lock.Unlock() + + if successor != nil && count != 0 { + successor.lock.Lock() + successor.refCount.Absorb(count) + successor.lock.Unlock() + } + + return true +} + func (s *StatsWorker) Closed(guard *ReferenceGuard) bool { s.lock.Lock() defer s.lock.Unlock() @@ -179,10 +305,15 @@ func (s *StatsWorker) Closed(guard *ReferenceGuard) bool { func (s *StatsWorker) collectStats( ts *timestamppb.Timestamp, + batch statsBatch, streamType livekit.StreamType, - perTrack map[livekit.TrackID][]*livekit.AnalyticsStat, stats []*livekit.AnalyticsStat, ) []*livekit.AnalyticsStat { + perTrack := batch.incomingPerTrack + if streamType == livekit.StreamType_DOWNSTREAM { + perTrack = batch.outgoingPerTrack + } + for trackID, analyticsStats := range perTrack { coalesced := coalesce(analyticsStats) if coalesced == nil { @@ -192,15 +323,19 @@ func (s *StatsWorker) collectStats( coalesced.TimeStamp = ts coalesced.TrackId = string(trackID) coalesced.Kind = streamType - coalesced.RoomId = string(s.roomID) + coalesced.RoomId = string(batch.roomID) coalesced.ParticipantId = string(s.participantID) - coalesced.RoomName = string(s.roomName) + coalesced.RoomName = string(batch.roomName) stats = append(stats, coalesced) } return stats } func (s *StatsWorker) MarshalLogObject(e zapcore.ObjectEncoder) error { + if s == nil { + return nil + } + s.lock.RLock() defer s.lock.RUnlock() diff --git a/pkg/telemetry/statsworker_test.go b/pkg/telemetry/statsworker_test.go index f7ae48816..82997f469 100644 --- a/pkg/telemetry/statsworker_test.go +++ b/pkg/telemetry/statsworker_test.go @@ -1,9 +1,13 @@ package telemetry import ( + "context" "testing" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/livekit/protocol/livekit" ) func TestStatsWorker(t *testing.T) { @@ -16,4 +20,83 @@ func TestStatsWorker(t *testing.T) { require.True(t, w.Close(&g1)) require.True(t, w.Closed(&g1)) }) + + // a ReferenceGuard records that it activated some worker, not which one, so a + // superseded worker has to hand its references to the one reachable in its place + t.Run("force close hands references to the successor", func(t *testing.T) { + t.Run("a guard shared by both workers", func(t *testing.T) { + // the second worker never got a reference, the guard was already activated + var g ReferenceGuard + superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g) + survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &g) + require.Equal(t, 1, superseded.refCount.count) + require.Equal(t, 0, survivor.refCount.count) + + require.True(t, superseded.ForceClose(survivor)) + require.Equal(t, 0, superseded.refCount.count) + require.Equal(t, 1, survivor.refCount.count) + + // without the hand over this would leave the survivor at -1 and never closed + require.True(t, survivor.Close(&g)) + require.True(t, survivor.Closed(&g)) + }) + + t.Run("a guard per worker", func(t *testing.T) { + var gSuperseded, gSurvivor ReferenceGuard + superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &gSuperseded) + survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &gSurvivor) + + require.True(t, superseded.ForceClose(survivor)) + require.Equal(t, 2, survivor.refCount.count) + + // the superseded worker's owner departs, it must not close the survivor early + require.False(t, survivor.Close(&gSuperseded)) + require.True(t, survivor.Close(&gSurvivor)) + }) + + t.Run("closing an already closed worker holds on to its references", func(t *testing.T) { + var g ReferenceGuard + superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g) + survivor := newStatsWorker(t.Context(), nil, "", "", "", "", nil) + + require.True(t, superseded.ForceClose(nil)) + require.False(t, superseded.ForceClose(survivor)) + require.Equal(t, 0, survivor.refCount.count) + }) + }) + + t.Run("logging a nil worker does not panic", func(t *testing.T) { + var w *StatsWorker + require.NoError(t, w.MarshalLogObject(zapcore.NewMapObjectEncoder())) + }) +} + +func TestGetOrCreateWorkerReleasedGuard(t *testing.T) { + // ParticipantActive overtaken by the participant's close arrives with a guard that + // ParticipantLeft already released. It must not replace the closed worker with one + // nothing can release. + ts := &telemetryService{workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker)} + roomID, pID := livekit.RoomID("room"), livekit.ParticipantID("participant") + + var g ReferenceGuard + w, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g) + require.False(t, found) + require.True(t, w.Close(&g)) + + t.Run("closed worker still in the map", func(t *testing.T) { + late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g) + require.True(t, found) + require.Same(t, w, late) + require.Same(t, w, ts.workers[roomID][pID]) + }) + + t.Run("closed worker already reaped", func(t *testing.T) { + delete(ts.workers[roomID], pID) + + late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g) + require.True(t, found) + require.Nil(t, late) + require.Empty(t, ts.workers[roomID]) + late.SetConnected() + }) } diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 25a241bef..12d542127 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -129,11 +129,19 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.ReportInfo } - RoomEndedStub func(context.Context, *livekit.Room) + RoomEndedStub func(context.Context, *livekit.Room, livekit.RoomEndReason) roomEndedMutex sync.RWMutex roomEndedArgsForCall []struct { arg1 context.Context arg2 *livekit.Room + arg3 livekit.RoomEndReason + } + RoomIDChangedStub func(context.Context, livekit.RoomID, *livekit.Room) + roomIDChangedMutex sync.RWMutex + roomIDChangedArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomID + arg3 *livekit.Room } RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter roomProjectReporterMutex sync.RWMutex @@ -880,17 +888,18 @@ func (fake *FakeTelemetryService) ReportArgsForCall(i int) (context.Context, *li return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeTelemetryService) RoomEnded(arg1 context.Context, arg2 *livekit.Room) { +func (fake *FakeTelemetryService) RoomEnded(arg1 context.Context, arg2 *livekit.Room, arg3 livekit.RoomEndReason) { fake.roomEndedMutex.Lock() fake.roomEndedArgsForCall = append(fake.roomEndedArgsForCall, struct { arg1 context.Context arg2 *livekit.Room - }{arg1, arg2}) + arg3 livekit.RoomEndReason + }{arg1, arg2, arg3}) stub := fake.RoomEndedStub - fake.recordInvocation("RoomEnded", []interface{}{arg1, arg2}) + fake.recordInvocation("RoomEnded", []interface{}{arg1, arg2, arg3}) fake.roomEndedMutex.Unlock() if stub != nil { - fake.RoomEndedStub(arg1, arg2) + fake.RoomEndedStub(arg1, arg2, arg3) } } @@ -900,17 +909,51 @@ func (fake *FakeTelemetryService) RoomEndedCallCount() int { return len(fake.roomEndedArgsForCall) } -func (fake *FakeTelemetryService) RoomEndedCalls(stub func(context.Context, *livekit.Room)) { +func (fake *FakeTelemetryService) RoomEndedCalls(stub func(context.Context, *livekit.Room, livekit.RoomEndReason)) { fake.roomEndedMutex.Lock() defer fake.roomEndedMutex.Unlock() fake.RoomEndedStub = stub } -func (fake *FakeTelemetryService) RoomEndedArgsForCall(i int) (context.Context, *livekit.Room) { +func (fake *FakeTelemetryService) RoomEndedArgsForCall(i int) (context.Context, *livekit.Room, livekit.RoomEndReason) { fake.roomEndedMutex.RLock() defer fake.roomEndedMutex.RUnlock() argsForCall := fake.roomEndedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeTelemetryService) RoomIDChanged(arg1 context.Context, arg2 livekit.RoomID, arg3 *livekit.Room) { + fake.roomIDChangedMutex.Lock() + fake.roomIDChangedArgsForCall = append(fake.roomIDChangedArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomID + arg3 *livekit.Room + }{arg1, arg2, arg3}) + stub := fake.RoomIDChangedStub + fake.recordInvocation("RoomIDChanged", []interface{}{arg1, arg2, arg3}) + fake.roomIDChangedMutex.Unlock() + if stub != nil { + fake.RoomIDChangedStub(arg1, arg2, arg3) + } +} + +func (fake *FakeTelemetryService) RoomIDChangedCallCount() int { + fake.roomIDChangedMutex.RLock() + defer fake.roomIDChangedMutex.RUnlock() + return len(fake.roomIDChangedArgsForCall) +} + +func (fake *FakeTelemetryService) RoomIDChangedCalls(stub func(context.Context, livekit.RoomID, *livekit.Room)) { + fake.roomIDChangedMutex.Lock() + defer fake.roomIDChangedMutex.Unlock() + fake.RoomIDChangedStub = stub +} + +func (fake *FakeTelemetryService) RoomIDChangedArgsForCall(i int) (context.Context, livekit.RoomID, *livekit.Room) { + fake.roomIDChangedMutex.RLock() + defer fake.roomIDChangedMutex.RUnlock() + argsForCall := fake.roomIDChangedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeTelemetryService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter { diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index bdf9c820c..f316f6e4f 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" @@ -35,7 +36,7 @@ type TelemetryService interface { // events RoomStarted(ctx context.Context, room *livekit.Room) - RoomEnded(ctx context.Context, room *livekit.Room) + RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) // ParticipantJoined - a participant establishes signal connection to a room ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) @@ -45,6 +46,9 @@ type TelemetryService interface { ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) // ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) + // RoomIDChanged - the room kept its session, but got a different id (a provisional room id + // replaced by the resolved one), re-keys the stats workers of every participant in the room + RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) // TrackPublishRequested - a publication attempt has been received TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) // TrackPublished - a publication attempt has been successful @@ -105,7 +109,8 @@ type NullTelemetryService struct { func (n NullTelemetryService) TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) { } func (n NullTelemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {} -func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room) {} +func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) { +} func (n NullTelemetryService) ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) { } func (n NullTelemetryService) ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, isWarp bool, guard *ReferenceGuard) { @@ -114,6 +119,8 @@ func (n NullTelemetryService) ParticipantResumed(ctx context.Context, room *live } func (n NullTelemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) { } +func (n NullTelemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) { +} func (n NullTelemetryService) TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) { } func (n NullTelemetryService) TrackPublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) { @@ -165,11 +172,6 @@ const ( telemetryStatsUpdateInterval = time.Second * 30 ) -type statsWorkerKey struct { - roomID livekit.RoomID - participantID livekit.ParticipantID -} - type telemetryService struct { AnalyticsService @@ -177,7 +179,7 @@ type telemetryService struct { jobsQueue *utils.OpsQueue workersMu sync.RWMutex - workers map[statsWorkerKey]*StatsWorker + workers map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker workerList *StatsWorker flushMu sync.Mutex @@ -193,7 +195,7 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer FlushOnStop: true, Logger: logger.GetLogger(), }), - workers: make(map[statsWorkerKey]*StatsWorker), + workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker), } t.jobsQueue.Start() @@ -242,9 +244,12 @@ func (t *telemetryService) FlushStats() { if reap != nil { t.workersMu.Lock() for reap != nil { - key := statsWorkerKey{reap.roomID, reap.participantID} - if reap == t.workers[key] { - delete(t.workers, key) + roomID := reap.RoomID() + if roomWorkers := t.workers[roomID]; reap == roomWorkers[reap.participantID] { + delete(roomWorkers, reap.participantID) + if len(roomWorkers) == 0 { + delete(t.workers, roomID) + } } reap = reap.next } @@ -266,7 +271,7 @@ func (t *telemetryService) getWorker(roomID livekit.RoomID, participantID liveki t.workersMu.RLock() defer t.workersMu.RUnlock() - worker, ok = t.workers[statsWorkerKey{roomID, participantID}] + worker, ok = t.workers[roomID][participantID] return } @@ -281,17 +286,50 @@ func (t *telemetryService) getOrCreateWorker( t.workersMu.Lock() defer t.workersMu.Unlock() - key := statsWorkerKey{roomID, participantID} - worker, ok := t.workers[key] + if roomID == "" { + logger.Warnw( + "telemetry stats worker keyed under an empty room id", nil, + "room", roomName, + "participant", participantIdentity, + "participantID", participantID, + "guard", guard, + ) + } + + roomWorkers := t.workers[roomID] + worker, ok := roomWorkers[participantID] if ok && !worker.Closed(guard) { return worker, true } + // only ParticipantLeft releases a guard, so a released guard is a call landing after + // the participant left, e.g. ParticipantActive overtaken by the close. Do not create + // a worker nothing can ever release. The closed worker, if not yet reaped, is returned + // as found, otherwise nil is + if guard != nil && guard.released { + return worker, true + } + existingIsConnected := false if ok { existingIsConnected = worker.IsConnected() } + // a guard references at most once, so a nil or already activated guard leaves the + // new worker with no references and its owner's release drives it negative + if guard == nil || guard.activated { + logger.Infow( + "telemetry stats worker created without a reference", + "room", roomName, + "roomID", roomID, + "participant", participantIdentity, + "participantID", participantID, + "guard", guard, + "replacedClosed", ok, + "existing", worker, + ) + } + worker = newStatsWorker( ctx, t, @@ -305,7 +343,11 @@ func (t *telemetryService) getOrCreateWorker( worker.SetConnected() } - t.workers[key] = worker + if roomWorkers == nil { + roomWorkers = make(map[livekit.ParticipantID]*StatsWorker) + t.workers[roomID] = roomWorkers + } + roomWorkers[participantID] = worker worker.next = t.workerList t.workerList = worker @@ -313,6 +355,80 @@ func (t *telemetryService) getOrCreateWorker( return worker, false } +// reKeyRoom files every one of a room's stats workers under `roomID` instead of +// `prevRoomID`. +// +// A room can be restarted while participants are connected and reporting stats, which +// gives it a new id. As every worker of the room moves at once, the move is a single map +// splice - the workers themselves are untouched and keep their place in the flush list. +// Each worker then seals off what it collected under `prevRoomID` so those stats stay +// attributed to the session that ended (see StatsWorker.SetRoom). +func (t *telemetryService) reKeyRoom(prevRoomID livekit.RoomID, roomID livekit.RoomID, roomName livekit.RoomName) { + if prevRoomID == roomID { + return + } + + t.workersMu.Lock() + defer t.workersMu.Unlock() + + roomWorkers := t.workers[prevRoomID] + if len(roomWorkers) == 0 { + delete(t.workers, prevRoomID) + return + } + delete(t.workers, prevRoomID) + + existing := t.workers[roomID] + if existing == nil { + t.workers[roomID] = roomWorkers + } else { + // should not happen as a room id is only ever replaced by a freshly minted one + logger.Warnw( + "telemetry re-keying room into an existing entry", nil, + "prevRoomID", prevRoomID, + "room", roomName, + "roomID", roomID, + "numWorkers", len(roomWorkers), + "numExistingWorkers", len(existing), + ) + } + + for participantID, worker := range roomWorkers { + if existing != nil { + if survivor, ok := existing[participantID]; ok { + // only one worker can be keyed at (room, participant) and the one already + // filed there wins, close the superseded one so that it drains and is + // reaped instead of lingering in the flush list unreachable + forceClosed := worker.ForceClose(survivor) + logger.Infow( + "telemetry force closing superseded stats worker", + "prevRoomID", prevRoomID, + "roomID", roomID, + "participantID", participantID, + "forceClosed", forceClosed, + "superseded", worker, + "survivor", survivor, + ) + if forceClosed { + prometheus.SubParticipant() + } + continue + } + existing[participantID] = worker + } + + worker.SetRoom(roomID, roomName) + } + + logger.Infow( + "telemetry re-keyed room", + "prevRoomID", prevRoomID, + "room", roomName, + "roomID", roomID, + "numWorkers", len(roomWorkers), + ) +} + func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) { t.enqueue(func() { t.SendNodeRoomStates(ctx, info) diff --git a/pkg/testutils/vnettest/vnettest.go b/pkg/testutils/vnettest/vnettest.go new file mode 100644 index 000000000..eade3314d --- /dev/null +++ b/pkg/testutils/vnettest/vnettest.go @@ -0,0 +1,231 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vnettest sets up real pion peer connections on an in-memory virtual +// network, for integration tests that exercise media paths without a server. +// +// It depends only on pion, so it can be imported both by tests inside pkg/... and by +// the top level test package. +package vnettest + +import ( + "fmt" + "io" + "sync" + "testing" + "time" + + "github.com/pion/interceptor" + "github.com/pion/logging" + "github.com/pion/sdp/v3" + "github.com/pion/transport/v4/packetio" + "github.com/pion/transport/v4/vnet" + "github.com/pion/webrtc/v4" + "github.com/stretchr/testify/require" +) + +const ( + VP8PayloadType = 96 + RTXPayloadType = VP8PayloadType + 1 + OpusPayloadType = 111 +) + +// Hosts are the two ends of a started virtual network. +type Hosts struct { + OfferNet *vnet.Net + AnswerNet *vnet.Net +} + +// NewHosts returns two hosts on a started virtual network, torn down with the test. +func NewHosts(t *testing.T) *Hosts { + t.Helper() + + wan, err := vnet.NewRouter(&vnet.RouterConfig{ + CIDR: "1.2.3.0/24", + LoggerFactory: logging.NewDefaultLoggerFactory(), + }) + require.NoError(t, err) + + offerNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"1.2.3.4"}}) + require.NoError(t, err) + require.NoError(t, wan.AddNet(offerNet)) + + answerNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"1.2.3.5"}}) + require.NoError(t, err) + require.NoError(t, wan.AddNet(answerNet)) + + require.NoError(t, wan.Start()) + t.Cleanup(func() { _ = wan.Stop() }) + + return &Hosts{OfferNet: offerNet, AnswerNet: answerNet} +} + +// NewSettingEngine returns a setting engine bound to net, with ICE timeouts short +// enough to keep tests quick. +func NewSettingEngine(net *vnet.Net) webrtc.SettingEngine { + se := webrtc.SettingEngine{} + se.SetNet(net) + se.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + se.SetICETimeouts(5*time.Second, 5*time.Second, 500*time.Millisecond) + return se +} + +// MediaEngineConfig describes what to register on a media engine. +type MediaEngineConfig struct { + Video bool // VP8 and its RTX codec; otherwise opus + HeaderExtensions bool // abs-send-time + transport-cc + SimulcastExtensions bool // mid + rid + rsid +} + +func VideoRTCPFeedback() []webrtc.RTCPFeedback { + return []webrtc.RTCPFeedback{ + {Type: webrtc.TypeRTCPFBNACK}, + {Type: webrtc.TypeRTCPFBNACK, Parameter: "pli"}, + {Type: webrtc.TypeRTCPFBTransportCC}, + {Type: webrtc.TypeRTCPFBGoogREMB}, + } +} + +func NewMediaEngine(t *testing.T, cfg MediaEngineConfig) *webrtc.MediaEngine { + t.Helper() + + me := &webrtc.MediaEngine{} + kind := webrtc.RTPCodecTypeAudio + if cfg.Video { + kind = webrtc.RTPCodecTypeVideo + + require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: VideoRTCPFeedback(), + }, + PayloadType: VP8PayloadType, + }, kind)) + require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeRTX, + ClockRate: 90000, + SDPFmtpLine: fmt.Sprintf("apt=%d", VP8PayloadType), + }, + PayloadType: RTXPayloadType, + }, kind)) + } else { + require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, + }, + PayloadType: OpusPayloadType, + }, kind)) + } + + if cfg.HeaderExtensions { + require.NoError(t, me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: sdp.ABSSendTimeURI}, kind)) + require.NoError(t, me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: sdp.TransportCCURI}, kind)) + } + if cfg.SimulcastExtensions { + require.NoError(t, webrtc.ConfigureSimulcastExtensionHeaders(me)) + } + + return me +} + +// PCConfig describes a peer connection on the virtual network. +type PCConfig struct { + Net *vnet.Net + MediaEngine MediaEngineConfig + + // BufferFactory is SettingEngine.BufferFactory, e. g. buffer.Factory.GetOrNew. + // Optional. + BufferFactory func(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser +} + +// NewPeerConnection builds a peer connection on the virtual network with no +// interceptors, so nothing rewrites what a test puts on the wire. +func NewPeerConnection(t *testing.T, cfg PCConfig) *webrtc.PeerConnection { + t.Helper() + + se := NewSettingEngine(cfg.Net) + se.BufferFactory = cfg.BufferFactory + + api := webrtc.NewAPI( + webrtc.WithMediaEngine(NewMediaEngine(t, cfg.MediaEngine)), + webrtc.WithSettingEngine(se), + webrtc.WithInterceptorRegistry(&interceptor.Registry{}), + ) + + pc, err := api.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + t.Cleanup(func() { _ = pc.Close() }) + + return pc +} + +// GatheredOffer creates an offer and waits for gathering, so the SDP carries every +// candidate and the caller needs no trickle. +func GatheredOffer(t *testing.T, pc *webrtc.PeerConnection) webrtc.SessionDescription { + t.Helper() + + offer, err := pc.CreateOffer(nil) + require.NoError(t, err) + + gathered := webrtc.GatheringCompletePromise(pc) + require.NoError(t, pc.SetLocalDescription(offer)) + <-gathered + + return *pc.LocalDescription() +} + +// SignalPair performs a full offer/answer exchange between two peer connections and +// waits for both to connect. +func SignalPair(t *testing.T, offerer, answerer *webrtc.PeerConnection) { + t.Helper() + + connected := UntilConnected(offerer, answerer) + + require.NoError(t, answerer.SetRemoteDescription(GatheredOffer(t, offerer))) + + answer, err := answerer.CreateAnswer(nil) + require.NoError(t, err) + gathered := webrtc.GatheringCompletePromise(answerer) + require.NoError(t, answerer.SetLocalDescription(answer)) + <-gathered + + require.NoError(t, offerer.SetRemoteDescription(*answerer.LocalDescription())) + + select { + case <-connected: + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for peer connections to connect") + } +} + +// UntilConnected closes the returned channel once every peer connection is connected. +func UntilConnected(pcs ...*webrtc.PeerConnection) <-chan struct{} { + var wg sync.WaitGroup + wg.Add(len(pcs)) + for _, pc := range pcs { + var once sync.Once + pc.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { + if s == webrtc.PeerConnectionStateConnected { + once.Do(wg.Done) + } + }) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + return done +} diff --git a/pkg/utils/changenotifier_test.go b/pkg/utils/changenotifier_test.go new file mode 100644 index 000000000..c98920a82 --- /dev/null +++ b/pkg/utils/changenotifier_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestChangeNotifier(t *testing.T) { + t.Run("Observer management", func(t *testing.T) { + notifier := NewChangeNotifier() + require.False(t, notifier.HasObservers()) + + called := false + notifier.AddObserver("test-key", func() { + called = true + }) + require.True(t, notifier.HasObservers()) + + notifier.RemoveObserver("test-key") + require.False(t, notifier.HasObservers()) + require.False(t, called) + }) + + t.Run("Notification triggers callbacks asynchronously", func(t *testing.T) { + notifier := NewChangeNotifier() + var wg sync.WaitGroup + wg.Add(2) + + var mu sync.Mutex + callCounts := make(map[string]int) + + notifier.AddObserver("obs1", func() { + mu.Lock() + callCounts["obs1"]++ + mu.Unlock() + wg.Done() + }) + + notifier.AddObserver("obs2", func() { + mu.Lock() + callCounts["obs2"]++ + mu.Unlock() + wg.Done() + }) + + notifier.NotifyChanged() + + // Wait for async execution of observers + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + // Success + case <-time.After(1 * time.Second): + t.Fatal("Timeout waiting for change notification callbacks") + } + + mu.Lock() + require.Equal(t, 1, callCounts["obs1"]) + require.Equal(t, 1, callCounts["obs2"]) + mu.Unlock() + }) +} + +func TestChangeNotifierManager(t *testing.T) { + t.Run("Get and Create Notifiers", func(t *testing.T) { + manager := NewChangeNotifierManager() + require.Nil(t, manager.GetNotifier("non-existent")) + + notifier := manager.GetOrCreateNotifier("room1") + require.NotNil(t, notifier) + + retrieved := manager.GetNotifier("room1") + require.Equal(t, notifier, retrieved) + + // GetOrCreate should return the existing one + again := manager.GetOrCreateNotifier("room1") + require.Equal(t, notifier, again) + }) + + t.Run("Remove Notifiers with HasObservers check", func(t *testing.T) { + manager := NewChangeNotifierManager() + _ = manager.GetOrCreateNotifier("room1") + + // Case 1: notifier has no observers, should be removed + manager.RemoveNotifier("room1", false) + require.Nil(t, manager.GetNotifier("room1")) + + // Re-create and add an observer + notifier := manager.GetOrCreateNotifier("room1") + notifier.AddObserver("observer", func() {}) + + // Case 2: notifier has observer, RemoveNotifier(..., false) should not remove it + manager.RemoveNotifier("room1", false) + require.NotNil(t, manager.GetNotifier("room1")) + + // Case 3: notifier has observer, RemoveNotifier(..., true) (force) should remove it + manager.RemoveNotifier("room1", true) + require.Nil(t, manager.GetNotifier("room1")) + }) +} diff --git a/pkg/utils/math.go b/pkg/utils/math.go index 3a419db2b..253514461 100644 --- a/pkg/utils/math.go +++ b/pkg/utils/math.go @@ -14,24 +14,45 @@ package utils -import "slices" +import ( + "cmp" + "slices" +) -// Median gets median value for an array -func Median[T float32](input []T) T { +// OrderedNumber defines a constraint for numeric types that can be ordered and divided. +type OrderedNumber interface { + cmp.Ordered + ~int | ~int8 | ~int16 | ~int32 | ~int64 | + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | + ~uintptr | ~float32 | ~float64 +} + +// Median gets the median value for a slice without modifying the original slice. +// +// Note: +// 1. For integer types, if the slice has an even length, the division (/ 2) +// is performed using integer division, which truncates the result towards zero. +// 2. Uses an overflow-safe formula left + (right-left)/2 to support narrow integer types. +func Median[T OrderedNumber](input []T) T { num := len(input) switch num { case 0: - return 0 + var zero T + return zero case 1: return input[0] } - slices.Sort(input) + + // Clone the slice to avoid mutating the caller's slice + sortedInput := slices.Clone(input) + slices.Sort(sortedInput) + if num%2 != 0 { - return input[num/2] + return sortedInput[num/2] } - left := input[num/2-1] - right := input[num/2] - return (left + right) / 2 + left := sortedInput[num/2-1] + right := sortedInput[num/2] + return left + (right-left)/T(2) } func Signum[T int | int8 | int16 | int32 | int64 | float32 | float64](val T) int { diff --git a/pkg/utils/math_test.go b/pkg/utils/math_test.go new file mode 100644 index 000000000..48014b9b9 --- /dev/null +++ b/pkg/utils/math_test.go @@ -0,0 +1,89 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMedian(t *testing.T) { + t.Run("Empty slice", func(t *testing.T) { + require.Equal(t, float32(0), Median([]float32{})) + require.Equal(t, int(0), Median([]int{})) + }) + + t.Run("Single element", func(t *testing.T) { + require.Equal(t, float32(42), Median([]float32{42})) + require.Equal(t, int(42), Median([]int{42})) + }) + + t.Run("Odd length float32", func(t *testing.T) { + input := []float32{3.0, 1.0, 2.0} + require.Equal(t, float32(2.0), Median(input)) + }) + + t.Run("Even length float32 - exact average", func(t *testing.T) { + input := []float32{1.0, 2.0, 3.0, 4.0} + require.Equal(t, float32(2.5), Median(input)) + }) + + t.Run("Even length int - integer truncation", func(t *testing.T) { + input := []int{1, 2} + // (1 + 2) / 2 = 1.5 -> truncates to 1 + require.Equal(t, int(1), Median(input)) + + inputOddAverage := []int{1, 3} + // (1 + 3) / 2 = 2 + require.Equal(t, int(2), Median(inputOddAverage)) + }) + + t.Run("Int8 overflow prevention", func(t *testing.T) { + // Without overflow protection: 120 + 126 = 246 (overflows int8 to -10) -> -10 / 2 = -5 + // With overflow protection: 120 + (126-120)/2 = 123 + input := []int8{120, 126} + require.Equal(t, int8(123), Median(input)) + }) + + t.Run("Uint8 overflow prevention", func(t *testing.T) { + input := []uint8{250, 254} + require.Equal(t, uint8(252), Median(input)) + }) + + t.Run("Immutability test - caller slice is not sorted/mutated", func(t *testing.T) { + original := []int{3, 1, 4, 2} + input := slices.Clone(original) + + median := Median(input) + require.Equal(t, int(2), median) + require.Equal(t, original, input, "Input slice must not be modified by Median") + }) +} + +func TestSignum(t *testing.T) { + t.Run("Integer values", func(t *testing.T) { + require.Equal(t, -1, Signum(-42)) + require.Equal(t, 0, Signum(0)) + require.Equal(t, 1, Signum(42)) + }) + + t.Run("Floating point values", func(t *testing.T) { + require.Equal(t, -1, Signum(float32(-0.01))) + require.Equal(t, 0, Signum(float32(0.0))) + require.Equal(t, 1, Signum(float32(0.01))) + }) +} diff --git a/pkg/utils/slice_test.go b/pkg/utils/slice_test.go new file mode 100644 index 000000000..055c0c4c3 --- /dev/null +++ b/pkg/utils/slice_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDedupeSlice(t *testing.T) { + t.Run("Empty slice", func(t *testing.T) { + var input []int + result := DedupeSlice(input) + require.Empty(t, result) + }) + + t.Run("Single element", func(t *testing.T) { + input := []string{"hello"} + result := DedupeSlice(input) + require.Equal(t, []string{"hello"}, result) + }) + + t.Run("Unsorted slice with duplicates", func(t *testing.T) { + input := []int{4, 2, 4, 1, 3, 2} + result := DedupeSlice(input) + require.Equal(t, []int{1, 2, 3, 4}, result) + }) + + t.Run("Already sorted and unique", func(t *testing.T) { + input := []string{"apple", "banana", "cherry"} + result := DedupeSlice(input) + require.Equal(t, []string{"apple", "banana", "cherry"}, result) + }) +} diff --git a/renovate.json b/renovate.json index f4320b2d0..f2106f3b1 100644 --- a/renovate.json +++ b/renovate.json @@ -36,11 +36,12 @@ "groupName": "pion deps" }, { - "description": "First-party deps, no need to quarantine new releases", + "description": "First-party deps, no need to quarantine new releases; they are co-released and depend on each other, so they are grouped into a single PR", "matchManagers": ["gomod"], "matchPackageNames": [ "github.com/livekit{/,}**" ], + "groupName": "livekit deps", "minimumReleaseAge": null }, { diff --git a/test/client/client.go b/test/client/client.go index bb38cdaf3..7a74c4841 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -46,10 +46,10 @@ import ( "github.com/livekit/protocol/signalling" "github.com/livekit/livekit-server/pkg/rtc" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/datatrack" ) type SignalRequestHandler func(msg *livekit.SignalRequest) error @@ -867,9 +867,9 @@ func (c *RTCClient) SetAttributes(attrs map[string]string) error { func (c *RTCClient) hasPrimaryEverConnected() bool { if c.subscriberAsPrimary.Load() { - return c.subscriber.HasEverConnected() + return c.subscriber.PeerConnectionHasEverConnected() } else { - return c.publisher.HasEverConnected() + return c.publisher.PeerConnectionHasEverConnected() } } @@ -1113,7 +1113,7 @@ func (c *RTCClient) ensurePublisherConnected() error { return c.ctx.Err() } - if c.publisher.HasEverConnected() { + if c.publisher.PeerConnectionHasEverConnected() { return nil } diff --git a/test/client/datatrack_remote.go b/test/client/datatrack_remote.go index 2d2f66365..97097620a 100644 --- a/test/client/datatrack_remote.go +++ b/test/client/datatrack_remote.go @@ -2,7 +2,7 @@ package client import ( "github.com/frostbyte73/core" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/protocol/datatrack" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "go.uber.org/atomic" diff --git a/test/client/datatrack_writer.go b/test/client/datatrack_writer.go index 82c6a188b..aaadde9f6 100644 --- a/test/client/datatrack_writer.go +++ b/test/client/datatrack_writer.go @@ -19,8 +19,9 @@ import ( "math/rand" "time" - "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/datatrack" + "github.com/livekit/protocol/datatrack/datatracktest" "github.com/livekit/protocol/logger" ) @@ -59,7 +60,7 @@ func (d *dataTrackWriter) writeFrames() { return default: - packets := datatrack.GenerateRawDataPackets(d.handle, seqNum, frameNum, 1, rand.Intn(2048)+1, 100*time.Millisecond) + packets := datatracktest.GenerateRawDataPackets(d.handle, seqNum, frameNum, 1, rand.Intn(2048)+1, 100*time.Millisecond) for _, packet := range packets { if err := d.transport.SendDataTrackMessage(packet); err != nil { logger.Errorw("could not send data track packet", err) diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 0ac06febd..ec3469620 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -18,11 +18,14 @@ import ( "context" "fmt" "net/http" + "strings" "sync" "testing" "time" + "github.com/pion/transport/v4/vnet" "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" "github.com/twitchtv/twirp" "github.com/livekit/mediatransportutil/pkg/rtcconfig" @@ -33,9 +36,12 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/livekit-server/pkg/testutils/vnettest" testclient "github.com/livekit/livekit-server/test/client" ) @@ -386,3 +392,67 @@ func stopClients(clients ...*testclient.RTCClient) { c.Stop() } } + +// ----------------------------------------------------------------------------- +// vnet media harness +// +// Setup specific to driving a real server transport over a virtual network. The +// pion side lives in pkg/testutils/vnettest, shared with the pkg/sfu media tests. +// ----------------------------------------------------------------------------- + +// newVNetWebRTCConfig builds the server side WebRTCConfig on net. The direction +// configs come from the production NewWebRTCConfig so the negotiated extensions and +// feedback stay in step with it; the setting engine is replaced so no real socket or +// ICE mux is bound. +func newVNetWebRTCConfig(t *testing.T, net *vnet.Net, bufferFactory *buffer.Factory) *rtc.WebRTCConfig { + t.Helper() + + conf, err := config.NewConfig("", true, nil, nil) + require.NoError(t, err) + + // an ephemeral port range instead of the dev mode single port, which would bind a mux + conf.RTC.TCPPort = 0 + conf.RTC.UDPPort = rtcconfig.PortRange{} + conf.RTC.ICEPortRangeStart = 50000 + conf.RTC.ICEPortRangeEnd = 60000 + + rtcConf, err := rtc.NewWebRTCConfig(conf) + require.NoError(t, err) + require.Nil(t, rtcConf.UDPMux, "test config must not bind a udp mux") + + rtcConf.SettingEngine = vnettest.NewSettingEngine(net) + rtcConf.SetBufferFactory(bufferFactory) + + return rtcConf +} + +// stripDeclaredSSRCs removes the a=ssrc lines pion puts in its offer. Browsers doing +// rid based simulcast do not declare per-layer SSRCs, which is why a repair SSRC has to +// be learned at all; leaving them in would let the receiver resolve everything from SDP. +func stripDeclaredSSRCs(offer string) string { + lines := strings.Split(offer, "\r\n") + filtered := lines[:0] + for _, line := range lines { + if strings.HasPrefix(line, "a=ssrc") { + continue + } + filtered = append(filtered, line) + } + return strings.Join(filtered, "\r\n") +} + +// sendUntil calls send every 20ms until done reports true or the timeout expires, +// returning the final state of done. +func sendUntil(t *testing.T, timeout time.Duration, done func() bool, send func()) bool { + t.Helper() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if done() { + return true + } + send() + time.Sleep(20 * time.Millisecond) + } + return done() +} diff --git a/test/rtx_pairing_integration_test.go b/test/rtx_pairing_integration_test.go new file mode 100644 index 000000000..7a497bb8e --- /dev/null +++ b/test/rtx_pairing_integration_test.go @@ -0,0 +1,562 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +// RTX repair stream pairing on simulcast (rid) streams, over a real PCTransport driven +// by a real pion publisher on a virtual network. +// +// RID based simulcast has no a=ssrc-group:FID line, so the pairing cannot come from +// SDP. The repair SSRC of a layer comes either from the mid/rsid header extensions on +// its packets, or - for a migrated publisher, which is mid-stream and no longer sends +// those extensions - from the migration info in TransportParams.SimTracks. +// +// Neither source fails loudly when it breaks: the repair buffer just accumulates +// packets that are never applied and NACK recovery for simulcast stops working. Both +// paths are therefore asserted end to end, by retransmitting a sequence number that is +// never sent on the primary stream and requiring it to surface on the primary buffer. + +import ( + "encoding/binary" + "fmt" + "sync" + "testing" + "time" + + "github.com/pion/rtp" + "github.com/pion/sdp/v3" + "github.com/pion/transport/v4/vnet" + "github.com/pion/webrtc/v4" + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" + "github.com/livekit/livekit-server/pkg/testutils/vnettest" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +// the single video m-line of the publisher's offer +const rtxTestMid = "0" + +const ( + sendsExtensions = false + omitsExtensions = true +) + +// SSRCs are chosen by the test rather than taken from the offer; see stripDeclaredSSRCs. +var rtxTestLayers = []struct { + rid string + ssrc uint32 + rtxSSRC uint32 + recoveredSeq uint16 +}{ + {rid: "q", ssrc: 1001, rtxSSRC: 2001, recoveredSeq: 50001}, + {rid: "h", ssrc: 1002, rtxSSRC: 2002, recoveredSeq: 50002}, + {rid: "f", ssrc: 1003, rtxSSRC: 2003, recoveredSeq: 50003}, +} + +// TestSimulcastRTXPairing covers a publisher sending mid/rid/rsid: the pairing comes +// from probing the packets. +func TestSimulcastRTXPairing(t *testing.T) { + h := newRTXHarness(t, nil) + + h.run(t, sendsExtensions) + + // pairing is also reported through the callback mediatrack subscribes to + require.Equal(t, len(rtxTestLayers), h.tracker.pairCount(), "not all rtx pairs found: %s", h.tracker.describe()) + for _, w := range h.writers { + base, repair := h.tracker.pair(w.rid) + require.Equal(t, w.ssrc, base, "wrong base ssrc paired for rid %q", w.rid) + require.Equal(t, w.rtxSSRC, repair, "wrong repair ssrc paired for rid %q", w.rid) + } +} + +// TestSimulcastRTXPairingAfterMigration covers a migrated publisher: it is mid-stream +// and sends no mid/rid/rsid, so UnhandleSimulcastInterceptor synthesises them for pion +// and the pairing has to come from SimTracks. RepairSSRC names the repair stream. +func TestSimulcastRTXPairingAfterMigration(t *testing.T) { + simTracks := make(map[uint32]sfuinterceptor.SimulcastTrackInfo, 2*len(rtxTestLayers)) + for _, l := range rtxTestLayers { + simTracks[l.ssrc] = sfuinterceptor.SimulcastTrackInfo{ + Mid: rtxTestMid, + StreamID: l.rid, + RepairSSRC: l.rtxSSRC, + } + simTracks[l.rtxSSRC] = sfuinterceptor.SimulcastTrackInfo{ + Mid: rtxTestMid, + StreamID: l.rid, + IsRepairStream: true, + } + } + + newRTXHarness(t, simTracks).run(t, omitsExtensions) +} + +// TestSimulcastRTXPairingAfterMigrationWithoutRepairSSRC covers migration info that +// marks the repair stream but leaves RepairSSRC unset on the primary entry. +func TestSimulcastRTXPairingAfterMigrationWithoutRepairSSRC(t *testing.T) { + simTracks := make(map[uint32]sfuinterceptor.SimulcastTrackInfo, 2*len(rtxTestLayers)) + for _, l := range rtxTestLayers { + simTracks[l.ssrc] = sfuinterceptor.SimulcastTrackInfo{Mid: rtxTestMid, StreamID: l.rid} + simTracks[l.rtxSSRC] = sfuinterceptor.SimulcastTrackInfo{ + Mid: rtxTestMid, + StreamID: l.rid, + IsRepairStream: true, + } + } + + newRTXHarness(t, simTracks).run(t, omitsExtensions) +} + +// ----------------------------------------------------------------------------- +// harness +// ----------------------------------------------------------------------------- + +type rtxHarness struct { + transport *rtc.PCTransport + pubPC *webrtc.PeerConnection + writers []*simulcastWriter + tracker *rtxPairTracker +} + +func newRTXHarness(t *testing.T, simTracks map[uint32]sfuinterceptor.SimulcastTrackInfo) *rtxHarness { + t.Helper() + + hosts := vnettest.NewHosts(t) + tracker := newRTXPairTracker() + + bufferFactory := buffer.NewFactoryOfBufferFactory(500, 200).CreateBufferFactory() + pcTransport := newPublisherTransportForTest(t, hosts.AnswerNet, bufferFactory, simTracks, tracker) + pubPC, writers := newSimulcastPublisherPC(t, hosts.OfferNet) + + return &rtxHarness{ + transport: pcTransport, + pubPC: pubPC, + writers: writers, + tracker: tracker, + } +} + +// run negotiates, publishes every layer, then retransmits a sequence number that is +// never sent on the primary stream and waits for it to surface on the primary buffer. +func (h *rtxHarness) run(t *testing.T, omitExtensions bool) { + t.Helper() + + signalToTransport(t, h.pubPC, h.transport) + require.Equal(t, rtxTestMid, h.pubPC.GetTransceivers()[0].Mid()) + for _, w := range h.writers { + w.mid = rtxTestMid + w.omitExtensions = omitExtensions + } + + // every layer has to bind before RTX is sent, which is also the production ordering: + // a retransmission only follows a NACK for an established layer + require.True( + t, + sendUntil(t, 20*time.Second, func() bool { return h.tracker.boundCount() == len(rtxTestLayers) }, func() { + for _, w := range h.writers { + w.writePrimary(t) + } + }), + "timed out waiting for all simulcast layers to bind: %s", h.tracker.describe(), + ) + + require.True( + t, + sendUntil(t, 20*time.Second, func() bool { + for _, w := range h.writers { + if !h.tracker.sawSeq(w.rid, w.recoveredSeq) { + return false + } + } + return true + }, func() { + for _, w := range h.writers { + w.writeRepair(t, h.tracker.rtxPayloadType(w.rid), w.recoveredSeq) + } + }), + "retransmissions never recovered into the primary buffers: %s", h.tracker.describe(), + ) +} + +// newPublisherTransportForTest builds the production publisher transport on net and +// wires up what ParticipantImpl/MediaTrack do with a published layer. +func newPublisherTransportForTest( + t *testing.T, + net *vnet.Net, + bufferFactory *buffer.Factory, + simTracks map[uint32]sfuinterceptor.SimulcastTrackInfo, + tracker *rtxPairTracker, +) *rtc.PCTransport { + t.Helper() + + rtcConf := newVNetWebRTCConfig(t, net, bufferFactory) + + handler := &transportfakes.FakeHandler{} + params := rtc.TransportParams{ + Handler: handler, + Config: rtcConf, + DirectionConfig: rtcConf.Publisher, + ProtocolVersion: 6, + Logger: logger.GetLogger(), + Transport: livekit.SignalTarget_PUBLISHER, + SimTracks: simTracks, + EnabledPublishCodecs: []*livekit.Codec{ + {Mime: webrtc.MimeTypeVP8}, + {Mime: webrtc.MimeTypeRTX}, + }, + // all candidates are carried in the answer, so the test needs no trickle + UseOneShotSignallingMode: true, + } + + pcTransport, err := rtc.NewPCTransport(params) + require.NoError(t, err) + t.Cleanup(pcTransport.Close) + + // mirror mediatrack.addReceiver: bind the buffer of each published layer, subscribe + // to the pairing notification, and drain the buffer the way WebRTCReceiver does + handler.OnTrackCalls(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { + rid, ssrc := track.RID(), uint32(track.SSRC()) + + buff := bufferFactory.GetBuffer(ssrc) + if buff == nil { + t.Errorf("no buffer for published ssrc %d (rid %q)", ssrc, rid) + return + } + if err := buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, 0); err != nil { + t.Errorf("binding buffer for rid %q failed: %v", rid, err) + return + } + buff.OnNotifyRTX(func(base, repair uint32, rsid string) { + tracker.pairFound(rsid, base, repair) + }) + + // mirror ParticipantImpl.onMediaTrack + pcTransport.RTPStreamPublished(ssrc, pcTransport.GetMid(receiver), rid) + + tracker.layerBound(rid, ssrc, buff, receiver.GetParameters()) + go tracker.drain(rid, buff) + }) + + return pcTransport +} + +// signalToTransport runs a one-shot offer/answer against the transport and waits for +// the publisher to connect. +func signalToTransport(t *testing.T, pub *webrtc.PeerConnection, pcTransport *rtc.PCTransport) { + t.Helper() + + connected := vnettest.UntilConnected(pub) + + offer := vnettest.GatheredOffer(t, pub) + offer.SDP = stripDeclaredSSRCs(offer.SDP) + require.NoError(t, pcTransport.HandleRemoteDescription(offer, 1)) + + answer, _, err := pcTransport.GetAnswer() + require.NoError(t, err) + require.NoError(t, pub.SetRemoteDescription(answer)) + + select { + case <-connected: + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for the publisher to connect") + } +} + +// ----------------------------------------------------------------------------- +// publisher: raw simulcast writer with a per-layer repair stream +// ----------------------------------------------------------------------------- + +// rawTrackLocal hands the test the negotiated write stream directly. Unlike +// TrackLocalStaticRTP it does not rewrite SSRC or payload type, which is what lets a +// repair stream be emitted on its own SSRC alongside the primary stream of the same rid. +type rawTrackLocal struct { + id string + streamID string + rid string + + lock sync.Mutex + writers []webrtc.TrackLocalWriter + exts []webrtc.RTPHeaderExtensionParameter +} + +func (t *rawTrackLocal) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { + for _, c := range ctx.CodecParameters() { + if c.PayloadType != vnettest.VP8PayloadType { + continue + } + + t.lock.Lock() + t.writers = append(t.writers, ctx.WriteStream()) + t.exts = ctx.HeaderExtensions() + t.lock.Unlock() + return c, nil + } + return webrtc.RTPCodecParameters{}, fmt.Errorf("vp8 not negotiated for rid %q", t.rid) +} + +func (t *rawTrackLocal) Unbind(webrtc.TrackLocalContext) error { return nil } +func (t *rawTrackLocal) ID() string { return t.id } +func (t *rawTrackLocal) RID() string { return t.rid } +func (t *rawTrackLocal) StreamID() string { return t.streamID } +func (t *rawTrackLocal) Kind() webrtc.RTPCodecType { return webrtc.RTPCodecTypeVideo } + +func (t *rawTrackLocal) extensionID(uri string) uint8 { + t.lock.Lock() + defer t.lock.Unlock() + + for _, e := range t.exts { + if e.URI == uri { + return uint8(e.ID) + } + } + return 0 +} + +func (t *rawTrackLocal) write(header *rtp.Header, payload []byte) { + t.lock.Lock() + writers := append([]webrtc.TrackLocalWriter(nil), t.writers...) + t.lock.Unlock() + + for _, w := range writers { + _, _ = w.WriteRTP(header, payload) + } +} + +// simulcastWriter emits the primary and repair streams of one simulcast layer. +type simulcastWriter struct { + track *rawTrackLocal + mid string + rid string + ssrc uint32 + rtxSSRC uint32 + + // omitExtensions emulates a migrated publisher, which sends no mid/rid/rsid + omitExtensions bool + + // recoveredSeq is only ever sent inside an RTX payload, never on the primary + // stream, so its arrival on the primary buffer proves RTX recovery worked + recoveredSeq uint16 + + lock sync.Mutex + seq uint16 +} + +func (w *simulcastWriter) nextSeq() uint16 { + w.lock.Lock() + defer w.lock.Unlock() + + w.seq++ + return w.seq +} + +func (w *simulcastWriter) header(t *testing.T, ssrc uint32, pt uint8, seq uint16, rid, rsid string) *rtp.Header { + t.Helper() + + h := &rtp.Header{ + Version: 2, + PayloadType: pt, + SequenceNumber: seq, + Timestamp: uint32(seq) * 3000, + SSRC: ssrc, + } + if w.omitExtensions { + return h + } + + midID := w.track.extensionID(sdp.SDESMidURI) + require.NotZero(t, midID, "sdes:mid not negotiated") + require.NoError(t, h.SetExtension(midID, []byte(w.mid))) + + if rid != "" { + ridID := w.track.extensionID(sdp.SDESRTPStreamIDURI) + require.NotZero(t, ridID, "sdes:rtp-stream-id not negotiated") + require.NoError(t, h.SetExtension(ridID, []byte(rid))) + } + if rsid != "" { + rsidID := w.track.extensionID(sdp.SDESRepairRTPStreamIDURI) + require.NotZero(t, rsidID, "sdes:repaired-rtp-stream-id not negotiated") + require.NoError(t, h.SetExtension(rsidID, []byte(rsid))) + } + return h +} + +func (w *simulcastWriter) writePrimary(t *testing.T) { + t.Helper() + + w.track.write(w.header(t, w.ssrc, vnettest.VP8PayloadType, w.nextSeq(), w.rid, ""), vp8TestPayload()) +} + +// writeRepair emits an RFC 4588 repair packet: the sequence number being retransmitted +// is prepended to the payload. +func (w *simulcastWriter) writeRepair(t *testing.T, rtxPT uint8, originalSeq uint16) { + t.Helper() + + if rtxPT == 0 { + rtxPT = vnettest.RTXPayloadType + } + + inner := vp8TestPayload() + payload := make([]byte, 2+len(inner)) + binary.BigEndian.PutUint16(payload[:2], originalSeq) + copy(payload[2:], inner) + + w.track.write(w.header(t, w.rtxSSRC, rtxPT, w.nextSeq(), "", w.rid), payload) +} + +func vp8TestPayload() []byte { + return []byte{0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00} +} + +// newSimulcastPublisherPC builds the publishing peer connection and one writer per +// simulcast layer. +func newSimulcastPublisherPC(t *testing.T, net *vnet.Net) (*webrtc.PeerConnection, []*simulcastWriter) { + t.Helper() + + pc := vnettest.NewPeerConnection(t, vnettest.PCConfig{ + Net: net, + MediaEngine: vnettest.MediaEngineConfig{ + Video: true, + HeaderExtensions: true, + SimulcastExtensions: true, + }, + }) + + writers := make([]*simulcastWriter, 0, len(rtxTestLayers)) + for _, l := range rtxTestLayers { + writers = append(writers, &simulcastWriter{ + track: &rawTrackLocal{id: "video", streamID: "pion", rid: l.rid}, + rid: l.rid, + ssrc: l.ssrc, + rtxSSRC: l.rtxSSRC, + recoveredSeq: l.recoveredSeq, + }) + } + + sender, err := pc.AddTrack(writers[0].track) + require.NoError(t, err) + for _, w := range writers[1:] { + require.NoError(t, sender.AddEncoding(w.track)) + } + + return pc, writers +} + +// ----------------------------------------------------------------------------- +// tracking +// ----------------------------------------------------------------------------- + +type rtxPairTracker struct { + lock sync.Mutex + + bound map[string]uint32 // rid -> base ssrc + params map[string]webrtc.RTPParameters + pairs map[string][2]uint32 // rsid -> {base ssrc, repair ssrc} + seen map[string]map[uint16]bool // rid -> sequence numbers read off the primary buffer +} + +func newRTXPairTracker() *rtxPairTracker { + return &rtxPairTracker{ + bound: make(map[string]uint32), + params: make(map[string]webrtc.RTPParameters), + pairs: make(map[string][2]uint32), + seen: make(map[string]map[uint16]bool), + } +} + +func (t *rtxPairTracker) layerBound(rid string, ssrc uint32, buff *buffer.Buffer, params webrtc.RTPParameters) { + t.lock.Lock() + defer t.lock.Unlock() + + t.bound[rid] = ssrc + t.params[rid] = params + t.seen[rid] = make(map[uint16]bool) +} + +func (t *rtxPairTracker) pairFound(rsid string, base, repair uint32) { + t.lock.Lock() + defer t.lock.Unlock() + + t.pairs[rsid] = [2]uint32{base, repair} +} + +func (t *rtxPairTracker) boundCount() int { + t.lock.Lock() + defer t.lock.Unlock() + + return len(t.bound) +} + +func (t *rtxPairTracker) pairCount() int { + t.lock.Lock() + defer t.lock.Unlock() + + return len(t.pairs) +} + +func (t *rtxPairTracker) pair(rid string) (uint32, uint32) { + t.lock.Lock() + defer t.lock.Unlock() + + p := t.pairs[rid] + return p[0], p[1] +} + +func (t *rtxPairTracker) rtxPayloadType(rid string) uint8 { + t.lock.Lock() + defer t.lock.Unlock() + + for _, c := range t.params[rid].Codecs { + if c.MimeType == webrtc.MimeTypeRTX { + return uint8(c.PayloadType) + } + } + return 0 +} + +func (t *rtxPairTracker) sawSeq(rid string, seq uint16) bool { + t.lock.Lock() + defer t.lock.Unlock() + + return t.seen[rid][seq] +} + +// drain consumes the primary buffer the way WebRTCReceiver does, recording which +// sequence numbers made it through. +func (t *rtxPairTracker) drain(rid string, buff *buffer.Buffer) { + b := make([]byte, 1500) + for { + ep, err := buff.ReadExtended(b) + if err != nil { + return + } + if ep == nil || ep.Packet == nil { + continue + } + + t.lock.Lock() + t.seen[rid][ep.Packet.SequenceNumber] = true + t.lock.Unlock() + } +} + +func (t *rtxPairTracker) describe() string { + t.lock.Lock() + defer t.lock.Unlock() + + return fmt.Sprintf("bound=%v pairs=%v", t.bound, t.pairs) +} diff --git a/test/webhook_test.go b/test/webhook_test.go index 739c47179..7fff2d226 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -111,7 +111,7 @@ func TestWebhooks(t *testing.T) { // room closed rm := server.RoomManager().GetRoom(context.Background(), testRoom) - rm.Close(types.ParticipantCloseReasonNone) + rm.Close(types.RoomCloseReasonAPIDelete) testutils.WithTimeout(t, func() string { if ts.GetEvent(webhook.EventRoomFinished) == nil { return "did not receive RoomFinished" @@ -119,6 +119,7 @@ func TestWebhooks(t *testing.T) { return "" }) require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name) + require.Equal(t, livekit.RoomEndReason_ROOM_END_API_DELETE, ts.GetEvent(webhook.EventRoomFinished).RoomEndReason) }) } } diff --git a/version/version.go b/version/version.go index 487f7410a..29d306b8c 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.13.5" +const Version = "1.13.6"