Merge remote-tracking branch 'origin/master' into raja_1833

CHANGELOG

@@ -2,6 +2,44 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.5.3] - 2024-02-17
+
+### Added
+- Added dynamic playout delay if PlayoutDelay enabled in the room (#2403)
+- Allow creating SRT URL pull ingress (requires Ingress service release) (#2416)
+- Use default max playout delay as chrome (#2411)
+- RTX support on publisher transport (#2452)
+- Add exponential backoff to room service check retries (#2462)
+- Add support for ingress ParticipantMetadata (#2461)
+
+### Fixed
+- Prevent race of new track and new receiver. (#2345)
+- Fixed race condition when applying metadata update. (#2363 #2478)
+- Fixed race condition in DownTrack.Bind. (#2388)
+- Improved PSRPC over redis reliability with keepalive (#2398)
+- Fix race condition on Participant.updateState (#2401)
+- Replace /bin/bash with env call for FreeBSD compatibility (#2409)
+- Fix startup with -dev and -config (#2442)
+- Fix published track leaks: close published tracks on participant close (#2446)
+- Enforce empty SID for UserPacket from hidden participants (#2469)
+- Ignore duplicate RID. (Fix for spec breakage by Firefox on Windows 10) (#2471)
+
+### Changed
+- Logging improvements (various PRs)
+- Server shuts down after a second SIGINT to simplify development lifecycle (#2364)
+- A/V sync improvements (#2369 #2437 #2472)
+- Prometheus: larger max session start time bin size (#2380)
+- Updated SIP protocol for creating participants. (requires latest SIP release) (#2404 #2474)
+- Improved reliability of signal stream starts with retries (#2414)
+- Use Deque instead of channels in internal communications to reduce memory usage. (#2418 #2419)
+- Do not synthesise DISCONNECT on session change. (#2412)
+- Prometheus: larger buckets for jitter histogram (#2468)
+- Support for improved Ingress internal RPC (#2485)
+- Let track events go through after participant close. (#2487)
+
+### Removed
+- Removed code related to legacy (pre 1.5.x) RPC protocol (#2384 #2385)
+
 ## [1.5.2] - 2023-12-21
 
 Support for LiveKit SIP Bridge

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.21-alpine as builder
+FROM golang:1.22-alpine as builder
 
 ARG TARGETPLATFORM
 ARG TARGETARCH

README.md

@@ -191,8 +191,11 @@ enabling you to build automations that behave like end-users.
 
 ## Install
 
-We recommend installing [livekit-cli](https://github.com/livekit/livekit-cli) along with the server. It lets you access
-server APIs, create tokens, and generate test traffic.
+> [!TIP]
+> We recommend installing [LiveKit CLI](https://github.com/livekit/livekit-cli) along with the server. It lets you access
+> server APIs, create tokens, and generate test traffic.
+
+The following will install LiveKit's media server:
 
 ### MacOS
 

cmd/server/main.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/livekit/livekit-server/pkg/rtc"
 	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
-	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
 	"github.com/livekit/protocol/logger"
 
 	"github.com/livekit/livekit-server/pkg/config"
@@ -205,11 +204,7 @@ func getConfig(c *cli.Context) (*config.Config, error) {
 	}
 	config.InitLoggerFromConfig(&conf.Logging)
 
-	if c.String("config") == "" && c.String("config-body") == "" && conf.Development {
-		// use single port UDP when no config is provided
-		conf.RTC.UDPPort = rtcconfig.PortRange{Start: 7882}
-		conf.RTC.ICEPortRangeStart = 0
-		conf.RTC.ICEPortRangeEnd = 0
+	if conf.Development {
 		logger.Infow("starting in development mode")
 
 		if len(conf.Keys) == 0 {
@@ -248,10 +243,6 @@ func getConfig(c *cli.Context) (*config.Config, error) {
 }
 
 func startServer(c *cli.Context) error {
-	rand.Seed(time.Now().UnixNano())
-
-	memProfile := c.String("memprofile")
-
 	conf, err := getConfig(c)
 	if err != nil {
 		return err
@@ -263,7 +254,7 @@ func startServer(c *cli.Context) error {
 		return err
 	}
 
-	if memProfile != "" {
+	if memProfile := c.String("memprofile"); memProfile != "" {
 		if f, err := os.Create(memProfile); err != nil {
 			return err
 		} else {

config-sample.yaml

@@ -1,3 +1,17 @@
+# Copyright 2024 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # main TCP port for RoomService and RTC endpoint
 # for production setups, this port should be placed behind a load balancer with TLS
 port: 7880
@@ -6,8 +20,6 @@ port: 7880
 # clients could connect to any node and be routed to the same room
 redis:
   address: redis.host:6379
-  # To require TLS transport
-  # use_tls: true
   # db: 0
   # username: myuser
   # password: mypassword
@@ -20,6 +32,17 @@ redis:
   # sentinel_username: user
   # sentinel_password: pass
   #
+  # To use TLS with redis
+  # tls:
+  #   enabled: true
+  #   # when set to true, LiveKit will not verify the server's certificate, defaults to true
+  #   insecure: false
+  #   server_name: myserver.com
+  #   # file containing trusted root certificates for verification
+  #   ca_cert_file: /path/to/ca.crt
+  #   client_cert_file: /path/to/client.crt
+  #   client_key_file: /path/to/client.key
+  #
   # To use cluster remove the address key above and add the following
   # cluster_addresses:
   # - livekit-redis-node-0.livekit-redis-headless:6379

go.mod

@@ -3,6 +3,7 @@ module github.com/livekit/livekit-server
 go 1.20
 
 require (
+	github.com/avast/retry-go/v4 v4.5.1
 	github.com/bep/debounce v1.2.1
 	github.com/d5/tengo/v2 v2.16.1
 	github.com/dustin/go-humanize v1.0.1
@@ -11,30 +12,30 @@ require (
 	github.com/frostbyte73/core v0.0.9
 	github.com/gammazero/deque v0.2.1
 	github.com/gammazero/workerpool v1.1.3
-	github.com/google/wire v0.5.0
+	github.com/google/wire v0.6.0
 	github.com/gorilla/websocket v1.5.1
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jxskiss/base62 v1.1.0
 	github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1
-	github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f
-	github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e
-	github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289
+	github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76
+	github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582
+	github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58
 	github.com/mackerelio/go-osstat v0.2.4
 	github.com/magefile/mage v1.15.0
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pion/dtls/v2 v2.2.9
-	github.com/pion/ice/v2 v2.3.12
+	github.com/pion/dtls/v2 v2.2.10
+	github.com/pion/ice/v2 v2.3.13
 	github.com/pion/interceptor v0.1.25
 	github.com/pion/rtcp v1.2.13
 	github.com/pion/rtp v1.8.3
-	github.com/pion/sctp v1.8.9
+	github.com/pion/sctp v1.8.12
 	github.com/pion/sdp/v3 v3.0.6
 	github.com/pion/transport/v2 v2.2.4
-	github.com/pion/turn/v2 v2.1.4
-	github.com/pion/webrtc/v3 v3.2.24
+	github.com/pion/turn/v2 v2.1.5
+	github.com/pion/webrtc/v3 v3.2.28
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.18.0
 	github.com/redis/go-redis/v9 v9.4.0
@@ -46,7 +47,7 @@ require (
 	github.com/urfave/cli/v2 v2.27.1
 	github.com/urfave/negroni/v3 v3.0.0
 	go.uber.org/atomic v1.11.0
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
+	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/sync v0.6.0
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -65,12 +66,12 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/subcommands v1.2.0 // indirect
-	github.com/google/uuid v1.5.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/josharian/native v1.1.0 // indirect
-	github.com/klauspost/compress v1.17.5 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/lithammer/shortuuid/v4 v4.0.0 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
@@ -82,7 +83,7 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pion/datachannel v1.5.5 // indirect
 	github.com/pion/logging v0.2.2 // indirect
-	github.com/pion/mdns v0.0.9 // indirect
+	github.com/pion/mdns v0.0.12 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/srtp/v2 v2.0.18 // indirect
 	github.com/pion/stun v0.6.1 // indirect
@@ -95,13 +96,13 @@ require (
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
-	google.golang.org/grpc v1.61.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect
+	google.golang.org/grpc v1.61.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
+github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
@@ -68,15 +70,14 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
-github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
-github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI=
+github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -108,8 +109,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79
 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
 github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw=
 github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc=
-github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E=
-github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -124,12 +125,12 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw
 github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58=
 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
-github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg=
-github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc=
-github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e h1:aQGmivssjKlvca8Mmmm4XzHQrFV+vwrGIhAxQE8tnZk=
-github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e/go.mod h1:lSJlMeTJfQBEv8/D2p3zdCo+i+jTmTtn24ysL4ePK28=
-github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289 h1:oTgNH7v9TXsBgoltKk5mnWjv4qqcPF2iV+WtEVQ6ROM=
-github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g=
+github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76 h1:Zw88krOHni51OzDUlrduYb3m7VcsaKw06TnnDhsQpjg=
+github.com/livekit/mediatransportutil v0.0.0-20240206082112-9bf41dcbce76/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc=
+github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582 h1:hSOSQs2pKF6TD9CEwu7+LatqfvF/LiyIbeCoUPCGRho=
+github.com/livekit/protocol v1.9.10-0.20240217202122-51aba73c0582/go.mod h1:/kviHT6yTNqHdZ9QsvRuxAHf7LaBROa7qe5naT1oVrU=
+github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58 h1:yH55rBGLRO+ict2mu6bKZ5iPwTIrIwU1i0ydgThi4+k=
+github.com/livekit/psrpc v0.5.3-0.20240209001357-380f59f00c58/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g=
 github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs=
 github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
@@ -182,18 +183,16 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8=
 github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0=
 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
-github.com/pion/dtls/v2 v2.2.9 h1:K+D/aVf9/REahQvqk6G5JavdrD8W1PWDKC11UlwN7ts=
-github.com/pion/dtls/v2 v2.2.9/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
-github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E=
-github.com/pion/ice/v2 v2.3.12 h1:NWKW2b3+oSZS3klbQMIEWQ0i52Kuo0KBg505a5kQv4s=
-github.com/pion/ice/v2 v2.3.12/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E=
+github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA=
+github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
+github.com/pion/ice/v2 v2.3.13 h1:xOxP+4V9nSDlUaGFRf/LvAuGHDXRcjIdsbbXPK/w7c8=
+github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw=
 github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc=
 github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
 github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
-github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI=
-github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4=
-github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc=
+github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
+github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I=
@@ -204,9 +203,8 @@ github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU
 github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8=
 github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
 github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0=
-github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs=
-github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g=
-github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI=
+github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY=
+github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI=
 github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw=
 github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw=
 github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo=
@@ -223,10 +221,10 @@ github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLh
 github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM=
 github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8=
-github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/webrtc/v3 v3.2.24 h1:MiFL5DMo2bDaaIFWr0DDpwiV/L4EGbLZb+xoRvfEo1Y=
-github.com/pion/webrtc/v3 v3.2.24/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs=
+github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4=
+github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
+github.com/pion/webrtc/v3 v3.2.28 h1:ienStxZ6HcjtH2UlmnFpMM0loENiYjaX437uIUpQSKo=
+github.com/pion/webrtc/v3 v3.2.28/go.mod h1:PNRCEuQlibrmuBhOTnol9j6KkIbUG11aHLEfNpUYey0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -292,19 +290,21 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE=
+golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -329,18 +329,19 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -384,18 +385,19 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -403,28 +405,28 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s=
-google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
-google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 h1:FSL3lRCkhaPFxqi0s9o+V4UI2WTzAVOvkgbd4kVV4Wg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4=
+google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY=
+google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

pkg/config/config.go

@@ -33,8 +33,10 @@ import (
 	"github.com/livekit/protocol/rpc"
 )
 
-type CongestionControlProbeMode string
-type StreamTrackerType string
+type (
+	CongestionControlProbeMode string
+	StreamTrackerType          string
+)
 
 const (
 	generatedCLIFlagUsage = "generated"
@@ -294,22 +296,24 @@ type IngressConfig struct {
 	WHIPBaseURL string `yaml:"whip_base_url,omitempty"`
 }
 
-type SIPConfig struct {
-}
+type SIPConfig struct{}
 
-// not exposed to YAML
 type APIConfig struct {
 	// amount of time to wait for API to execute, default 2s
-	ExecutionTimeout time.Duration
+	ExecutionTimeout time.Duration `yaml:"execution_timeout,omitempty"`
 
-	// amount of time to wait before checking for operation complete
-	CheckInterval time.Duration
+	// min amount of time to wait before checking for operation complete
+	CheckInterval time.Duration `yaml:"check_interval,omitempty"`
+
+	// max amount of time to wait before checking for operation complete
+	MaxCheckInterval time.Duration `yaml:"max_check_interval,omitempty"`
 }
 
 func DefaultAPIConfig() APIConfig {
 	return APIConfig{
 		ExecutionTimeout: 2 * time.Second,
 		CheckInterval:    100 * time.Millisecond,
+		MaxCheckInterval: 300 * time.Second,
 	}
 }
 
@@ -516,16 +520,16 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c
 		}
 	}
 
-	if err := conf.RTC.Validate(conf.Development); err != nil {
-		return nil, fmt.Errorf("could not validate RTC config: %v", err)
-	}
-
 	if c != nil {
 		if err := conf.updateFromCLI(c, baseFlags); err != nil {
 			return nil, err
 		}
 	}
 
+	if err := conf.RTC.Validate(conf.Development); err != nil {
+		return nil, fmt.Errorf("could not validate RTC config: %v", err)
+	}
+
 	// expand env vars in filenames
 	file, err := homedir.Expand(os.ExpandEnv(conf.KeyFile))
 	if err != nil {
@@ -636,10 +640,10 @@ func (conf *Config) ToCLIFlagNames(existingFlags []cli.Flag) map[string]reflect.
 func (conf *Config) ValidateKeys() error {
 	// prefer keyfile if set
 	if conf.KeyFile != "" {
-		var otherFilter os.FileMode = 0007
+		var otherFilter os.FileMode = 0o007
 		if st, err := os.Stat(conf.KeyFile); err != nil {
 			return err
-		} else if st.Mode().Perm()&otherFilter != 0000 {
+		} else if st.Mode().Perm()&otherFilter != 0o000 {
 			return ErrKeyFileIncorrectPermission
 		}
 		f, err := os.Open(conf.KeyFile)
@@ -744,6 +748,9 @@ func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error)
 		case reflect.Map:
 			// TODO
 			continue
+		case reflect.Struct:
+			// TODO
+			continue
 		default:
 			return flags, fmt.Errorf("cli flag generation unsupported for config type: %s is a %s", name, kind.String())
 		}

pkg/rtc/config.go

@@ -25,7 +25,8 @@ import (
 )
 
 const (
-	frameMarking = "urn:ietf:params:rtp-hdrext:framemarking"
+	frameMarking        = "urn:ietf:params:rtp-hdrext:framemarking"
+	repairedRTPStreamID = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
 )
 
 type WebRTCConfig struct {
@@ -87,6 +88,7 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) {
 				sdp.TransportCCURI,
 				frameMarking,
 				dd.ExtensionURI,
+				repairedRTPStreamID,
 			},
 		},
 		RTCPFeedback: RTCPFeedbackConfig{

pkg/rtc/dynacastmanager.go

@@ -59,7 +59,7 @@ func NewDynacastManager(params DynacastManagerParams) *DynacastManager {
 		maxSubscribedQuality:          make(map[string]livekit.VideoQuality),
 		committedMaxSubscribedQuality: make(map[string]livekit.VideoQuality),
 		maxSubscribedQualityDebounce:  debounce.New(params.DynacastPauseDelay),
-		qualityNotifyOpQueue:          utils.NewOpsQueue("quality-notify", 0, true),
+		qualityNotifyOpQueue:          utils.NewOpsQueue("quality-notify", 64, true),
 	}
 	d.qualityNotifyOpQueue.Start()
 	return d

pkg/rtc/mediaengine.go

@@ -15,6 +15,7 @@
 package rtc
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/pion/webrtc/v3"
@@ -24,8 +25,13 @@ import (
 	"github.com/livekit/protocol/livekit"
 )
 
+const (
+	videoRTXMimeType = "video/rtx"
+)
+
 var opusCodecCapability = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"}
 var redCodecCapability = webrtc.RTPCodecCapability{MimeType: sfu.MimeTypeAudioRed, ClockRate: 48000, Channels: 2, SDPFmtpLine: "111/111"}
+var videoRTX = webrtc.RTPCodecCapability{MimeType: videoRTXMimeType, ClockRate: 90000}
 
 func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error {
 	opusCodec := opusCodecCapability
@@ -50,6 +56,8 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 		}
 	}
 
+	rtxEnabled := IsCodecEnabled(codecs, videoRTX)
+
 	h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032"
 	for _, codec := range []webrtc.RTPCodecParameters{
 		{
@@ -84,10 +92,25 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 		if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp {
 			continue
 		}
+		if codec.MimeType == videoRTXMimeType {
+			continue
+		}
 		if IsCodecEnabled(codecs, codec.RTPCodecCapability) {
 			if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
 				return err
 			}
+			if rtxEnabled {
+				if err := me.RegisterCodec(webrtc.RTPCodecParameters{
+					RTPCodecCapability: webrtc.RTPCodecCapability{
+						MimeType:    videoRTXMimeType,
+						ClockRate:   90000,
+						SDPFmtpLine: fmt.Sprintf("apt=%d", codec.PayloadType),
+					},
+					PayloadType: codec.PayloadType + 1,
+				}, webrtc.RTPCodecTypeVideo); err != nil {
+					return err
+				}
+			}
 		}
 	}
 	return nil

pkg/rtc/mediatrack.go

@@ -23,7 +23,6 @@ import (
 	"github.com/pion/webrtc/v3"
 	"go.uber.org/atomic"
 
-	"github.com/livekit/mediatransportutil/pkg/twcc"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 
@@ -56,17 +55,16 @@ type MediaTrackParams struct {
 	ParticipantID       livekit.ParticipantID
 	ParticipantIdentity livekit.ParticipantIdentity
 	ParticipantVersion  uint32
-	// channel to send RTCP packets to the source
-	RTCPChan          chan []rtcp.Packet
-	BufferFactory     *buffer.Factory
-	ReceiverConfig    ReceiverConfig
-	SubscriberConfig  DirectionConfig
-	PLIThrottleConfig config.PLIThrottleConfig
-	AudioConfig       config.AudioConfig
-	VideoConfig       config.VideoConfig
-	Telemetry         telemetry.TelemetryService
-	Logger            logger.Logger
-	SimTracks         map[uint32]SimulcastTrackInfo
+	BufferFactory       *buffer.Factory
+	ReceiverConfig      ReceiverConfig
+	SubscriberConfig    DirectionConfig
+	PLIThrottleConfig   config.PLIThrottleConfig
+	AudioConfig         config.AudioConfig
+	VideoConfig         config.VideoConfig
+	Telemetry           telemetry.TelemetryService
+	Logger              logger.Logger
+	SimTracks           map[uint32]SimulcastTrackInfo
+	OnRTCP              func([]rtcp.Packet)
 }
 
 func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack {
@@ -183,7 +181,7 @@ func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) {
 }
 
 // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec
-func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, twcc *twcc.Responder, mid string) bool {
+func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, mid string) bool {
 	var newCodec bool
 	buff, rtcpReader := t.params.BufferFactory.GetBufferPair(uint32(track.SSRC()))
 	if buff == nil || rtcpReader == nil {
@@ -251,18 +249,19 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra
 			track,
 			ti,
 			LoggerWithCodecMime(t.params.Logger, mime),
-			twcc,
+			t.params.OnRTCP,
 			t.params.VideoConfig.StreamTracker,
 			sfu.WithPliThrottleConfig(t.params.PLIThrottleConfig),
 			sfu.WithAudioConfig(t.params.AudioConfig),
 			sfu.WithLoadBalanceThreshold(20),
 			sfu.WithStreamTrackers(),
 		)
-		newWR.SetRTCPCh(t.params.RTCPChan)
 		newWR.OnCloseHandler(func() {
+			t.params.Logger.Infow("webrtc receiver closed")
 			t.MediaTrackReceiver.SetClosing()
 			t.MediaTrackReceiver.ClearReceiver(mime, false)
 			if t.MediaTrackReceiver.TryClose() {
+				t.params.Logger.Infow("mediaTrack closed")
 				if t.dynacastManager != nil {
 					t.dynacastManager.Close()
 				}
@@ -310,7 +309,17 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra
 	}
 	t.lock.Unlock()
 
-	wr.(*sfu.WebRTCReceiver).AddUpTrack(track, buff)
+	if err := wr.(*sfu.WebRTCReceiver).AddUpTrack(track, buff); err != nil {
+		t.params.Logger.Warnw(
+			"adding up track failed", err,
+			"rid", track.RID(),
+			"layer", layer,
+			"ssrc", track.SSRC(),
+			"newCodec", newCodec,
+		)
+		buff.Close()
+		return false
+	}
 
 	// LK-TODO: can remove this completely when VideoLayers protocol becomes the default as it has info from client or if we decide to use TrackInfo.Simulcast
 	if t.numUpTracks.Inc() > 1 || track.RID() != "" {

pkg/rtc/mediatracksubscriptions.go

@@ -404,10 +404,7 @@ func (t *MediaTrackSubscriptions) DebugInfo() []map[string]interface{} {
 	subscribedTrackInfo := make([]map[string]interface{}, 0)
 	for _, val := range t.getAllSubscribedTracks() {
 		if st, ok := val.(*SubscribedTrack); ok {
-			dt := st.DownTrack().DebugInfo()
-			dt["PubMuted"] = st.pubMuted.Load()
-			dt["SubMuted"] = st.subMuted.Load()
-			subscribedTrackInfo = append(subscribedTrackInfo, dt)
+			subscribedTrackInfo = append(subscribedTrackInfo, st.DownTrack().DebugInfo())
 		}
 	}
 

pkg/rtc/participant.go

@@ -137,6 +137,10 @@ type ParticipantParams struct {
 }
 
 type ParticipantImpl struct {
+	// utils.TimedVersion is a atomic. To be correctly aligned also on 32bit archs
+	// 64it atomics need to be at the front of a struct
+	timedVersion utils.TimedVersion
+
 	params ParticipantParams
 
 	isClosed    atomic.Bool
@@ -158,7 +162,7 @@ type ParticipantImpl struct {
 	disconnectTimer *time.Timer
 	migrationTimer  *time.Timer
 
-	rtcpCh chan []rtcp.Packet
+	pubRTCPQueue *sutils.OpsQueue
 
 	// hold reference for MediaTrack
 	twcc *twcc.Responder
@@ -198,9 +202,8 @@ type ParticipantImpl struct {
 
 	lock utils.RWMutex
 
-	dirty        atomic.Bool
-	version      atomic.Uint32
-	timedVersion utils.TimedVersion
+	dirty   atomic.Bool
+	version atomic.Uint32
 
 	// callbacks & handlers
 	onTrackPublished     func(types.LocalParticipant, types.MediaTrack)
@@ -240,7 +243,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) {
 	}
 	p := &ParticipantImpl{
 		params:                  params,
-		rtcpCh:                  make(chan []rtcp.Packet, 100),
+		pubRTCPQueue:            sutils.NewOpsQueue("pub-rtcp", 64, false),
 		pendingTracks:           make(map[string]*pendingTrackInfo),
 		pendingPublishingTracks: make(map[livekit.TrackID]*pendingTrackInfo),
 		connectedAt:             time.Now(),
@@ -511,8 +514,37 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.
 		IsPublisher: p.IsPublisher(),
 	}
 	p.lock.RUnlock()
+
+	p.pendingTracksLock.RLock()
 	pi.Tracks = p.UpTrackManager.ToProto()
 
+	// add any pending migrating tracks, else an update could delete/unsubscribe from yet to be published, migrating tracks
+	maybeAdd := func(pti *pendingTrackInfo) {
+		if !pti.migrated {
+			return
+		}
+
+		found := false
+		for _, ti := range pi.Tracks {
+			if ti.Sid == pti.trackInfos[0].Sid {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			pi.Tracks = append(pi.Tracks, pti.trackInfos[0])
+		}
+	}
+
+	for _, pt := range p.pendingTracks {
+		maybeAdd(pt)
+	}
+	for _, ppt := range p.pendingPublishingTracks {
+		maybeAdd(ppt)
+	}
+	p.pendingTracksLock.RUnlock()
+
 	return pi, piv
 }
 
@@ -651,18 +683,11 @@ func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) er
 
 	p.pubLogger.Debugw("sending answer", "transport", livekit.SignalTarget_PUBLISHER)
 	answer = p.configurePublisherAnswer(answer)
-	if err := p.writeMessage(&livekit.SignalResponse{
+	return p.writeMessage(&livekit.SignalResponse{
 		Message: &livekit.SignalResponse_Answer{
 			Answer: ToProtoSessionDescription(answer),
 		},
-	}); err != nil {
-		return err
-	}
-
-	if p.MigrateState() == types.MigrateStateSync {
-		go p.handleMigrateTracks()
-	}
-	return nil
+	})
 }
 
 func (p *ParticipantImpl) handleMigrateTracks() {
@@ -752,6 +777,10 @@ func (p *ParticipantImpl) SetMigrateInfo(
 	}
 	p.pendingTracksLock.Unlock()
 
+	if len(mediaTracks) != 0 {
+		p.setIsPublisher(true)
+	}
+
 	p.TransportManager.SetMigrateInfo(previousOffer, previousAnswer, dataChannels)
 }
 
@@ -902,17 +931,13 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) {
 	p.migrateState.Store(s)
 	p.dirty.Store(true)
 
-	processPendingOffer := false
-	if s == types.MigrateStateSync {
-		processPendingOffer = true
-	}
-
-	if s == types.MigrateStateComplete {
-		p.TransportManager.ProcessPendingPublisherDataChannels()
-	}
-
-	if processPendingOffer {
+	switch s {
+	case types.MigrateStateSync:
 		p.TransportManager.ProcessPendingPublisherOffer()
+
+	case types.MigrateStateComplete:
+		p.handleMigrateTracks()
+		p.TransportManager.ProcessPendingPublisherDataChannels()
 	}
 
 	if onMigrateStateChange := p.getOnMigrateStateChange(); onMigrateStateChange != nil {
@@ -1137,6 +1162,8 @@ func (p *ParticipantImpl) UpdateMediaRTT(rtt uint32) {
 	}
 }
 
+// ----------------------------------------------------------
+
 type AnyTransportHandler struct {
 	transport.UnimplementedHandler
 	p *ParticipantImpl
@@ -1154,6 +1181,8 @@ func (h AnyTransportHandler) OnICECandidate(c *webrtc.ICECandidate, target livek
 	return h.p.onICECandidate(c, target)
 }
 
+// ----------------------------------------------------------
+
 type PublisherTransportHandler struct {
 	AnyTransportHandler
 }
@@ -1174,6 +1203,8 @@ func (h PublisherTransportHandler) OnDataPacket(kind livekit.DataPacket_Kind, da
 	h.p.onDataMessage(kind, data)
 }
 
+// ----------------------------------------------------------
+
 type SubscriberTransportHandler struct {
 	AnyTransportHandler
 }
@@ -1190,6 +1221,8 @@ func (h SubscriberTransportHandler) OnInitialConnected() {
 	h.p.onSubscriberInitialConnected()
 }
 
+// ----------------------------------------------------------
+
 type PrimaryTransportHandler struct {
 	transport.Handler
 	p *ParticipantImpl
@@ -1204,7 +1237,13 @@ func (h PrimaryTransportHandler) OnFullyEstablished() {
 	h.p.onPrimaryTransportFullyEstablished()
 }
 
+// ----------------------------------------------------------
+
 func (p *ParticipantImpl) setupTransportManager() error {
+	p.twcc = twcc.NewTransportWideCCResponder()
+	p.twcc.OnFeedback(func(pkts []rtcp.Packet) {
+		p.postRtcp(pkts)
+	})
 	ath := AnyTransportHandler{p: p}
 	var pth transport.Handler = PublisherTransportHandler{ath}
 	var sth transport.Handler = SubscriberTransportHandler{ath}
@@ -1223,6 +1262,7 @@ func (p *ParticipantImpl) setupTransportManager() error {
 		// after the participant has joined
 		SubscriberAsPrimary:          subscriberAsPrimary,
 		Config:                       p.params.Config,
+		Twcc:                         p.twcc,
 		ProtocolVersion:              p.params.ProtocolVersion,
 		CongestionControlConfig:      p.params.CongestionControlConfig,
 		EnabledPublishCodecs:         p.enabledPublishCodecs,
@@ -1440,8 +1480,13 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt
 		onDataPacket := p.onDataPacket
 		p.lock.RUnlock()
 		if onDataPacket != nil {
-			payload.User.ParticipantSid = string(p.params.SID)
-			payload.User.ParticipantIdentity = string(p.params.Identity)
+			if p.Hidden() {
+				payload.User.ParticipantSid = ""
+				payload.User.ParticipantIdentity = ""
+			} else {
+				payload.User.ParticipantSid = string(p.params.SID)
+				payload.User.ParticipantIdentity = string(p.params.Identity)
+			}
 			onDataPacket(p, &dp)
 		}
 	default:
@@ -1464,10 +1509,13 @@ func (p *ParticipantImpl) onICECandidate(c *webrtc.ICECandidate, target livekit.
 }
 
 func (p *ParticipantImpl) onPublisherInitialConnected() {
+	p.SetMigrateState(types.MigrateStateComplete)
+
 	if p.supervisor != nil {
 		p.supervisor.SetPublisherPeerConnectionConnected(true)
 	}
-	go p.publisherRTCPWorker()
+
+	p.pubRTCPQueue.Start()
 }
 
 func (p *ParticipantImpl) onSubscriberInitialConnected() {
@@ -1477,7 +1525,9 @@ func (p *ParticipantImpl) onSubscriberInitialConnected() {
 }
 
 func (p *ParticipantImpl) onPrimaryTransportInitialConnected() {
-	if !p.hasPendingMigratedTrack() && p.MigrateState() == types.MigrateStateSync {
+	if !p.hasPendingMigratedTrack() && len(p.GetPublishedTracks()) == 0 {
+		// if there are no published tracks, declare migration complete on primary transport initial connect,
+		// else, wait for all tracks to be published and publisher peer connection established
 		p.SetMigrateState(types.MigrateStateComplete)
 	}
 }
@@ -1508,8 +1558,7 @@ func (p *ParticipantImpl) setupDisconnectTimer() {
 		if p.IsClosed() || p.IsDisconnected() {
 			return
 		}
-		reason := types.ParticipantCloseReasonPeerConnectionDisconnected
-		_ = p.Close(true, reason, false)
+		_ = p.Close(true, types.ParticipantCloseReasonPeerConnectionDisconnected, false)
 	})
 	p.lock.Unlock()
 }
@@ -1848,7 +1897,7 @@ func (p *ParticipantImpl) setTrackMuted(trackID livekit.TrackID, muted bool) *li
 	}
 
 	if !isPending && track == nil {
-		p.pubLogger.Warnw("could not locate track", nil, "trackID", trackID)
+		p.pubLogger.Debugw("could not locate track", "trackID", trackID)
 	}
 
 	return trackInfo
@@ -1913,16 +1962,9 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei
 		p.dirty.Store(true)
 	}
 
-	ssrc := uint32(track.SSRC())
-	if p.twcc == nil {
-		p.twcc = twcc.NewTransportWideCCResponder(ssrc)
-		p.twcc.OnFeedback(func(pkts []rtcp.Packet) {
-			p.postRtcp(pkts)
-		})
-	}
 	p.pendingTracksLock.Unlock()
 
-	if mt.AddReceiver(rtpReceiver, track, p.twcc, mid) {
+	if mt.AddReceiver(rtpReceiver, track, mid) {
 		p.removePendingMigratedTrack(mt)
 	}
 
@@ -2000,7 +2042,6 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv
 		ParticipantID:       p.params.SID,
 		ParticipantIdentity: p.params.Identity,
 		ParticipantVersion:  p.version.Load(),
-		RTCPChan:            p.rtcpCh,
 		BufferFactory:       p.params.Config.BufferFactory,
 		ReceiverConfig:      p.params.Config.Receiver,
 		AudioConfig:         p.params.AudioConfig,
@@ -2010,6 +2051,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv
 		SubscriberConfig:    p.params.Config.Subscriber,
 		PLIThrottleConfig:   p.params.PLIThrottleConfig,
 		SimTracks:           p.params.SimTracks,
+		OnRTCP:              p.postRtcp,
 	}, ti)
 
 	mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange)
@@ -2048,7 +2090,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv
 			p.ID(),
 			p.Identity(),
 			mt.ToProto(),
-			!p.IsClosed(),
+			true,
 		)
 
 		// re-use Track sid
@@ -2062,12 +2104,9 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv
 
 		p.dirty.Store(true)
 
-		if !p.IsClosed() {
-			// unpublished events aren't necessary when participant is closed
-			p.pubLogger.Debugw("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti))
-			if onTrackUnpublished := p.getOnTrackUnpublished(); onTrackUnpublished != nil {
-				onTrackUnpublished(p, mt)
-			}
+		p.pubLogger.Infow("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti))
+		if onTrackUnpublished := p.getOnTrackUnpublished(); onTrackUnpublished != nil {
+			onTrackUnpublished(p, mt)
 		}
 	})
 
@@ -2091,10 +2130,6 @@ func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack) {
 	p.pendingTracksLock.Lock()
 	delete(p.pendingPublishingTracks, track.ID())
 	p.pendingTracksLock.Unlock()
-
-	if !p.hasPendingMigratedTrack() {
-		p.SetMigrateState(types.MigrateStateComplete)
-	}
 }
 
 func (p *ParticipantImpl) hasPendingMigratedTrack() bool {
@@ -2117,7 +2152,7 @@ func (p *ParticipantImpl) hasPendingMigratedTrack() bool {
 }
 
 func (p *ParticipantImpl) onUpTrackManagerClose() {
-	p.postRtcp(nil)
+	p.pubRTCPQueue.Stop()
 }
 
 func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackType) (string, *livekit.TrackInfo, bool) {
@@ -2241,28 +2276,6 @@ func (p *ParticipantImpl) getPublishedTrackBySdpCid(clientId string) types.Media
 	return nil
 }
 
-func (p *ParticipantImpl) publisherRTCPWorker() {
-	defer func() {
-		if r := Recover(p.GetLogger()); r != nil {
-			os.Exit(1)
-		}
-	}()
-
-	// read from rtcpChan
-	for pkts := range p.rtcpCh {
-		if pkts == nil {
-			p.pubLogger.Debugw("exiting publisher RTCP worker")
-			return
-		}
-
-		if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil {
-			if !IsEOF(err) {
-				p.pubLogger.Errorw("could not write RTCP to participant", err)
-			}
-		}
-	}
-}
-
 func (p *ParticipantImpl) DebugInfo() map[string]interface{} {
 	info := map[string]interface{}{
 		"ID":    p.params.SID,
@@ -2291,11 +2304,11 @@ func (p *ParticipantImpl) DebugInfo() map[string]interface{} {
 }
 
 func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) {
-	select {
-	case p.rtcpCh <- pkts:
-	default:
-		p.params.Logger.Warnw("rtcp channel full", nil)
-	}
+	p.pubRTCPQueue.Enqueue(func() {
+		if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) {
+			p.pubLogger.Errorw("could not write RTCP to participant", err)
+		}
+	})
 }
 
 func (p *ParticipantImpl) setDowntracksConnected() {
@@ -2399,7 +2412,7 @@ func (p *ParticipantImpl) onAnyTransportNegotiationFailed() {
 func (p *ParticipantImpl) UpdateSubscribedQuality(nodeID livekit.NodeID, trackID livekit.TrackID, maxQualities []types.SubscribedCodecQuality) error {
 	track := p.GetPublishedTrack(trackID)
 	if track == nil {
-		p.pubLogger.Warnw("could not find track", nil, "trackID", trackID)
+		p.pubLogger.Debugw("could not find track", "trackID", trackID)
 		return errors.New("could not find published track")
 	}
 
@@ -2410,7 +2423,7 @@ func (p *ParticipantImpl) UpdateSubscribedQuality(nodeID livekit.NodeID, trackID
 func (p *ParticipantImpl) UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error {
 	track := p.GetPublishedTrack(trackID)
 	if track == nil {
-		p.pubLogger.Warnw("could not find track", nil, "trackID", trackID)
+		p.pubLogger.Debugw("could not find track", "trackID", trackID)
 		return errors.New("could not find published track")
 	}
 

pkg/rtc/participant_signal.go

@@ -100,7 +100,13 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit.
 			// this is a message delivered out of order, a more recent version of the message had already been
 			// sent.
 			if pi.Version < lastVersion.version {
-				p.params.Logger.Debugw("skipping outdated participant update", "otherParticipant", pi.Identity, "otherPID", pi.Sid, "version", pi.Version, "lastVersion", lastVersion)
+				p.params.Logger.Debugw(
+					"skipping outdated participant update",
+					"otherParticipant", pi.Identity,
+					"otherPID", pi.Sid,
+					"version", pi.Version,
+					"lastVersion", lastVersion,
+				)
 				isValid = false
 			}
 		}

pkg/rtc/room.go

@@ -80,6 +80,15 @@ type disconnectSignalOnResumeNoMessages struct {
 }
 
 type Room struct {
+	// atomics always need to be 64bit/8byte aligned
+	// on 32bit arch only the beginning of the struct
+	// starts at such a boundary.
+	// time the first participant joined the room
+	joinedAt atomic.Int64
+	// time that the last participant left the room
+	leftAt atomic.Int64
+	holds    atomic.Int32
+
 	lock sync.RWMutex
 
 	protoRoom  *livekit.Room
@@ -109,11 +118,6 @@ type Room struct {
 	batchedUpdates   map[livekit.ParticipantIdentity]*participantUpdate
 	batchedUpdatesMu sync.Mutex
 
-	// time the first participant joined the room
-	joinedAt atomic.Int64
-	holds    atomic.Int32
-	// time that the last participant left the room
-	leftAt atomic.Int64
 	closed chan struct{}
 
 	trailer []byte
@@ -649,7 +653,7 @@ func (r *Room) UpdateSubscriptions(
 
 func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.SyncState) error {
 	pLogger := participant.GetLogger()
-	pLogger.Infow("setting sync state", "state", state)
+	pLogger.Infow("setting sync state", "state", logger.Proto(state))
 
 	shouldReconnect := false
 	pubTracks := state.GetPublishTracks()
@@ -686,6 +690,13 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync
 		return nil
 	}
 
+	// synthesize a track setting for each disabled track,
+	// can be set before addding subscriptions,
+	// in fact it is done before so that setting can be updated immediately upon subscription.
+	for _, trackSid := range state.TrackSidsDisabled {
+		participant.UpdateSubscribedTrackSettings(livekit.TrackID(trackSid), &livekit.UpdateTrackSettings{Disabled: true})
+	}
+
 	r.UpdateSubscriptions(
 		participant,
 		livekit.StringsAsIDs[livekit.TrackID](state.Subscription.TrackSids),

pkg/rtc/subscribedtrack.go

@@ -21,10 +21,12 @@ import (
 	"github.com/bep/debounce"
 	"github.com/pion/webrtc/v3"
 	"go.uber.org/atomic"
+	"google.golang.org/protobuf/proto"
 
 	sutils "github.com/livekit/livekit-server/pkg/utils"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
 
 	"github.com/livekit/livekit-server/pkg/rtc/types"
 	"github.com/livekit/livekit-server/pkg/sfu"
@@ -47,17 +49,20 @@ type SubscribedTrackParams struct {
 
 type SubscribedTrack struct {
 	params           SubscribedTrackParams
-	subMuted         atomic.Bool
-	pubMuted         atomic.Bool
-	settings         atomic.Pointer[livekit.UpdateTrackSettings]
 	logger           logger.Logger
 	sender           atomic.Pointer[webrtc.RTPSender]
 	needsNegotiation atomic.Bool
 
+	versionGenerator utils.TimedVersionGenerator
+	settingsLock     sync.Mutex
+	settings         *livekit.UpdateTrackSettings
+	settingsVersion  *utils.TimedVersion
+
 	bindLock        sync.Mutex
+	bound           bool
 	onBindCallbacks []func(error)
-	onClose         atomic.Value // func(bool)
-	bound           atomic.Bool
+
+	onClose atomic.Value // func(bool)
 
 	debouncer func(func())
 }
@@ -70,7 +75,8 @@ func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack {
 			"publisherID", params.PublisherID,
 			"publisher", params.PublisherIdentity,
 		),
-		debouncer: debounce.New(subscriptionDebounceInterval),
+		versionGenerator: utils.NewDefaultTimedVersionGenerator(),
+		debouncer:        debounce.New(subscriptionDebounceInterval),
 	}
 
 	return s
@@ -78,7 +84,7 @@ func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack {
 
 func (t *SubscribedTrack) AddOnBind(f func(error)) {
 	t.bindLock.Lock()
-	bound := t.bound.Load()
+	bound := t.bound
 	if !bound {
 		t.onBindCallbacks = append(t.onBindCallbacks, f)
 	}
@@ -94,7 +100,7 @@ func (t *SubscribedTrack) AddOnBind(f func(error)) {
 func (t *SubscribedTrack) Bound(err error) {
 	t.bindLock.Lock()
 	if err == nil {
-		t.bound.Store(true)
+		t.bound = true
 	}
 	callbacks := t.onBindCallbacks
 	t.onBindCallbacks = nil
@@ -110,17 +116,21 @@ func (t *SubscribedTrack) Bound(err error) {
 		//    time of subscription, we might not be able to trigger adaptive stream updates on the client side
 		//    (since there isn't any video frames coming through). this will leave the stream "stuck" on off, without
 		//    a trigger to re-enable it
-		var desiredLayer int32
-		if t.params.AdaptiveStream {
-			desiredLayer = buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_LOW, t.params.MediaTrack.ToProto())
+		t.settingsLock.Lock()
+		if t.settings != nil {
+			if t.params.AdaptiveStream {
+				// remove `disabled` flag to force a visibility update
+				t.settings.Disabled = false
+			}
 		} else {
-			desiredLayer = buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.MediaTrack.ToProto())
+			if t.params.AdaptiveStream {
+				t.settings = &livekit.UpdateTrackSettings{Quality: livekit.VideoQuality_LOW}
+			} else {
+				t.settings = &livekit.UpdateTrackSettings{Quality: livekit.VideoQuality_HIGH}
+			}
 		}
-		settings := t.settings.Load()
-		if settings != nil {
-			desiredLayer = t.spatialLayerFromSettings(settings)
-		}
-		t.DownTrack().SetMaxSpatialLayer(desiredLayer)
+		t.settingsLock.Unlock()
+		t.applySettings()
 	}
 
 	for _, cb := range callbacks {
@@ -140,7 +150,10 @@ func (t *SubscribedTrack) OnClose(f func(bool)) {
 }
 
 func (t *SubscribedTrack) IsBound() bool {
-	return t.bound.Load()
+	t.bindLock.Lock()
+	defer t.bindLock.Unlock()
+
+	return t.bound
 }
 
 func (t *SubscribedTrack) ID() livekit.TrackID {
@@ -181,47 +194,97 @@ func (t *SubscribedTrack) MediaTrack() types.MediaTrack {
 
 // has subscriber indicated it wants to mute this track
 func (t *SubscribedTrack) IsMuted() bool {
-	return t.subMuted.Load()
+	t.settingsLock.Lock()
+	defer t.settingsLock.Unlock()
+
+	return t.isMutedLocked()
+}
+
+func (t *SubscribedTrack) isMutedLocked() bool {
+	if t.settings == nil {
+		return false
+	}
+
+	return t.settings.Disabled
 }
 
 func (t *SubscribedTrack) SetPublisherMuted(muted bool) {
-	t.pubMuted.Store(muted)
-	t.updateDownTrackMute()
+	t.DownTrack().PubMute(muted)
 }
 
-func (t *SubscribedTrack) UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings) {
-	prevDisabled := t.subMuted.Swap(settings.Disabled)
-	t.settings.Store(settings)
-
-	if prevDisabled != settings.Disabled {
-		t.logger.Debugw("updated subscribed track enabled", "enabled", !settings.Disabled)
+func (t *SubscribedTrack) UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings, isImmediate bool) {
+	t.settingsLock.Lock()
+	if proto.Equal(t.settings, settings) {
+		t.settingsLock.Unlock()
+		return
 	}
 
-	// avoid frequent changes to mute & video layers, unless it became visible
-	if prevDisabled != settings.Disabled && !settings.Disabled {
-		t.UpdateVideoLayer()
+	isImmediate = isImmediate || (!settings.Disabled && settings.Disabled != t.isMutedLocked())
+	t.settings = proto.Clone(settings).(*livekit.UpdateTrackSettings)
+	t.settingsLock.Unlock()
+
+	if isImmediate {
+		t.applySettings()
 	} else {
-		t.debouncer(t.UpdateVideoLayer)
+		// avoid frequent changes to mute & video layers, unless it became visible
+		t.debouncer(t.applySettings)
 	}
 }
 
 func (t *SubscribedTrack) UpdateVideoLayer() {
-	t.updateDownTrackMute()
-	if t.DownTrack().Kind() != webrtc.RTPCodecTypeVideo {
+	t.applySettings()
+}
+
+func (t *SubscribedTrack) applySettings() {
+	t.settingsLock.Lock()
+	if t.settings == nil {
+		t.settingsLock.Unlock()
 		return
 	}
 
-	settings := t.settings.Load()
-	if settings == nil || settings.Disabled {
+	t.logger.Debugw("updating subscriber track settings", "settings", logger.Proto(t.settings))
+	t.settingsVersion = t.versionGenerator.New()
+	settingsVersion := t.settingsVersion
+	t.settingsLock.Unlock()
+
+	dt := t.DownTrack()
+	spatial := buffer.InvalidLayerSpatial
+	temporal := buffer.InvalidLayerTemporal
+	if dt.Kind() == webrtc.RTPCodecTypeVideo {
+		mt := t.MediaTrack()
+		quality := t.settings.Quality
+		if t.settings.Width > 0 {
+			quality = mt.GetQualityForDimension(t.settings.Width, t.settings.Height)
+		}
+
+		spatial = buffer.VideoQualityToSpatialLayer(quality, mt.ToProto())
+		if t.settings.Fps > 0 {
+			temporal = mt.GetTemporalLayerForSpatialFps(spatial, t.settings.Fps, dt.Codec().MimeType)
+		}
+	}
+
+	t.settingsLock.Lock()
+	if settingsVersion.Compare(t.settingsVersion) != 0 {
+		// a newer settings has superceded this one
+		t.settingsLock.Unlock()
 		return
 	}
 
-	t.logger.Debugw("updating video layer", "settings", settings)
-	spatial := t.spatialLayerFromSettings(settings)
-	t.DownTrack().SetMaxSpatialLayer(spatial)
-	if settings.Fps > 0 {
-		t.DownTrack().SetMaxTemporalLayer(t.MediaTrack().GetTemporalLayerForSpatialFps(spatial, settings.Fps, t.DownTrack().Codec().MimeType))
+	if t.settings.Disabled {
+		dt.Mute(true)
+		t.settingsLock.Unlock()
+		return
+	} else {
+		dt.Mute(false)
 	}
+
+	if dt.Kind() == webrtc.RTPCodecTypeVideo {
+		dt.SetMaxSpatialLayer(spatial)
+		if temporal != buffer.InvalidLayerTemporal {
+			dt.SetMaxTemporalLayer(temporal)
+		}
+	}
+	t.settingsLock.Unlock()
 }
 
 func (t *SubscribedTrack) NeedsNegotiation() bool {
@@ -239,17 +302,3 @@ func (t *SubscribedTrack) RTPSender() *webrtc.RTPSender {
 func (t *SubscribedTrack) SetRTPSender(sender *webrtc.RTPSender) {
 	t.sender.Store(sender)
 }
-
-func (t *SubscribedTrack) updateDownTrackMute() {
-	t.DownTrack().Mute(t.subMuted.Load())
-	t.DownTrack().PubMute(t.pubMuted.Load())
-}
-
-func (t *SubscribedTrack) spatialLayerFromSettings(settings *livekit.UpdateTrackSettings) int32 {
-	quality := settings.Quality
-	if settings.Width > 0 {
-		quality = t.MediaTrack().GetQualityForDimension(settings.Width, settings.Height)
-	}
-
-	return buffer.VideoQualityToSpatialLayer(quality, t.params.MediaTrack.ToProto())
-}

pkg/rtc/subscriptionmanager.go

@@ -344,7 +344,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) {
 			default:
 				// all other errors
 				if s.durationSinceStart() > subscriptionTimeout {
-					s.logger.Errorw("failed to subscribe, triggering error handler", err,
+					s.logger.Warnw("failed to subscribe, triggering error handler", err,
 						"attempt", numAttempts,
 					)
 					s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, false)
@@ -365,7 +365,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) {
 
 	if s.needsUnsubscribe() {
 		if err := m.unsubscribe(s); err != nil {
-			s.logger.Errorw("failed to unsubscribe", err)
+			s.logger.Warnw("failed to unsubscribe", err)
 		} else {
 			// successfully unsubscribed, remove from map
 			m.lock.Lock()
@@ -383,7 +383,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) {
 		// if a publisher leaves or closes the source track, SubscribedTrack will be closed as well and it will go
 		// back to needsSubscribe state
 		if s.durationSinceStart() > subscriptionTimeout {
-			s.logger.Errorw("track not bound after timeout", nil)
+			s.logger.Warnw("track not bound after timeout", nil)
 			s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), ErrTrackNotBound, false)
 			m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound)
 		}
@@ -662,7 +662,7 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w
 			context.Background(),
 			m.params.Participant.ID(),
 			&livekit.TrackInfo{Sid: string(s.trackID), Type: subTrack.MediaTrack().Kind()},
-			!willBeResumed && !m.params.Participant.IsClosed(),
+			!willBeResumed,
 		)
 
 		dt := subTrack.DownTrack()
@@ -823,8 +823,8 @@ func (s *trackSubscription) setSubscribedTrack(track types.SubscribedTrack) {
 	s.lock.Unlock()
 
 	if settings != nil && track != nil {
-		s.logger.Debugw("restoring subscriber settings", "settings", settings)
-		track.UpdateSubscriberSettings(settings)
+		s.logger.Debugw("restoring subscriber settings", "settings", logger.Proto(settings))
+		track.UpdateSubscriberSettings(settings, true)
 	}
 	if oldTrack != nil {
 		oldTrack.OnClose(nil)
@@ -895,7 +895,7 @@ func (s *trackSubscription) setSettings(settings *livekit.UpdateTrackSettings) {
 	subTrack := s.subscribedTrack
 	s.lock.Unlock()
 	if subTrack != nil {
-		subTrack.UpdateSubscriberSettings(settings)
+		subTrack.UpdateSubscriberSettings(settings, false)
 	}
 }
 

pkg/rtc/subscriptionmanager_test.go

@@ -344,7 +344,7 @@ func TestUpdateSettingsBeforeSubscription(t *testing.T) {
 		return st.UpdateSubscriberSettingsCallCount() == 1
 	}, subSettleTimeout, subCheckInterval, "UpdateSubscriberSettings should be called once")
 
-	applied := st.UpdateSubscriberSettingsArgsForCall(0)
+	applied, _ := st.UpdateSubscriberSettingsArgsForCall(0)
 	require.Equal(t, settings.Disabled, applied.Disabled)
 	require.Equal(t, settings.Width, applied.Width)
 	require.Equal(t, settings.Height, applied.Height)

pkg/rtc/transport.go

@@ -17,6 +17,7 @@ package rtc
 import (
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -37,11 +38,14 @@ import (
 	"github.com/livekit/livekit-server/pkg/config"
 	"github.com/livekit/livekit-server/pkg/rtc/transport"
 	"github.com/livekit/livekit-server/pkg/rtc/types"
+	lkinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor"
 	"github.com/livekit/livekit-server/pkg/sfu/pacer"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpextension"
 	"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
+	sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils"
 	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
 	sutils "github.com/livekit/livekit-server/pkg/utils"
+	lktwcc "github.com/livekit/mediatransportutil/pkg/twcc"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/logger/pionlogger"
@@ -199,6 +203,7 @@ type TransportParams struct {
 	ParticipantIdentity          livekit.ParticipantIdentity
 	ProtocolVersion              types.ProtocolVersion
 	Config                       *WebRTCConfig
+	Twcc                         *lktwcc.Responder
 	DirectionConfig              DirectionConfig
 	CongestionControlConfig      config.CongestionControlConfig
 	EnabledCodecs                []*livekit.Codec
@@ -321,11 +326,45 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat
 	if len(params.SimTracks) > 0 {
 		f, err := NewUnhandleSimulcastInterceptorFactory(UnhandleSimulcastTracks(params.SimTracks))
 		if err != nil {
-			params.Logger.Errorw("NewUnhandleSimulcastInterceptorFactory failed", err)
+			params.Logger.Warnw("NewUnhandleSimulcastInterceptorFactory failed", err)
 		} else {
 			ir.Add(f)
 		}
 	}
+
+	setTWCCForVideo := func(info *interceptor.StreamInfo) {
+		if !strings.HasPrefix(info.MimeType, "video") {
+			return
+		}
+		// rtx stream don't have rtcp feedback, always set twcc for rtx stream
+		twccFb := strings.HasSuffix(info.MimeType, "rtx")
+		if !twccFb {
+			for _, fb := range info.RTCPFeedback {
+				if fb.Type == webrtc.TypeRTCPFBTransportCC {
+					twccFb = true
+					break
+				}
+			}
+		}
+		if !twccFb {
+			return
+		}
+
+		twccExtID := sfuutils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.TransportCCURI})
+		if twccExtID != 0 {
+			if buffer := params.Config.BufferFactory.GetBuffer(info.SSRC); buffer != nil {
+				params.Logger.Debugw("set rtx twcc and ext id", "ssrc", info.SSRC, "twccExtID", twccExtID)
+				buffer.SetTWCCAndExtID(params.Twcc, uint8(twccExtID))
+			} else {
+				params.Logger.Warnw("failed to get buffer for rtx stream", nil, "ssrc", info.SSRC)
+			}
+		}
+	}
+	// put rtx interceptor behind unhandle simulcast interceptor so it can get the correct mid & rid
+	ir.Add(lkinterceptor.NewRTXInfoExtractorFactory(setTWCCForVideo, func(repair, base uint32) {
+		params.Logger.Debugw("rtx pair found from extension", "repair", repair, "base", base)
+		params.Config.BufferFactory.SetRTXPair(repair, base)
+	}, params.Logger))
 	api := webrtc.NewAPI(
 		webrtc.WithMediaEngine(me),
 		webrtc.WithSettingEngine(se),
@@ -565,7 +604,7 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) {
 		if isShort {
 			pair, err := t.getSelectedPair()
 			if err != nil {
-				t.params.Logger.Errorw("short ICE connection", err, "duration", duration)
+				t.params.Logger.Warnw("short ICE connection", err, "duration", duration)
 			} else {
 				t.params.Logger.Infow("short ICE connection", "pair", pair, "duration", duration)
 			}
@@ -593,11 +632,6 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState
 
 	case webrtc.ICEConnectionStateChecking:
 		t.setICEStartedAt(time.Now())
-
-	case webrtc.ICEConnectionStateDisconnected:
-		t.params.Logger.Infow("ice connection state change unexpected", "state", state.String())
-	case webrtc.ICEConnectionStateFailed:
-		t.params.Logger.Debugw("ice connection state change unexpected", "state", state.String())
 	}
 }
 
@@ -613,7 +647,6 @@ func (t *PCTransport) onPeerConnectionStateChange(state webrtc.PeerConnectionSta
 			t.maybeNotifyFullyEstablished()
 		}
 	case webrtc.PeerConnectionStateFailed:
-		t.params.Logger.Infow("peer connection failed")
 		t.clearConnTimer()
 		t.handleConnectionFailed(false)
 	}
@@ -784,7 +817,7 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni
 
 	dcErrorHandler := func(err error) {
 		if !errors.Is(err, sctp.ErrResetPacketInStateNotExist) && !errors.Is(err, sctp.ErrChunk) {
-			t.params.Logger.Errorw(dc.Label()+" data channel error", err)
+			t.params.Logger.Warnw(dc.Label()+" data channel error", err)
 		}
 	}
 
@@ -809,7 +842,7 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni
 		t.lossyDC.OnClose(dcCloseHandler)
 		t.lossyDC.OnError(dcErrorHandler)
 	default:
-		t.params.Logger.Errorw("unknown data channel label", nil, "label", dc.Label())
+		t.params.Logger.Warnw("unknown data channel label", nil, "label", dc.Label())
 	}
 	t.lock.Unlock()
 
@@ -825,7 +858,7 @@ func (t *PCTransport) CreateDataChannelIfEmpty(dcLabel string, dci *webrtc.DataC
 	case LossyDataChannel:
 		dc = t.lossyDC
 	default:
-		t.params.Logger.Errorw("unknown data channel label", nil, "label", label)
+		t.params.Logger.Warnw("unknown data channel label", nil, "label", label)
 		err = errors.New("unknown data channel label")
 	}
 	t.lock.RUnlock()
@@ -1125,7 +1158,7 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc
 			// because sdp can negotiate multi times before migration.(it will sticky to the last m-line atfirst negotiate)
 			// so use a dummy pc to negotiate sdp to fixed the datachannel's mid at same position with previous answer
 			if err := t.preparePC(previousAnswer); err != nil {
-				t.params.Logger.Errorw("prepare pc for migration failed", err)
+				t.params.Logger.Warnw("prepare pc for migration failed", err)
 				return senders, err
 			}
 			continue
@@ -1163,7 +1196,7 @@ func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) {
 	if t.pc.RemoteDescription() == nil && t.previousAnswer == nil {
 		t.previousAnswer = answer
 		if senders, err := t.initPCWithPreviousAnswer(*t.previousAnswer); err != nil {
-			t.params.Logger.Errorw("initPCWithPreviousAnswer failed", err)
+			t.params.Logger.Warnw("initPCWithPreviousAnswer failed", err)
 			t.lock.Unlock()
 
 			t.params.Handler.OnNegotiationFailed()
@@ -1172,7 +1205,7 @@ func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) {
 			// in migration case, can't reuse transceiver before negotiated except track subscribed at previous node
 			t.canReuseTransceiver = false
 			if err := t.parseTrackMid(*offer, senders); err != nil {
-				t.params.Logger.Errorw("parse previous offer failed", err, "offer", offer.SDP)
+				t.params.Logger.Warnw("parse previous offer failed", err, "offer", offer.SDP)
 			}
 		}
 	}
@@ -1212,7 +1245,7 @@ func (t *PCTransport) postEvent(event event) {
 		err := t.handleEvent(&event)
 		if err != nil {
 			if !t.isClosed.Load() {
-				t.params.Logger.Errorw("error handling event", err, "event", event.String())
+				t.params.Logger.Warnw("error handling event", err, "event", event.String())
 				t.params.Handler.OnNegotiationFailed()
 			}
 		}
@@ -1359,7 +1392,7 @@ func (t *PCTransport) setNegotiationState(state transport.NegotiationState) {
 func (t *PCTransport) filterCandidates(sd webrtc.SessionDescription, preferTCP bool) webrtc.SessionDescription {
 	parsed, err := sd.Unmarshal()
 	if err != nil {
-		t.params.Logger.Errorw("could not unmarshal SDP to filter candidates", err)
+		t.params.Logger.Warnw("could not unmarshal SDP to filter candidates", err)
 		return sd
 	}
 
@@ -1389,7 +1422,7 @@ func (t *PCTransport) filterCandidates(sd webrtc.SessionDescription, preferTCP b
 
 	bytes, err := parsed.Marshal()
 	if err != nil {
-		t.params.Logger.Errorw("could not marshal SDP to filter candidates", err)
+		t.params.Logger.Warnw("could not marshal SDP to filter candidates", err)
 		return sd
 	}
 	sd.SDP = string(bytes)
@@ -1532,11 +1565,7 @@ func (t *PCTransport) handleRemoteDescriptionReceived(e *event) error {
 	}
 }
 
-func (t *PCTransport) isRemoteOfferRestartICE(sd *webrtc.SessionDescription) (string, bool, error) {
-	parsed, err := sd.Unmarshal()
-	if err != nil {
-		return "", false, err
-	}
+func (t *PCTransport) isRemoteOfferRestartICE(parsed *sdp.SessionDescription) (string, bool, error) {
 	user, pwd, err := lksdp.ExtractICECredential(parsed)
 	if err != nil {
 		return "", false, err
@@ -1633,7 +1662,11 @@ func (t *PCTransport) createAndSendAnswer() error {
 }
 
 func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) error {
-	iceCredential, offerRestartICE, err := t.isRemoteOfferRestartICE(sd)
+	parsed, err := sd.Unmarshal()
+	if err != nil {
+		return nil
+	}
+	iceCredential, offerRestartICE, err := t.isRemoteOfferRestartICE(parsed)
 	if err != nil {
 		return errors.Wrap(err, "check remote offer restart ice failed")
 	}
@@ -1655,6 +1688,13 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e
 	if err := t.setRemoteDescription(*sd); err != nil {
 		return err
 	}
+	rtxRepairs := rtxRepairsFromSDP(parsed, t.params.Logger)
+	if len(rtxRepairs) > 0 {
+		t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs)
+		for repair, base := range rtxRepairs {
+			t.params.Config.BufferFactory.SetRTXPair(repair, base)
+		}
+	}
 
 	if t.currentOfferIceCredential == "" || offerRestartICE {
 		t.currentOfferIceCredential = iceCredential
@@ -1778,3 +1818,35 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool
 
 	tr.SetCodecPreferences(configCodecs)
 }
+
+func rtxRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 {
+	rtxRepairFlows := map[uint32]uint32{}
+	for _, media := range s.MediaDescriptions {
+		for _, attr := range media.Attributes {
+			switch attr.Key {
+			case sdp.AttrKeySSRCGroup:
+				split := strings.Split(attr.Value, " ")
+				if split[0] == sdp.SemanticTokenFlowIdentification {
+					// Essentially lines like `a=ssrc-group:FID 2231627014 632943048` are processed by this section
+					// as this declares that the second SSRC (632943048) is a rtx repair flow (RFC4588) for the first
+					// (2231627014) as specified in RFC5576
+					if len(split) == 3 {
+						baseSsrc, err := strconv.ParseUint(split[1], 10, 32)
+						if err != nil {
+							logger.Warnw("Failed to parse SSRC", err, "ssrc", split[1])
+							continue
+						}
+						rtxRepairFlow, err := strconv.ParseUint(split[2], 10, 32)
+						if err != nil {
+							logger.Warnw("Failed to parse SSRC", err, "ssrc", split[2])
+							continue
+						}
+						rtxRepairFlows[uint32(rtxRepairFlow)] = uint32(baseSsrc)
+					}
+				}
+			}
+		}
+	}
+
+	return rtxRepairFlows
+}

pkg/rtc/transport/handler.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package transport
 
 import (

pkg/rtc/transport/negotiationstate.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package transport
 
 import "fmt"

pkg/rtc/transportmanager.go

@@ -33,6 +33,7 @@ import (
 	"github.com/livekit/livekit-server/pkg/rtc/types"
 	"github.com/livekit/livekit-server/pkg/sfu"
 	"github.com/livekit/livekit-server/pkg/sfu/pacer"
+	"github.com/livekit/mediatransportutil/pkg/twcc"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 )
@@ -71,6 +72,7 @@ type TransportManagerParams struct {
 	SID                          livekit.ParticipantID
 	SubscriberAsPrimary          bool
 	Config                       *WebRTCConfig
+	Twcc                         *twcc.Responder
 	ProtocolVersion              types.ProtocolVersion
 	CongestionControlConfig      config.CongestionControlConfig
 	EnabledSubscribeCodecs       []*livekit.Codec
@@ -131,6 +133,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro
 		ParticipantIdentity:     params.Identity,
 		ProtocolVersion:         params.ProtocolVersion,
 		Config:                  params.Config,
+		Twcc:                    params.Twcc,
 		DirectionConfig:         params.Config.Publisher,
 		CongestionControlConfig: params.CongestionControlConfig,
 		EnabledCodecs:           params.EnabledPublishCodecs,

pkg/rtc/types/interfaces.go

@@ -523,7 +523,7 @@ type SubscribedTrack interface {
 	RTPSender() *webrtc.RTPSender
 	IsMuted() bool
 	SetPublisherMuted(muted bool)
-	UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings)
+	UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings, isImmediate bool)
 	// selects appropriate video layer according to subscriber preferences
 	UpdateVideoLayer()
 	NeedsNegotiation() bool

pkg/rtc/types/typesfakes/fake_subscribed_track.go

@@ -161,10 +161,11 @@ type FakeSubscribedTrack struct {
 	subscriberIdentityReturnsOnCall map[int]struct {
 		result1 livekit.ParticipantIdentity
 	}
-	UpdateSubscriberSettingsStub        func(*livekit.UpdateTrackSettings)
+	UpdateSubscriberSettingsStub        func(*livekit.UpdateTrackSettings, bool)
 	updateSubscriberSettingsMutex       sync.RWMutex
 	updateSubscriberSettingsArgsForCall []struct {
 		arg1 *livekit.UpdateTrackSettings
+		arg2 bool
 	}
 	UpdateVideoLayerStub        func()
 	updateVideoLayerMutex       sync.RWMutex
@@ -991,16 +992,17 @@ func (fake *FakeSubscribedTrack) SubscriberIdentityReturnsOnCall(i int, result1
 	}{result1}
 }
 
-func (fake *FakeSubscribedTrack) UpdateSubscriberSettings(arg1 *livekit.UpdateTrackSettings) {
+func (fake *FakeSubscribedTrack) UpdateSubscriberSettings(arg1 *livekit.UpdateTrackSettings, arg2 bool) {
 	fake.updateSubscriberSettingsMutex.Lock()
 	fake.updateSubscriberSettingsArgsForCall = append(fake.updateSubscriberSettingsArgsForCall, struct {
 		arg1 *livekit.UpdateTrackSettings
-	}{arg1})
+		arg2 bool
+	}{arg1, arg2})
 	stub := fake.UpdateSubscriberSettingsStub
-	fake.recordInvocation("UpdateSubscriberSettings", []interface{}{arg1})
+	fake.recordInvocation("UpdateSubscriberSettings", []interface{}{arg1, arg2})
 	fake.updateSubscriberSettingsMutex.Unlock()
 	if stub != nil {
-		fake.UpdateSubscriberSettingsStub(arg1)
+		fake.UpdateSubscriberSettingsStub(arg1, arg2)
 	}
 }
 
@@ -1010,17 +1012,17 @@ func (fake *FakeSubscribedTrack) UpdateSubscriberSettingsCallCount() int {
 	return len(fake.updateSubscriberSettingsArgsForCall)
 }
 
-func (fake *FakeSubscribedTrack) UpdateSubscriberSettingsCalls(stub func(*livekit.UpdateTrackSettings)) {
+func (fake *FakeSubscribedTrack) UpdateSubscriberSettingsCalls(stub func(*livekit.UpdateTrackSettings, bool)) {
 	fake.updateSubscriberSettingsMutex.Lock()
 	defer fake.updateSubscriberSettingsMutex.Unlock()
 	fake.UpdateSubscriberSettingsStub = stub
 }
 
-func (fake *FakeSubscribedTrack) UpdateSubscriberSettingsArgsForCall(i int) *livekit.UpdateTrackSettings {
+func (fake *FakeSubscribedTrack) UpdateSubscriberSettingsArgsForCall(i int) (*livekit.UpdateTrackSettings, bool) {
 	fake.updateSubscriberSettingsMutex.RLock()
 	defer fake.updateSubscriberSettingsMutex.RUnlock()
 	argsForCall := fake.updateSubscriberSettingsArgsForCall[i]
-	return argsForCall.arg1
+	return argsForCall.arg1, argsForCall.arg2
 }
 
 func (fake *FakeSubscribedTrack) UpdateVideoLayer() {

pkg/rtc/unhandlesimulcast.go

@@ -19,6 +19,8 @@ import (
 	"github.com/pion/rtp"
 	"github.com/pion/sdp/v3"
 	"github.com/pion/webrtc/v3"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
 )
 
 const (
@@ -109,20 +111,11 @@ type UnhandleSimulcastInterceptor struct {
 	simTracks map[uint32]SimulcastTrackInfo
 }
 
-func getHeaderExtensionID(extensions []interceptor.RTPHeaderExtension, extension webrtc.RTPHeaderExtensionCapability) int {
-	for _, h := range extensions {
-		if extension.URI == h.URI {
-			return h.ID
-		}
-	}
-	return 0
-}
-
 func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
 	if t, ok := u.simTracks[info.SSRC]; ok {
 		// if we support fec for simulcast streams at future, should get rsid extensions
-		midExtensionID := getHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
-		streamIDExtensionID := getHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+		midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+		streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
 		if midExtensionID == 0 || streamIDExtensionID == 0 {
 			return reader
 		}

pkg/rtc/uptrackmanager.go

@@ -37,6 +37,10 @@ type UpTrackManagerParams struct {
 
 // UpTrackManager manages all uptracks from a participant
 type UpTrackManager struct {
+	// utils.TimedVersion is a atomic. To be correctly aligned also on 32bit archs
+	// 64it atomics need to be at the front of a struct
+	subscriptionPermissionVersion utils.TimedVersion
+
 	params UpTrackManagerParams
 
 	closed bool
@@ -44,7 +48,6 @@ type UpTrackManager struct {
 	// publishedTracks that participant is publishing
 	publishedTracks               map[livekit.TrackID]types.MediaTrack
 	subscriptionPermission        *livekit.SubscriptionPermission
-	subscriptionPermissionVersion utils.TimedVersion
 	// subscriber permission for published tracks
 	subscriberPermissions map[livekit.ParticipantIdentity]*livekit.TrackPermission // subscriberIdentity => *livekit.TrackPermission
 
@@ -64,22 +67,37 @@ func NewUpTrackManager(params UpTrackManagerParams) *UpTrackManager {
 
 func (u *UpTrackManager) Close(willBeResumed bool) {
 	u.lock.Lock()
-	u.closed = true
-	notify := len(u.publishedTracks) == 0
-	u.lock.Unlock()
-
-	// remove all subscribers
-	for _, t := range u.GetPublishedTracks() {
-		t.ClearAllReceivers(willBeResumed)
+	if u.closed {
+		u.lock.Unlock()
+		return
 	}
 
-	if notify && u.onClose != nil {
-		u.onClose()
+	u.closed = true
+
+	publishedTracks := u.publishedTracks
+	u.publishedTracks = make(map[livekit.TrackID]types.MediaTrack)
+	u.lock.Unlock()
+
+	for _, t := range publishedTracks {
+		t.Close(willBeResumed)
+	}
+
+	if onClose := u.getOnUpTrackManagerClose(); onClose != nil {
+		onClose()
 	}
 }
 
 func (u *UpTrackManager) OnUpTrackManagerClose(f func()) {
+	u.lock.Lock()
 	u.onClose = f
+	u.lock.Unlock()
+}
+
+func (u *UpTrackManager) getOnUpTrackManagerClose() func() {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	return u.onClose
 }
 
 func (u *UpTrackManager) ToProto() []*livekit.TrackInfo {
@@ -247,22 +265,10 @@ func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) {
 	u.params.Logger.Debugw("added published track", "trackID", track.ID(), "trackInfo", logger.Proto(track.ToProto()))
 
 	track.AddOnClose(func() {
-		notifyClose := false
-
-		// cleanup
 		u.lock.Lock()
-		trackID := track.ID()
-		delete(u.publishedTracks, trackID)
+		delete(u.publishedTracks, track.ID())
 		// not modifying subscription permissions, will get reset on next update from participant
-
-		if u.closed && len(u.publishedTracks) == 0 {
-			notifyClose = true
-		}
 		u.lock.Unlock()
-
-		if notifyClose && u.onClose != nil {
-			u.onClose()
-		}
 	})
 }
 

pkg/service/clients.go

@@ -26,10 +26,15 @@ import (
 	"github.com/livekit/protocol/utils"
 )
 
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+
+//counterfeiter:generate . IOClient
 type IOClient interface {
 	CreateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error)
 	GetEgress(ctx context.Context, req *rpc.GetEgressRequest) (*livekit.EgressInfo, error)
 	ListEgress(ctx context.Context, req *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error)
+	CreateIngress(ctx context.Context, req *livekit.IngressInfo) (*emptypb.Empty, error)
+	UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error)
 }
 
 type egressLauncher struct {

pkg/service/ingress.go

@@ -39,6 +39,7 @@ type IngressService struct {
 	bus         psrpc.MessageBus
 	psrpcClient rpc.IngressClient
 	store       IngressStore
+	io          IOClient
 	roomService livekit.RoomService
 	telemetry   telemetry.TelemetryService
 	launcher    IngressLauncher
@@ -50,6 +51,7 @@ func NewIngressServiceWithIngressLauncher(
 	bus psrpc.MessageBus,
 	psrpcClient rpc.IngressClient,
 	store IngressStore,
+	io IOClient,
 	rs livekit.RoomService,
 	ts telemetry.TelemetryService,
 	launcher IngressLauncher,
@@ -61,6 +63,7 @@ func NewIngressServiceWithIngressLauncher(
 		bus:         bus,
 		psrpcClient: psrpcClient,
 		store:       store,
+		io:          io,
 		roomService: rs,
 		telemetry:   ts,
 		launcher:    launcher,
@@ -73,10 +76,11 @@ func NewIngressService(
 	bus psrpc.MessageBus,
 	psrpcClient rpc.IngressClient,
 	store IngressStore,
+	io IOClient,
 	rs livekit.RoomService,
 	ts telemetry.TelemetryService,
 ) *IngressService {
-	s := NewIngressServiceWithIngressLauncher(conf, nodeID, bus, psrpcClient, store, rs, ts, nil)
+	s := NewIngressServiceWithIngressLauncher(conf, nodeID, bus, psrpcClient, store, io, rs, ts, nil)
 
 	s.launcher = s
 
@@ -156,6 +160,7 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string
 		RoomName:            req.RoomName,
 		ParticipantIdentity: req.ParticipantIdentity,
 		ParticipantName:     req.ParticipantName,
+		ParticipantMetadata: req.ParticipantMetadata,
 		State:               &livekit.IngressState{},
 	}
 
@@ -191,11 +196,19 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string
 		}
 	}
 
-	if err = s.store.StoreIngress(ctx, info); err != nil {
-		logger.Errorw("could not write ingress info", err)
+	// TODO Remove this store Ingress call for URL pull as it is redundant since
+	// the ingress service sends a CreateIngress RPC
+	_, err = s.io.CreateIngress(ctx, info)
+	switch err {
+	case nil:
+		break
+	case ingress.ErrIngressOutOfDate:
+		// Error returned if the ingress was already created by the ingress service
+		err = nil
+	default:
+		logger.Errorw("could not create ingress object", err)
 		return nil, err
 	}
-	s.telemetry.IngressCreated(ctx, info)
 
 	return info, nil
 }
@@ -224,6 +237,9 @@ func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.Ing
 	if req.BypassTranscoding != nil {
 		info.BypassTranscoding = *req.BypassTranscoding
 	}
+	if req.ParticipantMetadata != "" {
+		info.ParticipantMetadata = req.ParticipantMetadata
+	}
 	if req.Audio != nil {
 		info.Audio = req.Audio
 	}
@@ -270,7 +286,10 @@ func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateI
 	switch info.State.Status {
 	case livekit.IngressState_ENDPOINT_ERROR:
 		info.State.Status = livekit.IngressState_ENDPOINT_INACTIVE
-		err = s.store.UpdateIngressState(ctx, req.IngressId, info.State)
+		_, err = s.io.UpdateIngressState(ctx, &rpc.UpdateIngressStateRequest{
+			IngressId: req.IngressId,
+			State:     info.State,
+		})
 		if err != nil {
 			logger.Warnw("could not store ingress state", err)
 		}

pkg/service/ioservice.go

@@ -16,7 +16,6 @@ package service
 
 import (
 	"context"
-	"errors"
 
 	"google.golang.org/protobuf/types/known/emptypb"
 
@@ -78,6 +77,14 @@ func (s *IOInfoService) Start() error {
 	return nil
 }
 
+func (s *IOInfoService) Stop() {
+	close(s.shutdown)
+
+	if s.ioServer != nil {
+		s.ioServer.Shutdown()
+	}
+}
+
 func (s *IOInfoService) CreateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) {
 	// check if egress already exists to avoid duplicate EgressStarted event
 	if _, err := s.es.LoadEgress(ctx, info.EgressId); err == nil {
@@ -156,79 +163,3 @@ func (s *IOInfoService) UpdateMetrics(ctx context.Context, req *rpc.UpdateMetric
 	)
 	return &emptypb.Empty{}, nil
 }
-
-func (s *IOInfoService) GetIngressInfo(ctx context.Context, req *rpc.GetIngressInfoRequest) (*rpc.GetIngressInfoResponse, error) {
-	info, err := s.loadIngressFromInfoRequest(req)
-	if err != nil {
-		return nil, err
-	}
-
-	return &rpc.GetIngressInfoResponse{Info: info}, nil
-}
-
-func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoRequest) (info *livekit.IngressInfo, err error) {
-	if req.IngressId != "" {
-		info, err = s.is.LoadIngress(context.Background(), req.IngressId)
-	} else if req.StreamKey != "" {
-		info, err = s.is.LoadIngressFromStreamKey(context.Background(), req.StreamKey)
-	} else {
-		err = errors.New("request needs to specify either IngressId or StreamKey")
-	}
-	return info, err
-}
-
-func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) {
-	info, err := s.is.LoadIngress(ctx, req.IngressId)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil {
-		logger.Errorw("could not update ingress", err)
-		return nil, err
-	}
-
-	if info.State.Status != req.State.Status {
-		info.State = req.State
-
-		switch req.State.Status {
-		case livekit.IngressState_ENDPOINT_ERROR,
-			livekit.IngressState_ENDPOINT_INACTIVE,
-			livekit.IngressState_ENDPOINT_COMPLETE:
-			s.telemetry.IngressEnded(ctx, info)
-
-			if req.State.Error != "" {
-				logger.Infow("ingress failed", "error", req.State.Error, "ingressID", req.IngressId)
-			} else {
-				logger.Infow("ingress ended", "ingressID", req.IngressId)
-			}
-
-		case livekit.IngressState_ENDPOINT_PUBLISHING:
-			s.telemetry.IngressStarted(ctx, info)
-
-			logger.Infow("ingress started", "ingressID", req.IngressId)
-
-		case livekit.IngressState_ENDPOINT_BUFFERING:
-			s.telemetry.IngressUpdated(ctx, info)
-
-			logger.Infow("ingress buffering", "ingressID", req.IngressId)
-		}
-	} else {
-		// Status didn't change, send Updated event
-		info.State = req.State
-
-		s.telemetry.IngressUpdated(ctx, info)
-
-		logger.Infow("ingress updated", "ingressID", req.IngressId, "status", info.State.Status)
-	}
-
-	return &emptypb.Empty{}, nil
-}
-
-func (s *IOInfoService) Stop() {
-	close(s.shutdown)
-
-	if s.ioServer != nil {
-		s.ioServer.Shutdown()
-	}
-}

pkg/service/ioservice_ingress.go

@@ -0,0 +1,104 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"context"
+	"errors"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/rpc"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func (s *IOInfoService) CreateIngress(ctx context.Context, info *livekit.IngressInfo) (*emptypb.Empty, error) {
+	err := s.is.StoreIngress(ctx, info)
+	if err != nil {
+		return nil, err
+	}
+
+	s.telemetry.IngressCreated(ctx, info)
+
+	return &emptypb.Empty{}, nil
+}
+
+func (s *IOInfoService) GetIngressInfo(ctx context.Context, req *rpc.GetIngressInfoRequest) (*rpc.GetIngressInfoResponse, error) {
+	info, err := s.loadIngressFromInfoRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rpc.GetIngressInfoResponse{Info: info}, nil
+}
+
+func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoRequest) (info *livekit.IngressInfo, err error) {
+	if req.IngressId != "" {
+		info, err = s.is.LoadIngress(context.Background(), req.IngressId)
+	} else if req.StreamKey != "" {
+		info, err = s.is.LoadIngressFromStreamKey(context.Background(), req.StreamKey)
+	} else {
+		err = errors.New("request needs to specify either IngressId or StreamKey")
+	}
+	return info, err
+}
+
+func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) {
+	info, err := s.is.LoadIngress(ctx, req.IngressId)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil {
+		logger.Errorw("could not update ingress", err)
+		return nil, err
+	}
+
+	if info.State.Status != req.State.Status {
+		info.State = req.State
+
+		switch req.State.Status {
+		case livekit.IngressState_ENDPOINT_ERROR,
+			livekit.IngressState_ENDPOINT_INACTIVE,
+			livekit.IngressState_ENDPOINT_COMPLETE:
+			s.telemetry.IngressEnded(ctx, info)
+
+			if req.State.Error != "" {
+				logger.Infow("ingress failed", "error", req.State.Error, "ingressID", req.IngressId)
+			} else {
+				logger.Infow("ingress ended", "ingressID", req.IngressId)
+			}
+
+		case livekit.IngressState_ENDPOINT_PUBLISHING:
+			s.telemetry.IngressStarted(ctx, info)
+
+			logger.Infow("ingress started", "ingressID", req.IngressId)
+
+		case livekit.IngressState_ENDPOINT_BUFFERING:
+			s.telemetry.IngressUpdated(ctx, info)
+
+			logger.Infow("ingress buffering", "ingressID", req.IngressId)
+		}
+	} else {
+		// Status didn't change, send Updated event
+		info.State = req.State
+
+		s.telemetry.IngressUpdated(ctx, info)
+
+		logger.Infow("ingress state updated", "ingressID", req.IngressId, "status", info.State.Status)
+	}
+
+	return &emptypb.Empty{}, nil
+}

pkg/service/ioservice_sip.go

@@ -16,6 +16,7 @@ package service
 
 import (
 	"context"
+	"errors"
 
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
@@ -57,6 +58,9 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E
 	}
 	best, err := s.matchSIPDispatchRule(ctx, trunk, req)
 	if err != nil {
+		if e := (*sip.ErrNoDispatchMatched)(nil); errors.As(err, &e) {
+			return &rpc.EvaluateSIPDispatchRulesResponse{Result: rpc.SIPDispatchResult_DROP}, nil
+		}
 		return nil, err
 	}
 	logger.Debugw("SIP dispatch rule matched", "dispatchRule", best.SipDispatchRuleId, "called", req.CalledNumber, "calling", req.CallingNumber)

pkg/service/roommanager.go

@@ -469,7 +469,7 @@ func (r *RoomManager) StartSession(
 	}
 
 	persistRoomForParticipantCount := func(proto *livekit.Room) {
-		if !participant.Hidden() {
+		if !participant.Hidden() && !room.IsClosed() {
 			err = r.roomStore.StoreRoom(ctx, proto, room.Internal())
 			if err != nil {
 				logger.Errorw("could not store room", err)

pkg/service/roomservice.go

@@ -17,8 +17,8 @@ package service
 import (
 	"context"
 	"strconv"
-	"time"
 
+	"github.com/avast/retry-go/v4"
 	"github.com/pkg/errors"
 	"github.com/twitchtv/twirp"
 
@@ -100,19 +100,6 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq
 	defer res.RequestSink.Close()
 	defer res.ResponseSource.Close()
 
-	// ensure it's created correctly
-	err = s.confirmExecution(func() error {
-		_, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Name), false)
-		if err != nil {
-			return ErrOperationFailed
-		} else {
-			return nil
-		}
-	})
-	if err != nil {
-		return nil, err
-	}
-
 	if created {
 		go func() {
 			s.agentClient.JobRequest(ctx, &livekit.Job{
@@ -299,7 +286,7 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat
 		return nil, err
 	}
 
-	err = s.confirmExecution(func() error {
+	err = s.confirmExecution(ctx, func() error {
 		room, _, err = s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false)
 		if err != nil {
 			return err
@@ -326,19 +313,14 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat
 	return room, nil
 }
 
-func (s *RoomService) confirmExecution(f func() error) error {
-	expired := time.After(s.apiConf.ExecutionTimeout)
-	var err error
-	for {
-		select {
-		case <-expired:
-			return err
-		default:
-			err = f()
-			if err == nil {
-				return nil
-			}
-			time.Sleep(s.apiConf.CheckInterval)
-		}
-	}
+func (s *RoomService) confirmExecution(ctx context.Context, f func() error) error {
+	ctx, cancel := context.WithTimeout(ctx, s.apiConf.ExecutionTimeout)
+	defer cancel()
+	return retry.Do(
+		f,
+		retry.Context(ctx),
+		retry.Delay(s.apiConf.CheckInterval),
+		retry.MaxDelay(s.apiConf.MaxCheckInterval),
+		retry.DelayType(retry.BackOffDelay),
+	)
 }

pkg/service/servicefakes/fake_ioclient.go

@@ -26,6 +26,20 @@ type FakeIOClient struct {
 		result1 *emptypb.Empty
 		result2 error
 	}
+	CreateIngressStub        func(context.Context, *livekit.IngressInfo) (*emptypb.Empty, error)
+	createIngressMutex       sync.RWMutex
+	createIngressArgsForCall []struct {
+		arg1 context.Context
+		arg2 *livekit.IngressInfo
+	}
+	createIngressReturns struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
+	createIngressReturnsOnCall map[int]struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
 	GetEgressStub        func(context.Context, *rpc.GetEgressRequest) (*livekit.EgressInfo, error)
 	getEgressMutex       sync.RWMutex
 	getEgressArgsForCall []struct {
@@ -54,6 +68,20 @@ type FakeIOClient struct {
 		result1 *livekit.ListEgressResponse
 		result2 error
 	}
+	UpdateIngressStateStub        func(context.Context, *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error)
+	updateIngressStateMutex       sync.RWMutex
+	updateIngressStateArgsForCall []struct {
+		arg1 context.Context
+		arg2 *rpc.UpdateIngressStateRequest
+	}
+	updateIngressStateReturns struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
+	updateIngressStateReturnsOnCall map[int]struct {
+		result1 *emptypb.Empty
+		result2 error
+	}
 	invocations      map[string][][]interface{}
 	invocationsMutex sync.RWMutex
 }
@@ -123,6 +151,71 @@ func (fake *FakeIOClient) CreateEgressReturnsOnCall(i int, result1 *emptypb.Empt
 	}{result1, result2}
 }
 
+func (fake *FakeIOClient) CreateIngress(arg1 context.Context, arg2 *livekit.IngressInfo) (*emptypb.Empty, error) {
+	fake.createIngressMutex.Lock()
+	ret, specificReturn := fake.createIngressReturnsOnCall[len(fake.createIngressArgsForCall)]
+	fake.createIngressArgsForCall = append(fake.createIngressArgsForCall, struct {
+		arg1 context.Context
+		arg2 *livekit.IngressInfo
+	}{arg1, arg2})
+	stub := fake.CreateIngressStub
+	fakeReturns := fake.createIngressReturns
+	fake.recordInvocation("CreateIngress", []interface{}{arg1, arg2})
+	fake.createIngressMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1, ret.result2
+	}
+	return fakeReturns.result1, fakeReturns.result2
+}
+
+func (fake *FakeIOClient) CreateIngressCallCount() int {
+	fake.createIngressMutex.RLock()
+	defer fake.createIngressMutex.RUnlock()
+	return len(fake.createIngressArgsForCall)
+}
+
+func (fake *FakeIOClient) CreateIngressCalls(stub func(context.Context, *livekit.IngressInfo) (*emptypb.Empty, error)) {
+	fake.createIngressMutex.Lock()
+	defer fake.createIngressMutex.Unlock()
+	fake.CreateIngressStub = stub
+}
+
+func (fake *FakeIOClient) CreateIngressArgsForCall(i int) (context.Context, *livekit.IngressInfo) {
+	fake.createIngressMutex.RLock()
+	defer fake.createIngressMutex.RUnlock()
+	argsForCall := fake.createIngressArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeIOClient) CreateIngressReturns(result1 *emptypb.Empty, result2 error) {
+	fake.createIngressMutex.Lock()
+	defer fake.createIngressMutex.Unlock()
+	fake.CreateIngressStub = nil
+	fake.createIngressReturns = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) CreateIngressReturnsOnCall(i int, result1 *emptypb.Empty, result2 error) {
+	fake.createIngressMutex.Lock()
+	defer fake.createIngressMutex.Unlock()
+	fake.CreateIngressStub = nil
+	if fake.createIngressReturnsOnCall == nil {
+		fake.createIngressReturnsOnCall = make(map[int]struct {
+			result1 *emptypb.Empty
+			result2 error
+		})
+	}
+	fake.createIngressReturnsOnCall[i] = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
 func (fake *FakeIOClient) GetEgress(arg1 context.Context, arg2 *rpc.GetEgressRequest) (*livekit.EgressInfo, error) {
 	fake.getEgressMutex.Lock()
 	ret, specificReturn := fake.getEgressReturnsOnCall[len(fake.getEgressArgsForCall)]
@@ -253,15 +346,84 @@ func (fake *FakeIOClient) ListEgressReturnsOnCall(i int, result1 *livekit.ListEg
 	}{result1, result2}
 }
 
+func (fake *FakeIOClient) UpdateIngressState(arg1 context.Context, arg2 *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) {
+	fake.updateIngressStateMutex.Lock()
+	ret, specificReturn := fake.updateIngressStateReturnsOnCall[len(fake.updateIngressStateArgsForCall)]
+	fake.updateIngressStateArgsForCall = append(fake.updateIngressStateArgsForCall, struct {
+		arg1 context.Context
+		arg2 *rpc.UpdateIngressStateRequest
+	}{arg1, arg2})
+	stub := fake.UpdateIngressStateStub
+	fakeReturns := fake.updateIngressStateReturns
+	fake.recordInvocation("UpdateIngressState", []interface{}{arg1, arg2})
+	fake.updateIngressStateMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1, ret.result2
+	}
+	return fakeReturns.result1, fakeReturns.result2
+}
+
+func (fake *FakeIOClient) UpdateIngressStateCallCount() int {
+	fake.updateIngressStateMutex.RLock()
+	defer fake.updateIngressStateMutex.RUnlock()
+	return len(fake.updateIngressStateArgsForCall)
+}
+
+func (fake *FakeIOClient) UpdateIngressStateCalls(stub func(context.Context, *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error)) {
+	fake.updateIngressStateMutex.Lock()
+	defer fake.updateIngressStateMutex.Unlock()
+	fake.UpdateIngressStateStub = stub
+}
+
+func (fake *FakeIOClient) UpdateIngressStateArgsForCall(i int) (context.Context, *rpc.UpdateIngressStateRequest) {
+	fake.updateIngressStateMutex.RLock()
+	defer fake.updateIngressStateMutex.RUnlock()
+	argsForCall := fake.updateIngressStateArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeIOClient) UpdateIngressStateReturns(result1 *emptypb.Empty, result2 error) {
+	fake.updateIngressStateMutex.Lock()
+	defer fake.updateIngressStateMutex.Unlock()
+	fake.UpdateIngressStateStub = nil
+	fake.updateIngressStateReturns = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
+func (fake *FakeIOClient) UpdateIngressStateReturnsOnCall(i int, result1 *emptypb.Empty, result2 error) {
+	fake.updateIngressStateMutex.Lock()
+	defer fake.updateIngressStateMutex.Unlock()
+	fake.UpdateIngressStateStub = nil
+	if fake.updateIngressStateReturnsOnCall == nil {
+		fake.updateIngressStateReturnsOnCall = make(map[int]struct {
+			result1 *emptypb.Empty
+			result2 error
+		})
+	}
+	fake.updateIngressStateReturnsOnCall[i] = struct {
+		result1 *emptypb.Empty
+		result2 error
+	}{result1, result2}
+}
+
 func (fake *FakeIOClient) Invocations() map[string][][]interface{} {
 	fake.invocationsMutex.RLock()
 	defer fake.invocationsMutex.RUnlock()
 	fake.createEgressMutex.RLock()
 	defer fake.createEgressMutex.RUnlock()
+	fake.createIngressMutex.RLock()
+	defer fake.createIngressMutex.RUnlock()
 	fake.getEgressMutex.RLock()
 	defer fake.getEgressMutex.RUnlock()
 	fake.listEgressMutex.RLock()
 	defer fake.listEgressMutex.RUnlock()
+	fake.updateIngressStateMutex.RLock()
+	defer fake.updateIngressStateMutex.RUnlock()
 	copiedInvocations := map[string][][]interface{}{}
 	for key, value := range fake.invocations {
 		copiedInvocations[key] = value

pkg/service/sip.go

@@ -157,7 +157,7 @@ func (s *SIPService) DeleteSIPDispatchRule(ctx context.Context, req *livekit.Del
 	return info, nil
 }
 
-func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.CreateSIPParticipantRequest) (*livekit.SIPParticipantInfo, error) {
+func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *livekit.CreateSIPParticipantRequest, wsUrl, token string) (*livekit.SIPParticipantInfo, error) {
 	if s.store == nil {
 		return nil, ErrSIPNotConnected
 	}
@@ -167,6 +167,8 @@ func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.Crea
 		CallTo:              req.SipCallTo,
 		RoomName:            req.RoomName,
 		ParticipantIdentity: req.ParticipantIdentity,
+		WsUrl:               wsUrl,
+		Token:               token,
 	}
 	if req.SipTrunkId != "" {
 		trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId)
@@ -202,3 +204,6 @@ func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.Crea
 		RoomName:            req.RoomName,
 	}, nil
 }
+func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.CreateSIPParticipantRequest) (*livekit.SIPParticipantInfo, error) {
+	return s.CreateSIPParticipantWithToken(ctx, req, "", "")
+}

pkg/service/utils.go

@@ -15,6 +15,8 @@
 package service
 
 import (
+	"context"
+	"errors"
 	"net"
 	"net/http"
 	"regexp"
@@ -27,7 +29,9 @@ func handleError(w http.ResponseWriter, r *http.Request, status int, err error,
 	if r != nil && r.URL != nil {
 		keysAndValues = append(keysAndValues, "method", r.Method, "path", r.URL.Path)
 	}
-	logger.GetLogger().WithCallDepth(1).Warnw("error handling request", err, keysAndValues...)
+	if !errors.Is(err, context.Canceled) {
+		logger.GetLogger().WithCallDepth(1).Warnw("error handling request", err, keysAndValues...)
+	}
 	w.WriteHeader(status)
 	_, _ = w.Write([]byte(err.Error()))
 }

pkg/service/wire_gen.go

@@ -106,7 +106,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live
 	if err != nil {
 		return nil, err
 	}
-	ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, roomService, telemetryService)
+	ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, ioInfoService, roomService, telemetryService)
 	sipConfig := getSIPConfig(conf)
 	sipClient, err := rpc.NewSIPClient(messageBus)
 	if err != nil {

pkg/sfu/buffer/buffer.go

@@ -71,6 +71,7 @@ type Buffer struct {
 	videoPool     *sync.Pool
 	audioPool     *sync.Pool
 	codecType     webrtc.RTPCodecType
+	payloadType   uint8
 	extPackets    deque.Deque[*ExtPacket]
 	pPackets      []pendingPacket
 	closeOnce     sync.Once
@@ -120,8 +121,11 @@ type Buffer struct {
 	frameRateCalculator [DefaultMaxLayerSpatial + 1]FrameRateCalculator
 	frameRateCalculated bool
 
-	packetNotFoundCount atomic.Uint32
-	packetTooOldCount   atomic.Uint32
+	packetNotFoundCount   atomic.Uint32
+	packetTooOldCount     atomic.Uint32
+	extPacketTooMuchCount atomic.Uint32
+
+	primaryBufferForRTX *Buffer
 }
 
 // NewBuffer constructs a new Buffer
@@ -143,7 +147,7 @@ func (b *Buffer) SetLogger(logger logger.Logger) {
 	b.Lock()
 	defer b.Unlock()
 
-	b.logger = logger.WithComponent(sutils.ComponentSFU)
+	b.logger = logger.WithComponent(sutils.ComponentSFU).WithValues("ssrc", b.mediaSSRC)
 	if b.rtpStats != nil {
 		b.rtpStats.SetLogger(b.logger)
 	}
@@ -156,11 +160,12 @@ func (b *Buffer) SetPaused(paused bool) {
 	b.paused = paused
 }
 
-func (b *Buffer) SetTWCC(twcc *twcc.Responder) {
+func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) {
 	b.Lock()
 	defer b.Unlock()
 
 	b.twcc = twcc
+	b.twccExt = extID
 }
 
 func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) {
@@ -187,6 +192,17 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
 	b.clockRate = codec.ClockRate
 	b.lastReport = time.Now()
 	b.mime = strings.ToLower(codec.MimeType)
+	for _, codecParameter := range params.Codecs {
+		if strings.EqualFold(codecParameter.MimeType, codec.MimeType) {
+			b.payloadType = uint8(codecParameter.PayloadType)
+			break
+		}
+	}
+
+	if b.payloadType == 0 {
+		b.logger.Warnw("could not find payload type for codec", nil, "codec", codec.MimeType, "parameters", params)
+		b.payloadType = uint8(params.Codecs[0].PayloadType)
+	}
 
 	for _, ext := range params.HeaderExtensions {
 		switch ext.URI {
@@ -235,16 +251,6 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
 		case webrtc.TypeRTCPFBGoogREMB:
 			b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBGoogREMB)
 			b.logger.Debugw("REMB not supported, RTCP feedback will not be generated")
-		case webrtc.TypeRTCPFBTransportCC:
-			if b.codecType == webrtc.RTPCodecTypeVideo {
-				b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBTransportCC)
-				for _, ext := range params.HeaderExtensions {
-					if ext.URI == sdp.TransportCCURI {
-						b.twccExt = uint8(ext.ID)
-						break
-					}
-				}
-			}
 		case webrtc.TypeRTCPFBNACK:
 			// pion use a single mediaengine to manage negotiated codecs of peerconnection, that means we can't have different
 			// codec settings at track level for same codec type, so enable nack for all audio receivers but don't create nack queue
@@ -258,22 +264,46 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili
 	}
 
 	for _, pp := range b.pPackets {
-		b.calc(pp.packet, pp.arrivalTime)
+		b.calc(pp.packet, nil, pp.arrivalTime, false)
 	}
 	b.pPackets = nil
 	b.bound = true
 }
 
-// Write adds an RTP Packet, out of order, new packet may be arrived later
+// Write adds an RTP Packet, ordering is not guaranteed, newer packets may arrive later
 func (b *Buffer) Write(pkt []byte) (n int, err error) {
-	b.Lock()
-	defer b.Unlock()
+	var rtpPacket rtp.Packet
+	err = rtpPacket.Unmarshal(pkt)
+	if err != nil {
+		return
+	}
 
+	b.Lock()
 	if b.closed.Load() {
+		b.Unlock()
 		err = io.EOF
 		return
 	}
 
+	if b.twcc != nil && b.twccExt != 0 && !b.closed.Load() {
+		if ext := rtpPacket.GetExtension(b.twccExt); ext != nil {
+			b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), time.Now().UnixNano(), rtpPacket.Marker)
+		}
+	}
+
+	// handle RTX packet
+	if pb := b.primaryBufferForRTX; pb != nil {
+		b.Unlock()
+
+		// skip padding only packets
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			return
+		}
+
+		pb.writeRTX(&rtpPacket)
+		return
+	}
+
 	if !b.bound {
 		packet := make([]byte, len(pkt))
 		copy(packet, pkt)
@@ -281,10 +311,58 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) {
 			packet:      packet,
 			arrivalTime: time.Now(),
 		})
+		b.Unlock()
 		return
 	}
 
-	b.calc(pkt, time.Now())
+	b.payloadType = rtpPacket.PayloadType
+	b.calc(pkt, &rtpPacket, time.Now(), false)
+	b.Unlock()
+	return
+}
+
+func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) {
+	b.Lock()
+	b.primaryBufferForRTX = primaryBuffer
+	pkts := b.pPackets
+	b.pPackets = nil
+	b.Unlock()
+	for _, pp := range pkts {
+		var rtpPacket rtp.Packet
+		err := rtpPacket.Unmarshal(pp.packet)
+		if err != nil {
+			continue
+		}
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			continue
+		}
+		primaryBuffer.writeRTX(&rtpPacket)
+	}
+}
+
+func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) {
+	b.Lock()
+	defer b.Unlock()
+	if !b.bound {
+		return
+	}
+
+	videoPktPtr := b.videoPool.Get().(*[]byte)
+	defer b.videoPool.Put(videoPktPtr)
+
+	videoPkt := *rtxPkt
+	videoPkt.PayloadType = b.payloadType
+	videoPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2])
+	videoPkt.SSRC = b.mediaSSRC
+	videoPkt.Payload = rtxPkt.Payload[2:]
+	n, err = videoPkt.MarshalTo((*videoPktPtr))
+
+	if err != nil {
+		b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", videoPkt.SequenceNumber)
+		return
+	}
+
+	b.calc((*videoPktPtr)[:n], &videoPkt, time.Now(), true)
 	return
 }
 
@@ -414,23 +492,25 @@ func (b *Buffer) SetRTT(rtt uint32) {
 	}
 }
 
-func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) {
+func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime time.Time, isRTX bool) {
 	defer func() {
 		b.doNACKs()
 
 		b.doReports(arrivalTime)
 	}()
 
-	var rtpPacket rtp.Packet
-	if err := rtpPacket.Unmarshal(pkt); err != nil {
-		b.logger.Errorw("could not unmarshal RTP packet", err)
-		return
+	if rtpPacket == nil {
+		rtpPacket = &rtp.Packet{}
+		if err := rtpPacket.Unmarshal(rawPkt); err != nil {
+			b.logger.Errorw("could not unmarshal RTP packet", err)
+			return
+		}
 	}
 
 	// process header extensions always as padding packets could be used for probing
-	b.processHeaderExtensions(&rtpPacket, arrivalTime)
+	b.processHeaderExtensions(rtpPacket, arrivalTime, isRTX)
 
-	flowState := b.updateStreamState(&rtpPacket, arrivalTime)
+	flowState := b.updateStreamState(rtpPacket, arrivalTime)
 	if flowState.IsNotHandled {
 		return
 	}
@@ -457,10 +537,6 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) {
 				b.logger.Errorw("could not exclude range", err, "sn", rtpPacket.SequenceNumber, "esn", flowState.ExtSequenceNumber)
 			}
 		}
-		// TODO-VP9-DEBUG-REMOVE-START
-		snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber)
-		b.logger.Debugw("dropping padding packet", "sn", rtpPacket.SequenceNumber, "osn", flowState.ExtSequenceNumber, "msn", flowState.ExtSequenceNumber-snAdjustment, "error", err)
-		// TODO-VP9-DEBUG-REMOVE-END
 		return
 	}
 
@@ -472,7 +548,7 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) {
 	}
 	flowState.ExtSequenceNumber -= snAdjustment
 	rtpPacket.Header.SequenceNumber = uint16(flowState.ExtSequenceNumber)
-	_, err = b.bucket.AddPacketWithSequenceNumber(pkt, rtpPacket.Header.SequenceNumber)
+	_, err = b.bucket.AddPacketWithSequenceNumber(rawPkt, rtpPacket.Header.SequenceNumber)
 	if err != nil {
 		if errors.Is(err, bucket.ErrPacketTooOld) {
 			packetTooOldCount := b.packetTooOldCount.Inc()
@@ -485,12 +561,18 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) {
 		return
 	}
 
-	ep := b.getExtPacket(&rtpPacket, arrivalTime, flowState)
+	ep := b.getExtPacket(rtpPacket, arrivalTime, flowState)
 	if ep == nil {
 		return
 	}
 	b.extPackets.PushBack(ep)
 
+	if b.extPackets.Len() > b.bucket.Capacity() {
+		if (b.extPacketTooMuchCount.Inc()-1)%100 == 0 {
+			b.logger.Warnw("too much ext packets", nil, "count", b.extPackets.Len())
+		}
+	}
+
 	b.doFpsCalc(ep)
 }
 
@@ -498,7 +580,7 @@ func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket {
 	n, err := b.getPacket(buf, ep.Packet.SequenceNumber)
 	if err != nil {
 		packetNotFoundCount := b.packetNotFoundCount.Inc()
-		if packetNotFoundCount%20 == 0 {
+		if (packetNotFoundCount-1)%20 == 0 {
 			b.logger.Warnw("could not get packet from bucket", err, "sn", ep.Packet.SequenceNumber, "headSN", b.bucket.HeadSequenceNumber(), "count", packetNotFoundCount)
 		}
 		return nil
@@ -570,16 +652,9 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) RTPFlow
 	return flowState
 }
 
-func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time) {
-	// submit to TWCC even if it is a padding only packet. Clients use padding only packets as probes
-	// for bandwidth estimation
-	if b.twcc != nil && b.twccExt != 0 {
-		if ext := p.GetExtension(b.twccExt); ext != nil {
-			b.twcc.Push(binary.BigEndian.Uint16(ext[0:2]), arrivalTime.UnixNano(), p.Marker)
-		}
-	}
+func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time, isRTX bool) {
 
-	if b.audioLevelExt != 0 {
+	if b.audioLevelExt != 0 && !isRTX {
 		if !b.latestTSForAudioLevelInitialized {
 			b.latestTSForAudioLevelInitialized = true
 			b.latestTSForAudioLevel = p.Timestamp

pkg/sfu/buffer/factory.go

@@ -51,6 +51,7 @@ func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory {
 		audioPool:   f.audioPool,
 		rtpBuffers:  make(map[uint32]*Buffer),
 		rtcpReaders: make(map[uint32]*RTCPReader),
+		rtxPair:     make(map[uint32]uint32),
 	}
 }
 
@@ -60,6 +61,7 @@ type Factory struct {
 	audioPool   *sync.Pool
 	rtpBuffers  map[uint32]*Buffer
 	rtcpReaders map[uint32]*RTCPReader
+	rtxPair     map[uint32]uint32 // repair -> base
 }
 
 func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser {
@@ -84,9 +86,25 @@ func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io
 		}
 		buffer := NewBuffer(ssrc, f.videoPool, f.audioPool)
 		f.rtpBuffers[ssrc] = buffer
+		for repair, base := range f.rtxPair {
+			if repair == ssrc {
+				baseBuffer, ok := f.rtpBuffers[base]
+				if ok {
+					buffer.SetPrimaryBufferForRTX(baseBuffer)
+				}
+				break
+			} else if base == ssrc {
+				repairBuffer, ok := f.rtpBuffers[repair]
+				if ok {
+					repairBuffer.SetPrimaryBufferForRTX(buffer)
+				}
+				break
+			}
+		}
 		buffer.OnClose(func() {
 			f.Lock()
 			delete(f.rtpBuffers, ssrc)
+			delete(f.rtxPair, ssrc)
 			f.Unlock()
 		})
 		return buffer
@@ -111,3 +129,15 @@ func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
 	defer f.RUnlock()
 	return f.rtcpReaders[ssrc]
 }
+
+func (f *Factory) SetRTXPair(repair, base uint32) {
+	f.Lock()
+	repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
+	if repairBuffer == nil || baseBuffer == nil {
+		f.rtxPair[repair] = base
+	}
+	f.Unlock()
+	if repairBuffer != nil && baseBuffer != nil {
+		repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
+	}
+}

pkg/sfu/buffer/frameintegrity.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package buffer
 
 import (

pkg/sfu/buffer/frameintegrity_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package buffer
 
 import (

pkg/sfu/buffer/rtpstats_receiver.go

@@ -271,6 +271,25 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData)
 
 	r.maybeAdjustFirstPacketTime(srDataCopy.RTPTimestamp, r.timestamp.GetStart())
 
+	if r.srNewest != nil && srDataCopy.RTPTimestampExt < r.srNewest.RTPTimestampExt {
+		// This can happen when a track is replaced with a null and then restored -
+		// i. e. muting replacing with null and unmute restoring the original track.
+		// Under such a condition reset the sender reports to start from this point.
+		// Resetting will ensure sample rate calculations do not go haywire due to negative time.
+		if r.outOfOrderSsenderReportCount%10 == 0 {
+			r.logger.Infow(
+				"received sender report, out-of-order, resetting",
+				"last", r.srNewest.ToString(),
+				"current", srDataCopy.ToString(),
+				"count", r.outOfOrderSsenderReportCount,
+			)
+		}
+		r.outOfOrderSsenderReportCount++
+
+		r.srFirst = nil
+		r.srNewest = nil
+	}
+
 	if r.srNewest != nil {
 		timeSinceLast := srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()).Seconds()
 		rtpDiffSinceLast := srDataCopy.RTPTimestampExt - r.srNewest.RTPTimestampExt
@@ -297,24 +316,6 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData)
 		}
 	}
 
-	if r.srNewest != nil && srDataCopy.RTPTimestampExt < r.srNewest.RTPTimestampExt {
-		// This can happen when a track is replaced with a null and then restored -
-		// i. e. muting replacing with null and unmute restoring the original track.
-		// Under such a condition reset the sender reports to start from this point.
-		// Resetting will ensure sample rate calculations do not go haywire due to negative time.
-		if r.outOfOrderSsenderReportCount%10 == 0 {
-			r.logger.Infow(
-				"received sender report, out-of-order, resetting",
-				"last", r.srNewest.ToString(),
-				"current", srDataCopy.ToString(),
-				"count", r.outOfOrderSsenderReportCount,
-			)
-		}
-		r.outOfOrderSsenderReportCount++
-
-		r.srFirst = nil
-	}
-
 	r.srNewest = &srDataCopy
 	if r.srFirst == nil {
 		r.srFirst = &srDataCopy

pkg/sfu/buffer/rtpstats_sender.go

@@ -160,6 +160,9 @@ type RTPStatsSender struct {
 	clockSkewCount              int
 	outOfOrderSenderReportCount int
 	metadataCacheOverflowCount  int
+
+	srFeedFirst  *RTCPSenderReportData
+	srFeedNewest *RTCPSenderReportData
 }
 
 func NewRTPStatsSender(params RTPStatsParams) *RTPStatsSender {
@@ -198,6 +201,15 @@ func (r *RTPStatsSender) Seed(from *RTPStatsSender) {
 	r.nextSenderSnapshotID = from.nextSenderSnapshotID
 	r.senderSnapshots = make([]senderSnapshot, cap(from.senderSnapshots))
 	copy(r.senderSnapshots, from.senderSnapshots)
+
+	if from.srFeedFirst != nil {
+		srFeedFirst := *from.srFeedFirst
+		r.srFeedFirst = &srFeedFirst
+	}
+	if from.srFeedNewest != nil {
+		srFeedNewest := *from.srFeedNewest
+		r.srFeedNewest = &srFeedNewest
+	}
 }
 
 func (r *RTPStatsSender) NewSnapshotId() uint32 {
@@ -571,10 +583,16 @@ func (r *RTPStatsSender) LastReceiverReportTime() time.Time {
 	return r.lastRRTime
 }
 
-func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(ts uint32) {
+func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData, ts uint32) {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
+	srFirstCopy := *srFirst
+	r.srFeedFirst = &srFirstCopy
+
+	srNewestCopy := *srNewest
+	r.srFeedNewest = &srNewestCopy
+
 	r.maybeAdjustFirstPacketTime(ts, uint32(r.extStartTS))
 }
 
@@ -637,8 +655,11 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui
 			if r.clockSkewCount%10 == 0 {
 				r.logger.Infow(
 					"sending sender report, clock skew",
+					"first", r.srFirst.ToString(),
 					"last", r.srNewest.ToString(),
 					"curr", srData.ToString(),
+					"firstFeed", r.srFeedFirst.ToString(),
+					"lastFeed", r.srFeedNewest.ToString(),
 					"timeNow", time.Now().String(),
 					"extStartTS", r.extStartTS,
 					"extHighestTS", r.extHighestTS,
@@ -674,8 +695,11 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui
 		if r.outOfOrderSenderReportCount%10 == 0 {
 			r.logger.Infow(
 				"sending sender report, out-of-order, repairing",
+				"first", r.srFirst.ToString(),
 				"last", r.srNewest.ToString(),
 				"curr", srData.ToString(),
+				"firstFeed", r.srFeedFirst.ToString(),
+				"lastFeed", r.srFeedNewest.ToString(),
 				"timeNow", time.Now().String(),
 				"extStartTS", r.extStartTS,
 				"extHighestTS", r.extHighestTS,

pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go

@@ -16,7 +16,8 @@ package dependencydescriptor
 
 import (
 	"fmt"
-	"reflect"
+
+	"golang.org/x/exp/slices"
 )
 
 type TemplateMatch struct {
@@ -126,26 +127,11 @@ func (w *DependencyDescriptorWriter) findBestTemplate() error {
 	return nil
 }
 
-// TODO: uncomment this when go 1.18 enabled
-// func sliceEqual[T comparable](a, b []T) bool {
-// 	if len(a) != len(b) {
-// 		return false
-// 	}
-
-// 	for i, v := range a {
-// 		if v != b[i] {
-// 			return false
-// 		}
-// 	}
-
-// 	return true
-// }
-
 func (w *DependencyDescriptorWriter) calculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch {
 	var result TemplateMatch
 	result.TemplateIdx = idx
-	result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !reflect.DeepEqual(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs)
-	result.NeedCustomDtis = w.descriptor.FrameDependencies.DecodeTargetIndications != nil && !reflect.DeepEqual(w.descriptor.FrameDependencies.DecodeTargetIndications, template.DecodeTargetIndications)
+	result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !slices.Equal(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs)
+	result.NeedCustomDtis = w.descriptor.FrameDependencies.DecodeTargetIndications != nil && !slices.Equal(w.descriptor.FrameDependencies.DecodeTargetIndications, template.DecodeTargetIndications)
 
 	for i := 0; i < w.structure.NumChains; i++ {
 		if w.activeChains&(1<<i) != 0 && (len(w.descriptor.FrameDependencies.ChainDiffs) <= i || len(template.ChainDiffs) <= i || w.descriptor.FrameDependencies.ChainDiffs[i] != template.ChainDiffs[i]) {

pkg/sfu/downtrack.go

@@ -38,6 +38,7 @@ import (
 	dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor"
 	"github.com/livekit/livekit-server/pkg/sfu/pacer"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpextension"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
 )
 
 // TrackSender defines an interface send media to remote peer
@@ -54,7 +55,13 @@ type TrackSender interface {
 	ID() string
 	SubscriberID() livekit.ParticipantID
 	TrackInfoAvailable()
-	HandleRTCPSenderReportData(payloadType webrtc.PayloadType, isSVC bool, layer int32, srData *buffer.RTCPSenderReportData) error
+	HandleRTCPSenderReportData(
+		payloadType webrtc.PayloadType,
+		isSVC bool,
+		layer int32,
+		srFirst *buffer.RTCPSenderReportData,
+		srNewest *buffer.RTCPSenderReportData,
+	) error
 }
 
 // -------------------------------------------------------------------
@@ -357,7 +364,7 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters,
 	}
 	var codec webrtc.RTPCodecParameters
 	for _, c := range d.upstreamCodecs {
-		matchCodec, err := codecParametersFuzzySearch(c, t.CodecParameters())
+		matchCodec, err := utils.CodecParametersFuzzySearch(c, t.CodecParameters())
 		if err == nil {
 			codec = matchCodec
 			break
@@ -1683,6 +1690,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) {
 				"eosn", epm.extSequenceNumber,
 				"eots", epm.extTimestamp,
 				"sid", epm.layer,
+				"error", err,
 			)
 			// TODO-VP9-DEBUG-REMOVE-END
 			nackMisses++
@@ -1961,9 +1969,19 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() {
 	}
 }
 
-func (d *DownTrack) HandleRTCPSenderReportData(_payloadType webrtc.PayloadType, isSVC bool, layer int32, srData *buffer.RTCPSenderReportData) error {
-	if (layer == d.forwarder.GetReferenceLayerSpatial() || (layer == 0 && isSVC)) && srData != nil {
-		d.rtpStats.MaybeAdjustFirstPacketTime(srData.RTPTimestamp + uint32(d.forwarder.GetReferenceTimestampOffset()))
+func (d *DownTrack) HandleRTCPSenderReportData(
+	_payloadType webrtc.PayloadType,
+	isSVC bool,
+	layer int32,
+	srFirst *buffer.RTCPSenderReportData,
+	srNewest *buffer.RTCPSenderReportData,
+) error {
+	if (layer == d.forwarder.GetReferenceLayerSpatial() || (layer == 0 && isSVC)) && srNewest != nil {
+		d.rtpStats.MaybeAdjustFirstPacketTime(
+			srFirst,
+			srNewest,
+			srNewest.RTPTimestamp+uint32(d.forwarder.GetReferenceTimestampOffset()),
+		)
 	}
 	return nil
 }

pkg/sfu/forwarder.go

@@ -1722,18 +1722,6 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
 			// call to update highest incoming sequence number and other internal structures
 			if tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, result.RTPMarker); err == nil {
 				if tpRTP.snOrdering == SequenceNumberOrderingContiguous {
-					// TODO-VP9-DEBUG-REMOVE-START
-					f.logger.Debugw(
-						"dropping packet",
-						"isn", extPkt.ExtSequenceNumber,
-						"its", extPkt.ExtTimestamp,
-						"osn", tpRTP.extSequenceNumber,
-						"ots", tpRTP.extTimestamp,
-						"payloadLen", len(extPkt.Packet.Payload),
-						"sid", extPkt.Spatial,
-						"tid", extPkt.Temporal,
-					)
-					// TODO-VP9-DEBUG-REMOVE-END
 					f.rtpMunger.PacketDropped(extPkt)
 				} else {
 					// TODO-VP9-DEBUG-REMOVE-START

pkg/sfu/interceptor/rtx.go

@@ -0,0 +1,182 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package interceptor
+
+import (
+	"sync"
+
+	"github.com/pion/interceptor"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v3"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	SDESRepairRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
+
+	rtxProbeCount = 10
+)
+
+type streamInfo struct {
+	mid  string
+	rid  string
+	rsid string
+}
+
+type RTXInfoExtractorFactory struct {
+	onStreamFound  func(*interceptor.StreamInfo)
+	onRTXPairFound func(repair, base uint32)
+	lock           sync.Mutex
+	streams        map[uint32]streamInfo
+	logger         logger.Logger
+}
+
+func NewRTXInfoExtractorFactory(onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32), logger logger.Logger) *RTXInfoExtractorFactory {
+	return &RTXInfoExtractorFactory{
+		onStreamFound:  onStreamFound,
+		onRTXPairFound: onRTXPairFound,
+		streams:        make(map[uint32]streamInfo),
+		logger:         logger,
+	}
+}
+
+func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
+	return &RTXInfoExtractor{
+		factory: f,
+		logger:  f.logger,
+	}, nil
+}
+
+func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid string) {
+	var repairSsrc, baseSsrc uint32
+	f.lock.Lock()
+
+	if rsid != "" {
+		// repair stream found, find base stream
+		for base, info := range f.streams {
+			if info.mid == mid && info.rid == rsid {
+				repairSsrc = ssrc
+				baseSsrc = base
+				delete(f.streams, base)
+				break
+			}
+		}
+	} else {
+		// base stream found, find repair stream
+		for repair, info := range f.streams {
+			if info.mid == mid && info.rsid == rid {
+				repairSsrc = repair
+				baseSsrc = ssrc
+				delete(f.streams, repair)
+				break
+			}
+		}
+	}
+
+	// no rtx pair found, save it for later
+	if repairSsrc == 0 || baseSsrc == 0 {
+		f.streams[ssrc] = streamInfo{
+			mid:  mid,
+			rid:  rid,
+			rsid: rsid,
+		}
+	}
+
+	f.lock.Unlock()
+
+	if repairSsrc != 0 && baseSsrc != 0 {
+		f.onRTXPairFound(repairSsrc, baseSsrc)
+	}
+}
+
+type RTXInfoExtractor struct {
+	interceptor.NoOp
+
+	factory *RTXInfoExtractorFactory
+	logger  logger.Logger
+}
+
+func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
+	u.factory.onStreamFound(info)
+
+	midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+	streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+	repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: SDESRepairRTPStreamIDURI})
+	if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
+		return reader
+	}
+
+	return &rtxInfoReader{
+		tryTimes:  rtxProbeCount,
+		reader:    reader,
+		midExtID:  uint8(midExtensionID),
+		ridExtID:  uint8(streamIDExtensionID),
+		rsidExtID: uint8(repairStreamIDExtensionID),
+		factory:   u.factory,
+		logger:    u.logger,
+	}
+}
+
+type rtxInfoReader struct {
+	tryTimes  int
+	reader    interceptor.RTPReader
+	midExtID  uint8
+	ridExtID  uint8
+	rsidExtID uint8
+	factory   *RTXInfoExtractorFactory
+	logger    logger.Logger
+}
+
+func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
+	n, a, err := r.reader.Read(b, a)
+	if r.tryTimes < 0 || err != nil {
+		return n, a, err
+	}
+
+	if a == nil {
+		a = make(interceptor.Attributes)
+	}
+	header, err := a.GetRTPHeader(b[:n])
+	if err != nil {
+		return n, a, nil
+	}
+
+	var mid, rid, rsid string
+	if payload := header.GetExtension(r.midExtID); payload != nil {
+		mid = string(payload)
+	}
+
+	if payload := header.GetExtension(r.ridExtID); payload != nil {
+		rid = string(payload)
+	}
+
+	if payload := header.GetExtension(r.rsidExtID); payload != nil {
+		rsid = string(payload)
+	}
+
+	if mid != "" && (rid != "" || rsid != "") {
+		r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC)
+		r.tryTimes = -1
+		go r.factory.setStreamInfo(header.SSRC, mid, rid, rsid)
+	} else {
+		// ignore padding only packet for probe count
+		if !(header.Padding && n-header.MarshalSize()-int(b[n-1]) == 0) {
+			r.tryTimes--
+		}
+	}
+	return n, a, nil
+}

pkg/sfu/playoutdelay.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package sfu
 
 import (

pkg/sfu/playoutdelay_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package sfu
 
 import (

pkg/sfu/receiver.go

@@ -42,6 +42,7 @@ var (
 	ErrReceiverClosed        = errors.New("receiver closed")
 	ErrDownTrackAlreadyExist = errors.New("DownTrack already exist")
 	ErrBufferNotFound        = errors.New("buffer not found")
+	ErrDuplicateLayer        = errors.New("duplicate layer")
 )
 
 type AudioLevelHandle func(level uint8, duration uint32)
@@ -108,17 +109,15 @@ type WebRTCReceiver struct {
 	useTrackers    bool
 	trackInfo      atomic.Pointer[livekit.TrackInfo]
 
-	rtcpCh chan []rtcp.Packet
+	onRTCP func([]rtcp.Packet)
 
 	twcc *twcc.Responder
 
 	bufferMu sync.RWMutex
 	buffers  [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
+	upTracks [buffer.DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote
 	rtt      uint32
 
-	upTrackMu sync.RWMutex
-	upTracks  [buffer.DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote
-
 	lbThreshold int
 
 	streamTrackerManager *StreamTrackerManager
@@ -197,7 +196,7 @@ func NewWebRTCReceiver(
 	track *webrtc.TrackRemote,
 	trackInfo *livekit.TrackInfo,
 	logger logger.Logger,
-	twcc *twcc.Responder,
+	onRTCP func([]rtcp.Packet),
 	trackersConfig config.StreamTrackersConfig,
 	opts ...ReceiverOpts,
 ) *WebRTCReceiver {
@@ -208,7 +207,7 @@ func NewWebRTCReceiver(
 		streamID: track.StreamID(),
 		codec:    track.Codec(),
 		kind:     track.Kind(),
-		twcc:     twcc,
+		onRTCP:   onRTCP,
 		isSVC:    IsSvcCodec(track.Codec().MimeType),
 		isRED:    IsRedCodec(track.Codec().MimeType),
 	}
@@ -265,9 +264,16 @@ func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.
 }
 
 func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) {
-	w.upTrackMu.Lock()
+	w.bufferMu.Lock()
 	w.onMaxLayerChange = fn
-	w.upTrackMu.Unlock()
+	w.bufferMu.Unlock()
+}
+
+func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) {
+	w.bufferMu.RLock()
+	defer w.bufferMu.RUnlock()
+
+	return w.onMaxLayerChange
 }
 
 func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
@@ -306,10 +312,7 @@ func (w *WebRTCReceiver) TrackID() livekit.TrackID {
 	return w.trackID
 }
 
-func (w *WebRTCReceiver) SSRC(layer int) uint32 {
-	w.upTrackMu.RLock()
-	defer w.upTrackMu.RUnlock()
-
+func (w *WebRTCReceiver) ssrc(layer int) uint32 {
 	if track := w.upTracks[layer]; track != nil {
 		return uint32(track.SSRC())
 	}
@@ -328,9 +331,9 @@ func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType {
 	return w.kind
 }
 
-func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buffer) {
+func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buffer) error {
 	if w.closed.Load() {
-		return
+		return ErrReceiverClosed
 	}
 
 	layer := int32(0)
@@ -338,7 +341,6 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
 		layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load())
 	}
 	buff.SetLogger(w.logger.WithValues("layer", layer))
-	buff.SetTWCC(w.twcc)
 	buff.SetAudioLevelParams(audio.AudioLevelParams{
 		ActiveLevel:     w.audioConfig.ActiveLevel,
 		MinPercentile:   w.audioConfig.MinPercentile,
@@ -351,7 +353,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
 		w.streamTrackerManager.SetRTCPSenderReportData(layer, srFirst, srNewest)
 
 		w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srNewest)
+			_ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srFirst, srNewest)
 		})
 	})
 
@@ -370,14 +372,16 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
 		buff.SetPLIThrottle(duration.Nanoseconds())
 	}
 
-	w.upTrackMu.Lock()
-	w.upTracks[layer] = track
-	w.upTrackMu.Unlock()
-
 	w.bufferMu.Lock()
+	if w.upTracks[layer] != nil {
+		w.bufferMu.Unlock()
+		return ErrDuplicateLayer
+	}
+	w.upTracks[layer] = track
 	w.buffers[layer] = buff
 	rtt := w.rtt
 	w.bufferMu.Unlock()
+
 	buff.SetRTT(rtt)
 	buff.SetPaused(w.streamTrackerManager.IsPaused())
 
@@ -386,6 +390,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff
 	}
 
 	go w.forwardRTP(layer)
+	return nil
 }
 
 // SetUpTrackPaused indicates upstream will not be sending any data.
@@ -472,11 +477,7 @@ func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int3
 
 // StreamTrackerManagerListener.OnMaxAvailableLayerChanged
 func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) {
-	w.upTrackMu.RLock()
-	onMaxLayerChange := w.onMaxLayerChange
-	w.upTrackMu.RUnlock()
-
-	if onMaxLayerChange != nil {
+	if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil {
 		onMaxLayerChange(maxAvailableLayer)
 	}
 }
@@ -514,10 +515,8 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
 		return
 	}
 
-	select {
-	case w.rtcpCh <- packets:
-	default:
-		w.logger.Warnw("sendRTCP failed, rtcp channel full", nil)
+	if w.onRTCP != nil {
+		w.onRTCP(packets)
 	}
 }
 
@@ -531,10 +530,6 @@ func (w *WebRTCReceiver) SendPLI(layer int32, force bool) {
 	buff.SendPLI(force)
 }
 
-func (w *WebRTCReceiver) SetRTCPCh(ch chan []rtcp.Packet) {
-	w.rtcpCh = ch
-}
-
 func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer {
 	w.bufferMu.RLock()
 	defer w.bufferMu.RUnlock()
@@ -626,7 +621,7 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer
 		patched[int32(layer)] = sswl.Layers[0]
 		sswl.Layers = patched
 
-		deltaStats[w.SSRC(layer)] = sswl
+		deltaStats[w.ssrc(layer)] = sswl
 	}
 
 	return deltaStats
@@ -718,7 +713,7 @@ func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
 		"Simulcast": isSimulcast,
 	}
 
-	w.upTrackMu.RLock()
+	w.bufferMu.RLock()
 	upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks))
 	for layer, ut := range w.upTracks {
 		if ut != nil {
@@ -730,7 +725,7 @@ func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
 			})
 		}
 	}
-	w.upTrackMu.RUnlock()
+	w.bufferMu.RUnlock()
 	info["UpTracks"] = upTrackInfo
 
 	return info

pkg/sfu/rtpextension/playoutdelay.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package rtpextension
 
 import (

pkg/sfu/rtpextension/playoutdelay_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package rtpextension
 
 import (

pkg/sfu/utils/helpers.go

@@ -12,17 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sfu
+package utils
 
 import (
 	"strings"
 
+	"github.com/pion/interceptor"
 	"github.com/pion/webrtc/v3"
 )
 
 // Do a fuzzy find for a codec in the list of codecs
 // Used for lookup up a codec in an existing list to find a match
-func codecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []webrtc.RTPCodecParameters) (webrtc.RTPCodecParameters, error) {
+func CodecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []webrtc.RTPCodecParameters) (webrtc.RTPCodecParameters, error) {
 	// First attempt to match on MimeType + SDPFmtpLine
 	for _, c := range haystack {
 		if strings.EqualFold(c.RTPCodecCapability.MimeType, needle.RTPCodecCapability.MimeType) &&
@@ -41,4 +42,12 @@ func codecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []web
 	return webrtc.RTPCodecParameters{}, webrtc.ErrCodecNotFound
 }
 
-// -----------------------------------------------
+// GetHeaderExtensionID returns the ID of a header extension, or 0 if not found
+func GetHeaderExtensionID(extensions []interceptor.RTPHeaderExtension, extension webrtc.RTPHeaderExtensionCapability) int {
+	for _, h := range extensions {
+		if extension.URI == h.URI {
+			return h.ID
+		}
+	}
+	return 0
+}

pkg/sfu/videolayerselector/framenumberwrapper.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package videolayerselector
 
 import "github.com/livekit/protocol/logger"

pkg/sfu/videolayerselector/framenumberwrapper_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package videolayerselector
 
 import (

pkg/telemetry/prometheus/packets.go

@@ -116,7 +116,9 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) {
 		Subsystem:   "jitter",
 		Name:        "us",
 		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env},
-		Buckets:     []float64{100, 500, 1500, 3000, 6000, 12000, 24000, 48000, 96000, 192000},
+
+		// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
+		Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
 	}, promStreamLabels)
 	promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   livekitNamespace,

pkg/telemetry/telemetryservice.go

@@ -24,6 +24,7 @@ import (
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/webhook"
+	"golang.org/x/exp/maps"
 )
 
 //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . TelemetryService
@@ -93,8 +94,9 @@ type telemetryService struct {
 	notifier  webhook.QueuedNotifier
 	jobsQueue *utils.OpsQueue
 
-	lock    sync.RWMutex
-	workers map[livekit.ParticipantID]*StatsWorker
+	lock          sync.RWMutex
+	workers       map[livekit.ParticipantID]*StatsWorker
+	workersShadow []*StatsWorker
 }
 
 func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService {
@@ -113,10 +115,11 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer
 }
 
 func (t *telemetryService) FlushStats() {
-	t.lock.Lock()
-	defer t.lock.Unlock()
+	t.lock.RLock()
+	workersShadow := t.workersShadow
+	t.lock.RUnlock()
 
-	for _, worker := range t.workers {
+	for _, worker := range workersShadow {
 		worker.Flush()
 	}
 }
@@ -167,21 +170,37 @@ func (t *telemetryService) createWorker(ctx context.Context,
 
 	t.lock.Lock()
 	t.workers[participantID] = worker
+	t.workersShadow = maps.Values(t.workers)
 	t.lock.Unlock()
 	return worker
 }
 
 func (t *telemetryService) cleanupWorkers() {
-	t.lock.Lock()
-	defer t.lock.Unlock()
+	t.lock.RLock()
+	workersShadow := t.workersShadow
+	t.lock.RUnlock()
 
-	for participantID, worker := range t.workers {
+	toReap := make([]livekit.ParticipantID, 0, len(workersShadow))
+	for _, worker := range workersShadow {
 		closedAt := worker.ClosedAt()
 		if !closedAt.IsZero() && time.Since(closedAt) > workerCleanupWait {
-			logger.Debugw("reaping analytics worker for participant", "pID", participantID)
-			delete(t.workers, participantID)
+			worker.Flush()
+
+			toReap = append(toReap, worker.ParticipantID())
 		}
 	}
+
+	if len(toReap) == 0 {
+		return
+	}
+
+	t.lock.Lock()
+	logger.Debugw("reaping analytics worker for participants", "pID", toReap)
+	for _, pID := range toReap {
+		delete(t.workers, pID)
+	}
+	t.workersShadow = maps.Values(t.workers)
+	t.lock.Unlock()
 }
 
 func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {

pkg/utils/opsqueue.go

@@ -41,7 +41,7 @@ func NewOpsQueue(name string, minSize uint, flushOnStop bool) *OpsQueue {
 		wake:        make(chan struct{}, 1),
 		doneChan:    make(chan struct{}),
 	}
-	oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(minSize-1)), 16)))
+	oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(minSize-1)), 7)))
 	return oq
 }
 
@@ -75,8 +75,12 @@ func (oq *OpsQueue) Enqueue(op func()) {
 	oq.lock.Lock()
 	defer oq.lock.Unlock()
 
+	if oq.isStopped {
+		return
+	}
+
 	oq.ops.PushBack(op)
-	if oq.ops.Len() == 1 && !oq.isStopped {
+	if oq.ops.Len() == 1 {
 		select {
 		case oq.wake <- struct{}{}:
 		default:

test/agent.go

@@ -1,3 +1,17 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package test
 
 import (

version/version.go

@@ -14,4 +14,4 @@
 
 package version
 
-const Version = "1.5.2"
+const Version = "1.5.3"