dandri/livekit
1
0
Fork 0
mirror of https://github.com/livekit/livekit.git synced 2026-03-29 20:09:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

LK-105 (Opus DTX) (#140)

Browse Source 
* LK-105 (Opus DTX)

https://linear.app/livekit/issue/LK-105/allow-enabling-of-opus-dtx

Enable/Disable Opus DTX using SDP answer based on setting in
`AddTrack` request.

Testing:
--------
Chrome and Firefox work. Having audio problems with Safari
(maybe the Safari 15 issue as I am not getting media)

* Check that receiver has no tracks

* Skip non-audio transceivers

* A small clean up to not use pendin track outside lock and also append with spread

* Address comments from review by David

* Update pkg/rtc/participant.go

Co-authored-by: David Zhao <david@davidzhao.com>

* Pull in tagged version of webrtc and lk protocol

Co-authored-by: David Zhao <david@davidzhao.com>
This commit is contained in:
Raja Subramanian
2021-10-13 11:22:33 +05:30
committed by GitHub
parent 84ab0f82af
commit ac4db4575f
4 changed files with 120 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@@ -14,7 +14,7 @@ require (
github.com/google/wire v0.5.0
github.com/gorilla/websocket v1.4.2
github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b
github.com/livekit/protocol v0.9.7
github.com/livekit/protocol v0.9.8
github.com/magefile/mage v1.11.0
github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0
github.com/mitchellh/go-homedir v1.1.0
@@ -29,7 +29,7 @@ require (
github.com/pion/stun v0.3.5
github.com/pion/transport v0.12.3
github.com/pion/turn/v2 v2.0.5
github.com/pion/webrtc/v3 v3.1.1
github.com/pion/webrtc/v3 v3.1.3
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0
github.com/prometheus/common v0.19.0 // indirect

16
go.sum
View File

@@ -246,8 +246,8 @@ github.com/lithammer/shortuuid/v3 v3.0.6 h1:pr15YQyvhiSX/qPxncFtqk+v4xLEpOZObbsY
github.com/lithammer/shortuuid/v3 v3.0.6/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/livekit/ion-sfu v1.20.12 h1:9u7KzTWvUgNUbPRaYDXxWHchuc7MuNGlRqGu9OAkqj4=
github.com/livekit/ion-sfu v1.20.12/go.mod h1:IVcCb8yMl5qHq+8InP5v9HgrjgnPZny8GhFS86YOtug=
github.com/livekit/protocol v0.9.7 h1:GrDvTJTD8wFeoPk6XsLUB/C6pz0RRwa6EVr1trWvPAY=
github.com/livekit/protocol v0.9.7/go.mod h1:MEKn847Iu/2U8ClZyUmEm2oHn8k9fnSHy81Wv8kkSDo=
github.com/livekit/protocol v0.9.8 h1:FJEUTyXTa8Q0+EaKwyL8XAeNgW4xQRKzik00DJgwJhg=
github.com/livekit/protocol v0.9.8/go.mod h1:MEKn847Iu/2U8ClZyUmEm2oHn8k9fnSHy81Wv8kkSDo=
github.com/lucsky/cuid v1.0.2 h1:z4XlExeoderxoPj2/dxKOyPxe9RCOu7yNq9/XWxIUMQ=
github.com/lucsky/cuid v1.0.2/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
@@ -329,8 +329,9 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8=
github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
github.com/pion/dtls/v2 v2.0.10 h1:wgys7gPR1NMbWjmjJ3CW7lkUGaun8djgH8nahpNLnxI=
github.com/pion/dtls/v2 v2.0.10/go.mod h1:00OxfeCRWHShcqT9jx8pKKmBWuTt0NCZoVPCaC4VKvU=
github.com/pion/ice/v2 v2.1.12 h1:ZDBuZz+fEI7iDifZCYFVzI4p0Foy0YhdSSZ87ZtRcRE=
github.com/pion/ice/v2 v2.1.12/go.mod h1:ovgYHUmwYLlRvcCLI67PnQ5YGe+upXZbGgllBDG/ktU=
github.com/pion/interceptor v0.1.0 h1:SlXKaDlEvSl7cr4j8fJykzVz4UdH+7UDtcvx+u01wLU=
@@ -364,8 +365,9 @@ github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
github.com/pion/webrtc/v3 v3.1.1 h1:UA4qU+zoAAlUkl0J7ek/YnV3e0orvE8V667imvmnAoo=
github.com/pion/webrtc/v3 v3.1.1/go.mod h1:t51XSam1k56eYLuO1Ubxjs3pDBfGYxkGBFhYf55Mn/s=
github.com/pion/webrtc/v3 v3.1.3 h1:jtUDBUz3HMVdyoR7sNPUsHPk1zyFg1J2hYAiQcACQmw=
github.com/pion/webrtc/v3 v3.1.3/go.mod h1:fMlPlz03ACPIkjqNq3CGbLaZjVv523u+fZgWfQ8yg6E=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -509,8 +511,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -566,8 +569,9 @@ golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50=
golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

1
magefile.go
View File

@@ -1,3 +1,4 @@
//go:build mage
// +build mage
package main

108
pkg/rtc/participant.go
View File

@@ -2,6 +2,7 @@ package rtc
import (
"fmt"
"strings"
"io"
"sync"
"sync/atomic"
@@ -281,6 +282,8 @@ func (p *ParticipantImpl) HandleOffer(sdp webrtc.SessionDescription) (answer web
return
}
p.configureReceiverDTX()
answer, err = p.publisher.pc.CreateAnswer(nil)
if err != nil {
stats.PromServiceOperationCounter.WithLabelValues("answer", "error", "create").Add(1)
@@ -296,7 +299,7 @@ func (p *ParticipantImpl) HandleOffer(sdp webrtc.SessionDescription) (answer web
logger.Debugw("sending answer to client",
"participant", p.Identity(), "pID", p.ID(),
//"sdp", sdp.SDP,
//"answer sdp", answer.SDP,
)
err = p.writeMessage(&livekit.SignalResponse{
Message: &livekit.SignalResponse_Answer{
@@ -340,6 +343,7 @@ func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) {
Width: req.Width,
Height: req.Height,
Muted: req.Muted,
DisableDtx: req.DisableDtx,
}
p.pendingTracks[req.Cid] = ti
@@ -1073,6 +1077,108 @@ func (p *ParticipantImpl) rtcpSendWorker() {
}
}
func (p *ParticipantImpl) configureReceiverDTX() {
//
// DTX (Discontinuous Transmission) allows audio bandwidth saving
// by not sending packets during silence periods.
//
// Publisher side DTX can enabled by included `usedtx=1` in
// the `fmtp` line corresponding to audio codec (Opus) in SDP.
// By doing this in the SDP `answer`, it can be controlled from
// server side and avoid doing it in all the client SDKs.
//
// Ideally, a publisher should be able to specify per audio
// track if DTX should be enabled. But, translating the
// DTX preference of publisher to the correct transceiver
// is non-deterministic due to the lack of a synchronizing id
// like the track id. The codec preference to set DTX needs
// to be done
// - after calling `SetRemoteDescription` which sets up
// the transceivers, but there are no tracks in the
// transceiver yet
// - before calling `CreateAnswer`
// Due to the absensce of tracks when it is required to set DTX,
// it is not possible to cross reference against a pending track
// with the same track id.
//
// Due to the restriction above and given that in practice
// most of the time there is going to be only one audio track
// that is published, do the following
// - if there is no pending audio track, no-op
// - if there are no audio transceivers without tracks, no-op
// - else, apply the DTX setting from pending audio track
// to the audio transceiver without no tracks
//
// NOTE: The above logic will fail if there is an `offer` SDP with
// multiple audio tracks. At that point, there might be a need to
// rely on something like order of tracks. TODO
//
enableDTX := false
p.lock.RLock()
var pendingTrack *livekit.TrackInfo
for _, track := range p.pendingTracks {
if track.Type == livekit.TrackType_AUDIO {
pendingTrack = track
break
}
}
if pendingTrack == nil {
p.lock.RUnlock()
return
}
enableDTX = !pendingTrack.DisableDtx
p.lock.RUnlock()
transceivers := p.publisher.pc.GetTransceivers()
for _, transceiver := range transceivers {
if transceiver.Kind() != webrtc.RTPCodecTypeAudio {
continue
}
receiver := transceiver.Receiver()
if receiver == nil || receiver.Track() != nil {
continue
}
modifiedReceiverCodecs := []webrtc.RTPCodecParameters{}
receiverCodecs := receiver.GetParameters().Codecs
for _, receiverCodec := range receiverCodecs {
if receiverCodec.MimeType == webrtc.MimeTypeOpus {
fmtpUseDTX := "usedtx=1"
// remove occurrence in the middle
sdpFmtpLine := strings.ReplaceAll(receiverCodec.SDPFmtpLine, fmtpUseDTX + ";", "")
// remove occurrence at the end
sdpFmtpLine = strings.ReplaceAll(sdpFmtpLine, fmtpUseDTX, "")
if enableDTX {
sdpFmtpLine += ";" + fmtpUseDTX
}
receiverCodec.SDPFmtpLine = sdpFmtpLine
}
modifiedReceiverCodecs = append(modifiedReceiverCodecs, receiverCodec)
}
//
// As `SetCodecPreferences` on a transceiver replaces all codecs,
// cycle through sender codecs also and add them before calling
// `SetCodecPreferences`
//
senderCodecs := []webrtc.RTPCodecParameters{}
sender := transceiver.Sender()
if sender != nil {
senderCodecs = sender.GetParameters().Codecs
}
err := transceiver.SetCodecPreferences(append(modifiedReceiverCodecs, senderCodecs...))
if err != nil {
logger.Warnw("failed to SetCodecPreferences", err)
}
}
}
func (p *ParticipantImpl) DebugInfo() map[string]interface{} {
info := map[string]interface{}{
"ID": p.id,