From 1e1aaeb86bcfd6b3ebaf84a49968dfd0a4754d0a Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 9 Nov 2021 12:03:16 +0800 Subject: [PATCH] Separate from ion-sfu (#171) * Separate from ion-sfu changes: 1. extract pkg/buffer, twcc, sfu, relay, stats, logger 2. to solve cycle import, move ion-sfu/pkg/logger to pkg/sfu/logger 3. replace pion/ion-sfu => ./ reason: will change import pion/ion-sfu/pkg/* to livekit-server/pkg/* after this pr merged. Just not change any code in this pr, because it will confused with the separate code from ion-sfu in review. * Move code from ion-sfu to pkg/sfu * fix build error for resovle conflict Co-authored-by: cnderrauber --- go.mod | 15 +- go.sum | 388 +--- pkg/logger/logger.go | 4 +- pkg/rtc/config.go | 2 +- pkg/rtc/mediatrack.go | 6 +- pkg/rtc/participant.go | 4 +- pkg/rtc/plithrottle.go | 2 +- pkg/rtc/room.go | 2 +- pkg/rtc/subscribedtrack.go | 2 +- pkg/rtc/types/interfaces.go | 2 +- .../types/typesfakes/fake_subscribed_track.go | 2 +- pkg/rtc/wrappedreceiver.go | 2 +- pkg/sfu/atomic.go | 74 + pkg/sfu/audioobserver.go | 112 ++ pkg/sfu/audioobserver_test.go | 271 +++ pkg/sfu/buffer/bucket.go | 114 ++ pkg/sfu/buffer/bucket_test.go | 139 ++ pkg/sfu/buffer/buffer.go | 715 +++++++ pkg/sfu/buffer/buffer_test.go | 363 ++++ pkg/sfu/buffer/errors.go | 10 + pkg/sfu/buffer/factory.go | 97 + pkg/sfu/buffer/helpers.go | 297 +++ pkg/sfu/buffer/helpers_test.go | 94 + pkg/sfu/buffer/nack.go | 106 ++ pkg/sfu/buffer/nack_test.go | 196 ++ pkg/sfu/buffer/rtcpreader.go | 44 + pkg/sfu/datachannel.go | 81 + pkg/sfu/downtrack.go | 1668 +++++++++++++++++ pkg/sfu/errors.go | 18 + pkg/sfu/helpers.go | 131 ++ pkg/sfu/helpers_test.go | 34 + pkg/sfu/logger/zerologr.go | 133 ++ pkg/sfu/logger/zerologr_test.go | 144 ++ pkg/sfu/mediaengine.go | 92 + pkg/sfu/peer.go | 311 +++ pkg/sfu/prober.go | 352 ++++ pkg/sfu/publisher.go | 448 +++++ pkg/sfu/receiver.go | 708 +++++++ pkg/sfu/receiver_test.go | 199 ++ pkg/sfu/relay.go | 13 + pkg/sfu/relay/README.md | 81 + pkg/sfu/relay/relay.go | 657 +++++++ pkg/sfu/relaypeer.go | 205 ++ pkg/sfu/router.go | 347 ++++ pkg/sfu/sequencer.go | 161 ++ pkg/sfu/sequencer_test.go | 99 + pkg/sfu/session.go | 414 ++++ pkg/sfu/sfu.go | 278 +++ pkg/sfu/sfu_test.go | 360 ++++ pkg/sfu/simulcast.go | 29 + pkg/sfu/stats/stream.go | 174 ++ pkg/sfu/streamallocator.go | 1051 +++++++++++ pkg/sfu/streamtracker.go | 147 ++ pkg/sfu/streamtracker_test.go | 45 + pkg/sfu/subscriber.go | 322 ++++ pkg/sfu/turn.go | 170 ++ pkg/sfu/twcc/twcc.go | 313 ++++ pkg/sfu/twcc/twcc_test.go | 346 ++++ 58 files changed, 12210 insertions(+), 384 deletions(-) create mode 100644 pkg/sfu/atomic.go create mode 100644 pkg/sfu/audioobserver.go create mode 100644 pkg/sfu/audioobserver_test.go create mode 100644 pkg/sfu/buffer/bucket.go create mode 100644 pkg/sfu/buffer/bucket_test.go create mode 100644 pkg/sfu/buffer/buffer.go create mode 100644 pkg/sfu/buffer/buffer_test.go create mode 100644 pkg/sfu/buffer/errors.go create mode 100644 pkg/sfu/buffer/factory.go create mode 100644 pkg/sfu/buffer/helpers.go create mode 100644 pkg/sfu/buffer/helpers_test.go create mode 100644 pkg/sfu/buffer/nack.go create mode 100644 pkg/sfu/buffer/nack_test.go create mode 100644 pkg/sfu/buffer/rtcpreader.go create mode 100644 pkg/sfu/datachannel.go create mode 100644 pkg/sfu/downtrack.go create mode 100644 pkg/sfu/errors.go create mode 100644 pkg/sfu/helpers.go create mode 100644 pkg/sfu/helpers_test.go create mode 100644 pkg/sfu/logger/zerologr.go create mode 100644 pkg/sfu/logger/zerologr_test.go create mode 100644 pkg/sfu/mediaengine.go create mode 100644 pkg/sfu/peer.go create mode 100644 pkg/sfu/prober.go create mode 100644 pkg/sfu/publisher.go create mode 100644 pkg/sfu/receiver.go create mode 100644 pkg/sfu/receiver_test.go create mode 100644 pkg/sfu/relay.go create mode 100644 pkg/sfu/relay/README.md create mode 100644 pkg/sfu/relay/relay.go create mode 100644 pkg/sfu/relaypeer.go create mode 100644 pkg/sfu/router.go create mode 100644 pkg/sfu/sequencer.go create mode 100644 pkg/sfu/sequencer_test.go create mode 100644 pkg/sfu/session.go create mode 100644 pkg/sfu/sfu.go create mode 100644 pkg/sfu/sfu_test.go create mode 100644 pkg/sfu/simulcast.go create mode 100644 pkg/sfu/stats/stream.go create mode 100644 pkg/sfu/streamallocator.go create mode 100644 pkg/sfu/streamtracker.go create mode 100644 pkg/sfu/streamtracker_test.go create mode 100644 pkg/sfu/subscriber.go create mode 100644 pkg/sfu/turn.go create mode 100644 pkg/sfu/twcc/twcc.go create mode 100644 pkg/sfu/twcc/twcc_test.go diff --git a/go.mod b/go.mod index 096bd4c10..b93292eb4 100644 --- a/go.mod +++ b/go.mod @@ -5,44 +5,43 @@ go 1.15 require ( github.com/bep/debounce v1.2.0 github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 - github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/gammazero/deque v0.1.0 github.com/gammazero/workerpool v1.1.2 github.com/go-logr/logr v1.2.0 github.com/go-logr/zapr v1.1.0 + github.com/go-logr/zerologr v1.2.0 github.com/go-redis/redis/v8 v8.11.3 - github.com/google/subcommands v1.2.0 // indirect github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.4.2 github.com/hashicorp/golang-lru v0.5.4 github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b github.com/livekit/protocol v0.10.1-0.20211109000312-b3847c8d35ff + github.com/lucsky/cuid v1.2.1 github.com/magefile/mage v1.11.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/pion/dtls/v2 v2.0.10 github.com/pion/ice/v2 v2.1.12 github.com/pion/interceptor v0.1.0 - github.com/pion/ion-sfu v1.10.5 github.com/pion/logging v0.2.2 github.com/pion/rtcp v1.2.8 github.com/pion/rtp v1.7.2 github.com/pion/sdp/v3 v3.0.4 github.com/pion/stun v0.3.5 + github.com/pion/transport v0.12.3 github.com/pion/turn/v2 v2.0.5 github.com/pion/webrtc/v3 v3.1.5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 - github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/rs/zerolog v1.25.0 github.com/stretchr/testify v1.7.0 github.com/thoas/go-funk v0.8.0 github.com/twitchtv/twirp v8.1.0+incompatible github.com/urfave/cli/v2 v2.3.0 github.com/urfave/negroni v1.0.0 go.uber.org/zap v1.19.1 - golang.org/x/mod v0.5.1 // indirect - golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect + google.golang.org/grpc v1.42.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) - -replace github.com/pion/ion-sfu => github.com/livekit/ion-sfu v1.20.16 diff --git a/go.sum b/go.sum index 061b5b9bd..3039f54db 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -41,61 +15,36 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK84APFuMvxqsk3tEIaKH/z4Rpu3g= github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -104,10 +53,8 @@ github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2K github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -121,21 +68,12 @@ github.com/go-logr/zerologr v1.2.0 h1:oS1fjSSEHwpv8Lam3SNmPTLTUw6V4DoB2ZzryqrkMB github.com/go-logr/zerologr v1.2.0/go.mod h1:O82obOiXzyxiBNgAMRT1m+XbOvY8K18Kf6XhT52oqoc= github.com/go-redis/redis/v8 v8.11.3 h1:GCjoYp8c+yQTJfc0n69iwSiHjvuAdruxl7elnZCxgt8= github.com/go-redis/redis/v8 v8.11.3/go.mod h1:xNJ9xDG09FsIPwh3bWdk+0oDWHbtF9rPN0F/oD9XeKc= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -151,9 +89,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -164,81 +99,28 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b h1:XUr8tvMEILhphQPp3TFcIudb5KTOzFeD0pJyDn5+5QI= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b/go.mod h1:a5Mn24iYVJRUQSkFupGByqykzD+k+wFI8J91zGHuPf8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -246,93 +128,46 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lithammer/shortuuid/v3 v3.0.6 h1:pr15YQyvhiSX/qPxncFtqk+v4xLEpOZObbsY/mKrcvA= github.com/lithammer/shortuuid/v3 v3.0.6/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= -github.com/livekit/ion-sfu v1.20.16 h1:B4+z0sf4t3zZSXFIwHive8malNn6Vje+7b1OW4ETDOM= -github.com/livekit/ion-sfu v1.20.16/go.mod h1:sUjL3tZRROs3NjCm6ZLT+IsisdYVRtxfq4OhVFHVd/A= github.com/livekit/protocol v0.10.1-0.20211109000312-b3847c8d35ff h1:21SZ2sh5e7ELCVdXT01hlpdSZyNlwDv6KTOlcplBrQ8= github.com/livekit/protocol v0.10.1-0.20211109000312-b3847c8d35ff/go.mod h1:7ir9zSlgnrPQoGGNv4f8U/c9QrWh+ogC9B5xVbJNedM= -github.com/lucsky/cuid v1.0.2 h1:z4XlExeoderxoPj2/dxKOyPxe9RCOu7yNq9/XWxIUMQ= -github.com/lucsky/cuid v1.0.2/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/lucsky/cuid v1.2.1 h1:MtJrL2OFhvYufUIn48d35QGXyeTC8tn0upumW9WwTHg= +github.com/lucsky/cuid v1.2.1/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE= github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0 h1:8E6DrFvII6QR4eJ3PkFvV+lc03P+2qwqTPLm1ax7694= github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0/go.mod h1:fcEyUyXZXoV4Abw8DX0t7wyL8mCDxXyU4iAFZfT3IHw= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0= github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg= github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho= @@ -377,82 +212,41 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.25.0 h1:Rj7XygbUHKUlDPcVdoLyR91fJBsduXj5fRxyqIQj/II= github.com/rs/zerolog v1.25.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI= -github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/sourcegraph/jsonrpc2 v0.0.0-20210201082850-366fbb520750/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -462,56 +256,29 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.8.0 h1:JP9tKSvnpFVclYgDM0Is7FD9M4fhPvqA0s0BsXmzSRQ= github.com/thoas/go-funk v0.8.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twitchtv/twirp v8.1.0+incompatible h1:KGXanpa9LXdVE/V5P/tA27rkKFmXRGCtSNT7zdeeVOY= github.com/twitchtv/twirp v8.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -519,49 +286,27 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -577,142 +322,77 @@ golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5R golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc/examples v0.0.0-20201209011439-fd32f6a4fefe/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -721,7 +401,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -733,19 +412,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -758,12 +429,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 2635ed910..223b08e0e 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -3,9 +3,9 @@ package serverlogger import ( "github.com/go-logr/logr" "github.com/go-logr/zapr" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/logger" - "github.com/pion/ion-sfu/pkg/buffer" - "github.com/pion/ion-sfu/pkg/sfu" "github.com/pion/logging" "go.uber.org/zap" "go.uber.org/zap/zapcore" diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index 64b08cbf3..ab93b99ae 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -4,8 +4,8 @@ import ( "errors" "net" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/pion/ice/v2" - "github.com/pion/ion-sfu/pkg/buffer" "github.com/pion/webrtc/v3" "github.com/livekit/livekit-server/pkg/config" diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 8404da146..f08e2fb96 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -6,12 +6,12 @@ import ( "sync/atomic" "time" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/twcc" "github.com/livekit/protocol/logger" livekit "github.com/livekit/protocol/proto" "github.com/livekit/protocol/utils" - "github.com/pion/ion-sfu/pkg/buffer" - "github.com/pion/ion-sfu/pkg/sfu" - "github.com/pion/ion-sfu/pkg/twcc" "github.com/pion/rtcp" "github.com/pion/webrtc/v3" "github.com/pion/webrtc/v3/pkg/rtcerr" diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 475e09b48..40bb2e84a 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -9,11 +9,11 @@ import ( "time" lru "github.com/hashicorp/golang-lru" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/twcc" "github.com/livekit/protocol/logger" livekit "github.com/livekit/protocol/proto" "github.com/livekit/protocol/utils" - "github.com/pion/ion-sfu/pkg/sfu" - "github.com/pion/ion-sfu/pkg/twcc" "github.com/pion/rtcp" "github.com/pion/webrtc/v3" "github.com/pkg/errors" diff --git a/pkg/rtc/plithrottle.go b/pkg/rtc/plithrottle.go index f97d815d6..db6105135 100644 --- a/pkg/rtc/plithrottle.go +++ b/pkg/rtc/plithrottle.go @@ -14,7 +14,7 @@ type pliThrottle struct { lastSent map[uint32]int64 } -// github.com/pion/ion-sfu/pkg/sfu/simulcast.go +// github.com/livekit/livekit-server/pkg/sfu/simulcast.go const ( fullResolution = "f" halfResolution = "h" diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index b72e96bf1..f179442c0 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -8,9 +8,9 @@ import ( "time" "github.com/go-logr/logr" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/logger" livekit "github.com/livekit/protocol/proto" - "github.com/pion/ion-sfu/pkg/buffer" "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 345902a99..4e562624e 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -4,9 +4,9 @@ import ( "time" "github.com/bep/debounce" + "github.com/livekit/livekit-server/pkg/sfu" livekit "github.com/livekit/protocol/proto" "github.com/livekit/protocol/utils" - "github.com/pion/ion-sfu/pkg/sfu" "github.com/pion/webrtc/v3" ) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 56bda3f93..caa8d6ec9 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -3,7 +3,7 @@ package types import ( "time" - "github.com/pion/ion-sfu/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu" "github.com/pion/rtcp" "github.com/pion/webrtc/v3" diff --git a/pkg/rtc/types/typesfakes/fake_subscribed_track.go b/pkg/rtc/types/typesfakes/fake_subscribed_track.go index 77ef8a2d4..7001b610a 100644 --- a/pkg/rtc/types/typesfakes/fake_subscribed_track.go +++ b/pkg/rtc/types/typesfakes/fake_subscribed_track.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/sfu" livekit "github.com/livekit/protocol/proto" - "github.com/pion/ion-sfu/pkg/sfu" ) type FakeSubscribedTrack struct { diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 2fddcf6b7..6ffed6e1e 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -1,7 +1,7 @@ package rtc import ( - "github.com/pion/ion-sfu/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu" ) // wrapper around WebRTC receiver, overriding its ID diff --git a/pkg/sfu/atomic.go b/pkg/sfu/atomic.go new file mode 100644 index 000000000..058a5d86e --- /dev/null +++ b/pkg/sfu/atomic.go @@ -0,0 +1,74 @@ +package sfu + +import "sync/atomic" + +type atomicBool int32 + +func (a *atomicBool) set(value bool) (swapped bool) { + if value { + return atomic.SwapInt32((*int32)(a), 1) == 0 + } + return atomic.SwapInt32((*int32)(a), 0) == 1 +} + +func (a *atomicBool) get() bool { + return atomic.LoadInt32((*int32)(a)) != 0 +} + +type atomicUint8 uint32 + +func (a *atomicUint8) set(value uint8) { + atomic.StoreUint32((*uint32)(a), uint32(value)) +} + +func (a *atomicUint8) get() uint8 { + return uint8(atomic.LoadUint32((*uint32)(a))) +} + +type atomicUint16 uint32 + +func (a *atomicUint16) set(value uint16) { + atomic.StoreUint32((*uint32)(a), uint32(value)) +} + +func (a *atomicUint16) get() uint16 { + return uint16(atomic.LoadUint32((*uint32)(a))) +} + +func (a *atomicUint16) add(value uint16) { + atomic.AddUint32((*uint32)(a), uint32(value)) +} + +type atomicInt32 int32 + +func (a *atomicInt32) set(value int32) { + atomic.StoreInt32((*int32)(a), value) +} + +func (a *atomicInt32) get() int32 { + return atomic.LoadInt32((*int32)(a)) +} + +type atomicUint32 uint32 + +func (a *atomicUint32) set(value uint32) { + atomic.StoreUint32((*uint32)(a), value) +} + +func (a *atomicUint32) add(value uint32) { + atomic.AddUint32((*uint32)(a), value) +} + +func (a *atomicUint32) get() uint32 { + return atomic.LoadUint32((*uint32)(a)) +} + +type atomicInt64 int64 + +func (a *atomicInt64) set(value int64) { + atomic.StoreInt64((*int64)(a), value) +} + +func (a *atomicInt64) get() int64 { + return atomic.LoadInt64((*int64)(a)) +} diff --git a/pkg/sfu/audioobserver.go b/pkg/sfu/audioobserver.go new file mode 100644 index 000000000..4291e1ff7 --- /dev/null +++ b/pkg/sfu/audioobserver.go @@ -0,0 +1,112 @@ +package sfu + +import ( + "sort" + "sync" +) + +type audioStream struct { + id string + sum int + total int +} + +type AudioObserver struct { + sync.RWMutex + streams []*audioStream + expected int + threshold uint8 + previous []string +} + +func NewAudioObserver(threshold uint8, interval, filter int) *AudioObserver { + if threshold > 127 { + threshold = 127 + } + if filter < 0 { + filter = 0 + } + if filter > 100 { + filter = 100 + } + + return &AudioObserver{ + threshold: threshold, + expected: interval * filter / 2000, + } +} + +func (a *AudioObserver) addStream(streamID string) { + a.Lock() + a.streams = append(a.streams, &audioStream{id: streamID}) + a.Unlock() +} + +func (a *AudioObserver) removeStream(streamID string) { + a.Lock() + defer a.Unlock() + idx := -1 + for i, s := range a.streams { + if s.id == streamID { + idx = i + break + } + } + if idx == -1 { + return + } + a.streams[idx] = a.streams[len(a.streams)-1] + a.streams[len(a.streams)-1] = nil + a.streams = a.streams[:len(a.streams)-1] +} + +func (a *AudioObserver) observe(streamID string, dBov uint8) { + a.RLock() + defer a.RUnlock() + for _, as := range a.streams { + if as.id == streamID { + if dBov <= a.threshold { + as.sum += int(dBov) + as.total++ + } + return + } + } +} + +func (a *AudioObserver) Calc() []string { + a.Lock() + defer a.Unlock() + + sort.Slice(a.streams, func(i, j int) bool { + si, sj := a.streams[i], a.streams[j] + switch { + case si.total != sj.total: + return si.total > sj.total + default: + return si.sum < sj.sum + } + }) + + streamIDs := make([]string, 0, len(a.streams)) + for _, s := range a.streams { + if s.total >= a.expected { + streamIDs = append(streamIDs, s.id) + } + s.total = 0 + s.sum = 0 + } + + if len(a.previous) == len(streamIDs) { + for i, s := range a.previous { + if s != streamIDs[i] { + a.previous = streamIDs + return streamIDs + } + } + return nil + } + + a.previous = streamIDs + return streamIDs +} diff --git a/pkg/sfu/audioobserver_test.go b/pkg/sfu/audioobserver_test.go new file mode 100644 index 000000000..53ae0b4b5 --- /dev/null +++ b/pkg/sfu/audioobserver_test.go @@ -0,0 +1,271 @@ +package sfu + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_audioLevel_addStream(t *testing.T) { + type args struct { + streamID string + } + tests := []struct { + name string + args args + want audioStream + }{ + { + name: "Must add stream to audio level monitor", + args: args{ + streamID: "a", + }, + want: audioStream{ + id: "a", + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + a := &AudioObserver{} + a.addStream(tt.args.streamID) + assert.Equal(t, tt.want, *a.streams[0]) + + }) + } +} + +func Test_audioLevel_calc(t *testing.T) { + type fields struct { + streams []*audioStream + expected int + previous []string + } + tests := []struct { + name string + fields fields + want []string + }{ + { + name: "Must return streams that are above filter", + fields: fields{ + streams: []*audioStream{ + { + id: "a", + sum: 1, + total: 5, + }, + { + id: "b", + sum: 2, + total: 5, + }, + { + id: "c", + sum: 2, + total: 2, + }, + }, + expected: 3, + }, + want: []string{"a", "b"}, + }, + { + name: "Must return nil if result is same as previous", + fields: fields{ + streams: []*audioStream{ + { + id: "a", + sum: 1, + total: 5, + }, + { + id: "b", + sum: 2, + total: 5, + }, + { + id: "c", + sum: 2, + total: 2, + }, + }, + expected: 3, + previous: []string{"a", "b"}, + }, + want: nil, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + a := &AudioObserver{ + streams: tt.fields.streams, + expected: tt.fields.expected, + previous: tt.fields.previous, + } + if got := a.Calc(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Calc() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_audioLevel_observe(t *testing.T) { + type fields struct { + streams []*audioStream + threshold uint8 + } + type args struct { + streamID string + dBov uint8 + } + tests := []struct { + name string + fields fields + args args + want audioStream + }{ + { + name: "Must increase sum and total when dBov is above threshold", + fields: fields{ + streams: []*audioStream{ + { + id: "a", + sum: 0, + total: 0, + }, + }, + threshold: 40, + }, + args: args{ + streamID: "a", + dBov: 20, + }, + want: audioStream{ + id: "a", + sum: 20, + total: 1, + }, + }, + { + name: "Must not increase sum and total when dBov is below threshold", + fields: fields{ + streams: []*audioStream{ + { + id: "a", + sum: 0, + total: 0, + }, + }, + threshold: 40, + }, + args: args{ + streamID: "a", + dBov: 60, + }, + want: audioStream{ + id: "a", + sum: 0, + total: 0, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + a := &AudioObserver{ + streams: tt.fields.streams, + threshold: tt.fields.threshold, + } + a.observe(tt.args.streamID, tt.args.dBov) + assert.Equal(t, *a.streams[0], tt.want) + }) + } +} + +func Test_audioLevel_removeStream(t *testing.T) { + type fields struct { + streams []*audioStream + } + type args struct { + streamID string + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "Must remove correct ID", + fields: fields{ + streams: []*audioStream{ + { + id: "a", + }, + { + id: "b", + }, + { + id: "c", + }, + { + id: "d", + }, + }, + }, + args: args{ + streamID: "b", + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + a := &AudioObserver{ + streams: tt.fields.streams, + } + a.removeStream(tt.args.streamID) + assert.Equal(t, len(a.streams), len(tt.fields.streams)-1) + for _, s := range a.streams { + assert.NotEqual(t, s.id, tt.args.streamID) + } + }) + } +} + +func Test_newAudioLevel(t *testing.T) { + type args struct { + threshold uint8 + interval int + filter int + } + tests := []struct { + name string + args args + want *AudioObserver + }{ + { + name: "Must return a new audio level", + args: args{ + threshold: 40, + interval: 1000, + filter: 20, + }, + want: &AudioObserver{ + expected: 1000 * 20 / 2000, + threshold: 40, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := NewAudioObserver(tt.args.threshold, tt.args.interval, tt.args.filter); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewAudioLevel() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/sfu/buffer/bucket.go b/pkg/sfu/buffer/bucket.go new file mode 100644 index 000000000..83444865d --- /dev/null +++ b/pkg/sfu/buffer/bucket.go @@ -0,0 +1,114 @@ +package buffer + +import ( + "encoding/binary" + "math" +) + +const maxPktSize = 1500 + +type Bucket struct { + buf []byte + src *[]byte + + init bool + step int + headSN uint16 + maxSteps int +} + +func NewBucket(buf *[]byte) *Bucket { + return &Bucket{ + src: buf, + buf: *buf, + maxSteps: int(math.Floor(float64(len(*buf))/float64(maxPktSize))) - 1, + } +} + +func (b *Bucket) AddPacket(pkt []byte, sn uint16, latest bool) ([]byte, error) { + if !b.init { + b.headSN = sn - 1 + b.init = true + } + if !latest { + return b.set(sn, pkt) + } + diff := sn - b.headSN + b.headSN = sn + for i := uint16(1); i < diff; i++ { + b.step++ + if b.step >= b.maxSteps { + b.step = 0 + } + } + return b.push(pkt), nil +} + +func (b *Bucket) GetPacket(buf []byte, sn uint16) (i int, err error) { + p := b.get(sn) + if p == nil { + err = errPacketNotFound + return + } + i = len(p) + if cap(buf) < i { + err = errBufferTooSmall + return + } + if len(buf) < i { + buf = buf[:i] + } + copy(buf, p) + return +} + +func (b *Bucket) push(pkt []byte) []byte { + binary.BigEndian.PutUint16(b.buf[b.step*maxPktSize:], uint16(len(pkt))) + off := b.step*maxPktSize + 2 + copy(b.buf[off:], pkt) + b.step++ + if b.step > b.maxSteps { + b.step = 0 + } + return b.buf[off : off+len(pkt)] +} + +func (b *Bucket) get(sn uint16) []byte { + pos := b.step - int(b.headSN-sn+1) + if pos < 0 { + if pos*-1 > b.maxSteps+1 { + return nil + } + pos = b.maxSteps + pos + 1 + } + off := pos * maxPktSize + if off > len(b.buf) { + return nil + } + if binary.BigEndian.Uint16(b.buf[off+4:off+6]) != sn { + return nil + } + sz := int(binary.BigEndian.Uint16(b.buf[off : off+2])) + return b.buf[off+2 : off+2+sz] +} + +func (b *Bucket) set(sn uint16, pkt []byte) ([]byte, error) { + if b.headSN-sn >= uint16(b.maxSteps+1) { + return nil, errPacketTooOld + } + pos := b.step - int(b.headSN-sn+1) + if pos < 0 { + pos = b.maxSteps + pos + 1 + } + off := pos * maxPktSize + if off > len(b.buf) || off < 0 { + return nil, errPacketTooOld + } + // Do not overwrite if packet exist + if binary.BigEndian.Uint16(b.buf[off+4:off+6]) == sn { + return nil, errRTXPacket + } + binary.BigEndian.PutUint16(b.buf[off:], uint16(len(pkt))) + copy(b.buf[off+2:], pkt) + return b.buf[off+2 : off+2+len(pkt)], nil +} diff --git a/pkg/sfu/buffer/bucket_test.go b/pkg/sfu/buffer/bucket_test.go new file mode 100644 index 000000000..706387f96 --- /dev/null +++ b/pkg/sfu/buffer/bucket_test.go @@ -0,0 +1,139 @@ +package buffer + +import ( + "testing" + + "github.com/pion/rtp" + "github.com/stretchr/testify/assert" +) + +var TestPackets = []*rtp.Packet{ + { + Header: rtp.Header{ + SequenceNumber: 1, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 3, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 4, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 6, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 7, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 10, + }, + }, +} + +func Test_queue(t *testing.T) { + b := make([]byte, 25000) + q := NewBucket(&b) + + for _, p := range TestPackets { + p := p + buf, err := p.Marshal() + assert.NoError(t, err) + assert.NotPanics(t, func() { + q.AddPacket(buf, p.SequenceNumber, true) + }) + } + var expectedSN uint16 + expectedSN = 6 + np := rtp.Packet{} + buff := make([]byte, maxPktSize) + i, err := q.GetPacket(buff, 6) + assert.NoError(t, err) + err = np.Unmarshal(buff[:i]) + assert.NoError(t, err) + assert.Equal(t, expectedSN, np.SequenceNumber) + + np2 := &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: 8, + }, + } + buf, err := np2.Marshal() + assert.NoError(t, err) + expectedSN = 8 + q.AddPacket(buf, 8, false) + i, err = q.GetPacket(buff, expectedSN) + assert.NoError(t, err) + err = np.Unmarshal(buff[:i]) + assert.NoError(t, err) + assert.Equal(t, expectedSN, np.SequenceNumber) + + _, err = q.AddPacket(buf, 8, false) + assert.ErrorIs(t, err, errRTXPacket) +} + +func Test_queue_edges(t *testing.T) { + var TestPackets = []*rtp.Packet{ + { + Header: rtp.Header{ + SequenceNumber: 65533, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 65534, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 2, + }, + }, + } + b := make([]byte, 25000) + q := NewBucket(&b) + for _, p := range TestPackets { + p := p + assert.NotNil(t, p) + assert.NotPanics(t, func() { + p := p + buf, err := p.Marshal() + assert.NoError(t, err) + assert.NotPanics(t, func() { + q.AddPacket(buf, p.SequenceNumber, true) + }) + }) + } + var expectedSN uint16 + expectedSN = 65534 + np := rtp.Packet{} + buff := make([]byte, maxPktSize) + i, err := q.GetPacket(buff, expectedSN) + assert.NoError(t, err) + err = np.Unmarshal(buff[:i]) + assert.NoError(t, err) + assert.Equal(t, expectedSN, np.SequenceNumber) + + np2 := rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: 65535, + }, + } + buf, err := np2.Marshal() + assert.NoError(t, err) + q.AddPacket(buf, np2.SequenceNumber, false) + i, err = q.GetPacket(buff, expectedSN+1) + assert.NoError(t, err) + err = np.Unmarshal(buff[:i]) + assert.NoError(t, err) + assert.Equal(t, expectedSN+1, np.SequenceNumber) +} diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go new file mode 100644 index 000000000..e3fe18b19 --- /dev/null +++ b/pkg/sfu/buffer/buffer.go @@ -0,0 +1,715 @@ +package buffer + +import ( + "encoding/binary" + "io" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gammazero/deque" + "github.com/go-logr/logr" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v3" +) + +const ( + MaxSN = 1 << 16 + + reportDelta = 1e9 +) + +// Logger is an implementation of logr.Logger. If is not provided - will be turned off. +var Logger logr.Logger = logr.Discard() + +type pendingPackets struct { + arrivalTime int64 + packet []byte +} + +type ExtPacket struct { + Head bool + Cycle uint32 + Arrival int64 + Packet rtp.Packet + Payload interface{} + KeyFrame bool +} + +// Buffer contains all packets +type Buffer struct { + sync.Mutex + bucket *Bucket + nacker *nackQueue + videoPool *sync.Pool + audioPool *sync.Pool + codecType webrtc.RTPCodecType + extPackets deque.Deque + pPackets []pendingPackets + closeOnce sync.Once + mediaSSRC uint32 + clockRate uint32 + maxBitrate uint64 + lastReport int64 + twccExt uint8 + audioExt uint8 + bound bool + closed atomicBool + mime string + + // supported feedbacks + remb bool + nack bool + twcc bool + audioLevel bool + + minPacketProbe int + lastPacketRead int + maxTemporalLayer int32 + bitrate atomic.Value + bitrateHelper [4]uint64 + lastSRNTPTime uint64 + lastSRRTPTime uint32 + lastSRRecv int64 // Represents wall clock of the most recent sender report arrival + baseSN uint16 + lastRtcpPacketTime int64 // Time the last RTCP packet was received. + lastRtcpSrTime int64 // Time the last RTCP SR was received. Required for DLSR computation. + lastTransit uint32 + seqHdlr SeqWrapHandler + + stats Stats + + latestTimestamp uint32 // latest received RTP timestamp on packet + latestTimestampTime int64 // Time of the latest timestamp (in nanos since unix epoch) + lastFractionLostToReport uint8 // Last fractionlost from subscribers, should report to publisher; Audio only + + // callbacks + onClose func() + onAudioLevel func(level uint8) + feedbackCB func([]rtcp.Packet) + feedbackTWCC func(sn uint16, timeNS int64, marker bool) + + // logger + logger logr.Logger +} + +type Stats struct { + LastExpected uint32 + LastReceived uint32 + LostRate float32 + PacketCount uint32 // Number of packets received from this source. + Jitter float64 // An estimate of the statistical variance of the RTP data packet inter-arrival time. + TotalByte uint64 +} + +// BufferOptions provides configuration options for the buffer +type Options struct { + MaxBitRate uint64 +} + +// NewBuffer constructs a new Buffer +func NewBuffer(ssrc uint32, vp, ap *sync.Pool, logger logr.Logger) *Buffer { + b := &Buffer{ + mediaSSRC: ssrc, + videoPool: vp, + audioPool: ap, + logger: logger, + } + b.bitrate.Store(make([]uint64, len(b.bitrateHelper))) + b.extPackets.SetMinCapacity(7) + return b +} + +func (b *Buffer) Bind(params webrtc.RTPParameters, o Options) { + b.Lock() + defer b.Unlock() + + codec := params.Codecs[0] + b.clockRate = codec.ClockRate + b.maxBitrate = o.MaxBitRate + b.mime = strings.ToLower(codec.MimeType) + + switch { + case strings.HasPrefix(b.mime, "audio/"): + b.codecType = webrtc.RTPCodecTypeAudio + b.bucket = NewBucket(b.audioPool.Get().(*[]byte)) + case strings.HasPrefix(b.mime, "video/"): + b.codecType = webrtc.RTPCodecTypeVideo + b.bucket = NewBucket(b.videoPool.Get().(*[]byte)) + default: + b.codecType = webrtc.RTPCodecType(0) + } + + for _, ext := range params.HeaderExtensions { + if ext.URI == sdp.TransportCCURI { + b.twccExt = uint8(ext.ID) + break + } + } + + if b.codecType == webrtc.RTPCodecTypeVideo { + for _, fb := range codec.RTCPFeedback { + switch fb.Type { + case webrtc.TypeRTCPFBGoogREMB: + b.logger.V(1).Info("Setting feedback", "type", "webrtc.TypeRTCPFBGoogREMB") + b.remb = true + case webrtc.TypeRTCPFBTransportCC: + b.logger.V(1).Info("Setting feedback", "type", webrtc.TypeRTCPFBTransportCC) + b.twcc = true + case webrtc.TypeRTCPFBNACK: + b.logger.V(1).Info("Setting feedback", "type", webrtc.TypeRTCPFBNACK) + b.nacker = newNACKQueue() + b.nack = true + } + } + } else if b.codecType == webrtc.RTPCodecTypeAudio { + for _, h := range params.HeaderExtensions { + if h.URI == sdp.AudioLevelURI { + b.audioLevel = true + b.audioExt = uint8(h.ID) + } + } + } + + for _, pp := range b.pPackets { + b.calc(pp.packet, pp.arrivalTime) + } + b.pPackets = nil + b.bound = true + + b.logger.V(1).Info("NewBuffer", "MaxBitRate", o.MaxBitRate) +} + +// Write adds a RTP Packet, out of order, new packet may be arrived later +func (b *Buffer) Write(pkt []byte) (n int, err error) { + b.Lock() + defer b.Unlock() + + if b.closed.get() { + err = io.EOF + return + } + + if !b.bound { + packet := make([]byte, len(pkt)) + copy(packet, pkt) + b.pPackets = append(b.pPackets, pendingPackets{ + packet: packet, + arrivalTime: time.Now().UnixNano(), + }) + return + } + + b.calc(pkt, time.Now().UnixNano()) + + return +} + +func (b *Buffer) Read(buff []byte) (n int, err error) { + for { + if b.closed.get() { + err = io.EOF + return + } + b.Lock() + if b.pPackets != nil && len(b.pPackets) > b.lastPacketRead { + if len(buff) < len(b.pPackets[b.lastPacketRead].packet) { + err = errBufferTooSmall + b.Unlock() + return + } + n = len(b.pPackets[b.lastPacketRead].packet) + copy(buff, b.pPackets[b.lastPacketRead].packet) + b.lastPacketRead++ + b.Unlock() + return + } + b.Unlock() + time.Sleep(25 * time.Millisecond) + } +} + +func (b *Buffer) ReadExtended() (*ExtPacket, error) { + for { + if b.closed.get() { + return nil, io.EOF + } + b.Lock() + if b.extPackets.Len() > 0 { + extPkt := b.extPackets.PopFront().(*ExtPacket) + b.Unlock() + return extPkt, nil + } + b.Unlock() + time.Sleep(10 * time.Millisecond) + } +} + +func (b *Buffer) Close() error { + b.Lock() + defer b.Unlock() + + b.closeOnce.Do(func() { + if b.bucket != nil && b.codecType == webrtc.RTPCodecTypeVideo { + b.videoPool.Put(b.bucket.src) + } + if b.bucket != nil && b.codecType == webrtc.RTPCodecTypeAudio { + b.audioPool.Put(b.bucket.src) + } + b.closed.set(true) + b.onClose() + }) + return nil +} + +func (b *Buffer) OnClose(fn func()) { + b.onClose = fn +} + +func (b *Buffer) calc(pkt []byte, arrivalTime int64) { + sn := binary.BigEndian.Uint16(pkt[2:4]) + + var headPkt bool + if b.stats.PacketCount == 0 { + b.baseSN = sn + b.lastReport = arrivalTime + b.seqHdlr.UpdateMaxSeq(uint32(sn)) + headPkt = true + } else { + extSN, isNewer := b.seqHdlr.Unwrap(sn) + if b.nack { + if isNewer { + for i := b.seqHdlr.MaxSeqNo() + 1; i < extSN; i++ { + b.nacker.push(i) + } + } else { + b.nacker.remove(extSN) + } + } + if isNewer { + b.seqHdlr.UpdateMaxSeq(extSN) + } + headPkt = isNewer + } + + var p rtp.Packet + pb, err := b.bucket.AddPacket(pkt, sn, headPkt) + if err != nil { + if err == errRTXPacket { + return + } + return + } + if err = p.Unmarshal(pb); err != nil { + return + } + + // submit to TWCC even if it is a padding only packet. Clients use padding only packets as probes + // for bandwidth estimation + if b.twcc { + if ext := p.GetExtension(b.twccExt); ext != nil && len(ext) > 1 { + b.feedbackTWCC(binary.BigEndian.Uint16(ext[0:2]), arrivalTime, p.Marker) + } + } + + b.stats.TotalByte += uint64(len(pkt)) + b.stats.PacketCount++ + + ep := ExtPacket{ + Head: headPkt, + Cycle: b.seqHdlr.Cycles(), + Packet: p, + Arrival: arrivalTime, + } + + if len(p.Payload) == 0 { + // padding only packet, nothing else to do + b.extPackets.PushBack(&ep) + return + } + + temporalLayer := int32(0) + switch b.mime { + case "video/vp8": + vp8Packet := VP8{} + if err := vp8Packet.Unmarshal(p.Payload); err != nil { + return + } + ep.Payload = vp8Packet + ep.KeyFrame = vp8Packet.IsKeyFrame + temporalLayer = int32(vp8Packet.TID) + case "video/h264": + ep.KeyFrame = isH264Keyframe(p.Payload) + } + + if b.minPacketProbe < 25 { + if sn < b.baseSN { + b.baseSN = sn + } + + if b.mime == "video/vp8" { + pld := ep.Payload.(VP8) + mtl := atomic.LoadInt32(&b.maxTemporalLayer) + if mtl < int32(pld.TID) { + atomic.StoreInt32(&b.maxTemporalLayer, int32(pld.TID)) + } + } + + b.minPacketProbe++ + } + + b.extPackets.PushBack(&ep) + + // if first time update or the timestamp is later (factoring timestamp wrap around) + latestTimestamp := atomic.LoadUint32(&b.latestTimestamp) + latestTimestampTimeInNanosSinceEpoch := atomic.LoadInt64(&b.latestTimestampTime) + if (latestTimestampTimeInNanosSinceEpoch == 0) || IsLaterTimestamp(p.Timestamp, latestTimestamp) { + atomic.StoreUint32(&b.latestTimestamp, p.Timestamp) + atomic.StoreInt64(&b.latestTimestampTime, arrivalTime) + } + + arrival := uint32(arrivalTime / 1e6 * int64(b.clockRate/1e3)) + transit := arrival - p.Timestamp + if b.lastTransit != 0 { + d := int32(transit - b.lastTransit) + if d < 0 { + d = -d + } + b.stats.Jitter += (float64(d) - b.stats.Jitter) / 16 + } + b.lastTransit = transit + + if b.audioLevel { + if e := p.GetExtension(b.audioExt); e != nil && b.onAudioLevel != nil { + ext := rtp.AudioLevelExtension{} + if err := ext.Unmarshal(e); err == nil { + b.onAudioLevel(ext.Level) + } + } + } + + if b.nacker != nil { + if r := b.buildNACKPacket(); r != nil { + b.feedbackCB(r) + } + } + + b.bitrateHelper[temporalLayer] += uint64(len(pkt)) + + diff := arrivalTime - b.lastReport + if diff >= reportDelta { + // LK-TODO-START + // As this happens in the data path, if there are no packets received + // in an interval, the bitrate is stuck with the old value. GetBitrate() + // method in sfu.Receiver uses the availableLayers set by stream + // tracker to report 0 bitrate if a layer is not available. But, stream + // tracker is not run for the lowest layer. So, if the lowest layer stops, + // stale bitrate will be reported. The simplest thing might be to run the + // stream tracker on all layers to address this. Another option to look at + // is some monitoring loop running at low frequency and reporting bitrate. + // LK-TODO-END + bitrates, ok := b.bitrate.Load().([]uint64) + if !ok { + bitrates = make([]uint64, len(b.bitrateHelper)) + } + for i := 0; i < len(b.bitrateHelper); i++ { + br := (8 * b.bitrateHelper[i] * uint64(reportDelta)) / uint64(diff) + bitrates[i] = br + b.bitrateHelper[i] = 0 + } + b.bitrate.Store(bitrates) + b.feedbackCB(b.getRTCP()) + b.lastReport = arrivalTime + } +} + +func (b *Buffer) buildNACKPacket() []rtcp.Packet { + if nacks, askKeyframe := b.nacker.pairs(b.seqHdlr.MaxSeqNo()); (nacks != nil && len(nacks) > 0) || askKeyframe { + var pkts []rtcp.Packet + if len(nacks) > 0 { + pkts = []rtcp.Packet{&rtcp.TransportLayerNack{ + MediaSSRC: b.mediaSSRC, + Nacks: nacks, + }} + } + + if askKeyframe { + pkts = append(pkts, &rtcp.PictureLossIndication{ + MediaSSRC: b.mediaSSRC, + }) + } + return pkts + } + return nil +} + +func (b *Buffer) buildREMBPacket() *rtcp.ReceiverEstimatedMaximumBitrate { + br := b.Bitrate() + if b.stats.LostRate < 0.02 { + br = uint64(float64(br)*1.09) + 2000 + } + if b.stats.LostRate > .1 { + br = uint64(float64(br) * float64(1-0.5*b.stats.LostRate)) + } + if br > b.maxBitrate { + br = b.maxBitrate + } + if br < 100000 { + br = 100000 + } + b.stats.TotalByte = 0 + + return &rtcp.ReceiverEstimatedMaximumBitrate{ + Bitrate: float32(br), + SSRCs: []uint32{b.mediaSSRC}, + } +} + +func (b *Buffer) buildReceptionReport() rtcp.ReceptionReport { + extMaxSeq := b.seqHdlr.MaxSeqNo() + expected := extMaxSeq - uint32(b.baseSN) + 1 + lost := uint32(0) + if b.stats.PacketCount < expected && b.stats.PacketCount != 0 { + lost = expected - b.stats.PacketCount + } + expectedInterval := expected - b.stats.LastExpected + b.stats.LastExpected = expected + + receivedInterval := b.stats.PacketCount - b.stats.LastReceived + b.stats.LastReceived = b.stats.PacketCount + + lostInterval := expectedInterval - receivedInterval + + b.stats.LostRate = float32(lostInterval) / float32(expectedInterval) + var fracLost uint8 + if expectedInterval != 0 && lostInterval > 0 { + fracLost = uint8((lostInterval << 8) / expectedInterval) + } + if b.lastFractionLostToReport > fracLost { + // If fractionlost from subscriber is bigger than sfu received, use it. + fracLost = b.lastFractionLostToReport + } + + var dlsr uint32 + + if b.lastSRRecv != 0 { + delayMS := uint32((time.Now().UnixNano() - b.lastSRRecv) / 1e6) + dlsr = (delayMS / 1e3) << 16 + dlsr |= (delayMS % 1e3) * 65536 / 1000 + } + + rr := rtcp.ReceptionReport{ + SSRC: b.mediaSSRC, + FractionLost: fracLost, + TotalLost: lost, + LastSequenceNumber: extMaxSeq, + Jitter: uint32(b.stats.Jitter), + LastSenderReport: uint32(b.lastSRNTPTime >> 16), + Delay: dlsr, + } + return rr +} + +func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { + b.Lock() + b.lastSRRTPTime = rtpTime + b.lastSRNTPTime = ntpTime + b.lastSRRecv = time.Now().UnixNano() + b.Unlock() +} + +func (b *Buffer) SetLastFractionLostReport(lost uint8) { + b.lastFractionLostToReport = lost +} + +func (b *Buffer) getRTCP() []rtcp.Packet { + var pkts []rtcp.Packet + + pkts = append(pkts, &rtcp.ReceiverReport{ + Reports: []rtcp.ReceptionReport{b.buildReceptionReport()}, + }) + + if b.remb && !b.twcc { + pkts = append(pkts, b.buildREMBPacket()) + } + + return pkts +} + +func (b *Buffer) GetPacket(buff []byte, sn uint16) (int, error) { + b.Lock() + defer b.Unlock() + if b.closed.get() { + return 0, io.EOF + } + return b.bucket.GetPacket(buff, sn) +} + +// Bitrate returns the current publisher stream bitrate. +func (b *Buffer) Bitrate() uint64 { + bitrates, ok := b.bitrate.Load().([]uint64) + bitrate := uint64(0) + if ok { + for _, b := range bitrates { + bitrate += b + } + } + return bitrate +} + +// BitrateTemporal returns the current publisher stream bitrate temporal layer wise. +func (b *Buffer) BitrateTemporal() []uint64 { + bitrates, ok := b.bitrate.Load().([]uint64) + if !ok { + return make([]uint64, len(b.bitrateHelper)) + } + + // copy and return + brs := make([]uint64, len(bitrates)) + copy(brs, bitrates) + + return brs +} + +// BitrateTemporalCumulative returns the current publisher stream bitrate temporal layer accumulated with lower temporal layers. +func (b *Buffer) BitrateTemporalCumulative() []uint64 { + bitrates, ok := b.bitrate.Load().([]uint64) + if !ok { + return make([]uint64, len(b.bitrateHelper)) + } + + // copy and process + brs := make([]uint64, len(bitrates)) + copy(brs, bitrates) + + for i := len(brs) - 1; i >= 1; i-- { + if brs[i] != 0 { + for j := i - 1; j >= 0; j-- { + brs[i] += brs[j] + } + } + } + + return brs +} + +func (b *Buffer) MaxTemporalLayer() int32 { + return atomic.LoadInt32(&b.maxTemporalLayer) +} + +func (b *Buffer) OnTransportWideCC(fn func(sn uint16, timeNS int64, marker bool)) { + b.feedbackTWCC = fn +} + +func (b *Buffer) OnFeedback(fn func(fb []rtcp.Packet)) { + b.feedbackCB = fn +} + +func (b *Buffer) OnAudioLevel(fn func(level uint8)) { + b.onAudioLevel = fn +} + +// GetMediaSSRC returns the associated SSRC of the RTP stream +func (b *Buffer) GetMediaSSRC() uint32 { + return b.mediaSSRC +} + +// GetClockRate returns the RTP clock rate +func (b *Buffer) GetClockRate() uint32 { + return b.clockRate +} + +// GetSenderReportData returns the rtp, ntp and nanos of the last sender report +func (b *Buffer) GetSenderReportData() (rtpTime uint32, ntpTime uint64, lastReceivedTimeInNanosSinceEpoch int64) { + rtpTime = atomic.LoadUint32(&b.lastSRRTPTime) + ntpTime = atomic.LoadUint64(&b.lastSRNTPTime) + lastReceivedTimeInNanosSinceEpoch = atomic.LoadInt64(&b.lastSRRecv) + + return rtpTime, ntpTime, lastReceivedTimeInNanosSinceEpoch +} + +// GetStats returns the raw statistics about a particular buffer state +func (b *Buffer) GetStats() (stats Stats) { + b.Lock() + stats = b.stats + b.Unlock() + return +} + +// GetLatestTimestamp returns the latest RTP timestamp factoring in potential RTP timestamp wrap-around +func (b *Buffer) GetLatestTimestamp() (latestTimestamp uint32, latestTimestampTimeInNanosSinceEpoch int64) { + latestTimestamp = atomic.LoadUint32(&b.latestTimestamp) + latestTimestampTimeInNanosSinceEpoch = atomic.LoadInt64(&b.latestTimestampTime) + + return latestTimestamp, latestTimestampTimeInNanosSinceEpoch +} + +// IsTimestampWrapAround returns true if wrap around happens from timestamp1 to timestamp2 +func IsTimestampWrapAround(timestamp1 uint32, timestamp2 uint32) bool { + return timestamp2 < timestamp1 && timestamp1 > 0xf0000000 && timestamp2 < 0x0fffffff +} + +// IsLaterTimestamp returns true if timestamp1 is later in time than timestamp2 factoring in timestamp wrap-around +func IsLaterTimestamp(timestamp1 uint32, timestamp2 uint32) bool { + if timestamp1 > timestamp2 { + if IsTimestampWrapAround(timestamp1, timestamp2) { + return false + } + return true + } + if IsTimestampWrapAround(timestamp2, timestamp1) { + return true + } + return false +} + +func isNewerUint16(val1, val2 uint16) bool { + return val1 != val2 && val1-val2 < 0x8000 +} + +type SeqWrapHandler struct { + maxSeqNo uint32 +} + +func (s *SeqWrapHandler) Cycles() uint32 { + return s.maxSeqNo & 0xffff0000 +} + +func (s *SeqWrapHandler) MaxSeqNo() uint32 { + return s.maxSeqNo +} + +// unwrap seq and update the maxSeqNo. return unwraped value, and whether seq is newer +func (s *SeqWrapHandler) Unwrap(seq uint16) (uint32, bool) { + + maxSeqNo := uint16(s.maxSeqNo) + delta := int32(seq) - int32(maxSeqNo) + + newer := isNewerUint16(seq, maxSeqNo) + + if newer { + if delta < 0 { + // seq is newer, but less than maxSeqNo, wrap around + delta += 0x10000 + } + } else { + // older value + if delta > 0 && (int32(s.maxSeqNo)+delta-0x10000) >= 0 { + // wrap backwards, should not less than 0 in this case: + // at start time, received seq 1, set s.maxSeqNo =1 , + // then a out of order seq 65534 coming, we can't unwrap + // the seq to -2 + delta -= 0x10000 + } + } + + unwrapped := uint32(int32(s.maxSeqNo) + delta) + return unwrapped, newer +} + +func (s *SeqWrapHandler) UpdateMaxSeq(extSeq uint32) { + s.maxSeqNo = extSeq +} diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go new file mode 100644 index 000000000..a178069a6 --- /dev/null +++ b/pkg/sfu/buffer/buffer_test.go @@ -0,0 +1,363 @@ +package buffer + +import ( + "sync" + "testing" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/logger" + "github.com/pion/rtcp" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3" + "github.com/stretchr/testify/assert" +) + +func CreateTestPacket(pktStamp *SequenceNumberAndTimeStamp) *rtp.Packet { + if pktStamp == nil { + return &rtp.Packet{ + Header: rtp.Header{}, + Payload: []byte{1, 2, 3}, + } + } + + return &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: pktStamp.SequenceNumber, + Timestamp: pktStamp.Timestamp, + }, + Payload: []byte{1, 2, 3}, + } +} + +type SequenceNumberAndTimeStamp struct { + SequenceNumber uint16 + Timestamp uint32 +} + +func CreateTestListPackets(snsAndTSs []SequenceNumberAndTimeStamp) (packetList []*rtp.Packet) { + for _, item := range snsAndTSs { + item := item + packetList = append(packetList, CreateTestPacket(&item)) + } + + return packetList +} + +func TestNack(t *testing.T) { + pool := &sync.Pool{ + New: func() interface{} { + b := make([]byte, 1500) + return &b + }, + } + logger.SetGlobalOptions(logger.GlobalConfig{V: 1}) // 2 - TRACE + logger := logger.New() + + t.Run("nack normal", func(t *testing.T) { + buff := NewBuffer(123, pool, pool, logger) + buff.codecType = webrtc.RTPCodecTypeVideo + assert.NotNil(t, buff) + var wg sync.WaitGroup + // 3 nacks 1 Pli + wg.Add(4) + buff.OnFeedback(func(fb []rtcp.Packet) { + for _, pkt := range fb { + switch p := pkt.(type) { + case *rtcp.TransportLayerNack: + if p.Nacks[0].PacketList()[0] == 1 && p.MediaSSRC == 123 { + wg.Done() + } + case *rtcp.PictureLossIndication: + if p.MediaSSRC == 123 { + wg.Done() + } + } + } + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{ + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: "video/vp8", + ClockRate: 90000, + RTCPFeedback: []webrtc.RTCPFeedback{{ + Type: "nack", + }}, + }, + PayloadType: 96, + }, + }, + }, Options{}) + for i := 0; i < 15; i++ { + if i == 1 { + continue + } + pkt := rtp.Packet{ + Header: rtp.Header{SequenceNumber: uint16(i), Timestamp: uint32(i)}, + Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}, + } + b, err := pkt.Marshal() + assert.NoError(t, err) + _, err = buff.Write(b) + assert.NoError(t, err) + } + wg.Wait() + + }) + + t.Run("nack with seq wrap", func(t *testing.T) { + buff := NewBuffer(123, pool, pool, logger) + buff.codecType = webrtc.RTPCodecTypeVideo + assert.NotNil(t, buff) + var wg sync.WaitGroup + expects := map[uint16]int{ + 65534: 0, + 65535: 0, + 0: 0, + 1: 0, + } + wg.Add(3 * len(expects)) // retry 3 times + buff.OnFeedback(func(fb []rtcp.Packet) { + for _, pkt := range fb { + switch p := pkt.(type) { + case *rtcp.TransportLayerNack: + if p.MediaSSRC == 123 { + for _, v := range p.Nacks { + v.Range(func(seq uint16) bool { + if _, ok := expects[seq]; ok { + wg.Done() + } else { + assert.Fail(t, "unexpected nack seq ", seq) + } + return true + }) + } + } + case *rtcp.PictureLossIndication: + if p.MediaSSRC == 123 { + // wg.Done() + } + } + } + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{ + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: "video/vp8", + ClockRate: 90000, + RTCPFeedback: []webrtc.RTCPFeedback{{ + Type: "nack", + }}, + }, + PayloadType: 96, + }, + }, + }, Options{}) + for i := 0; i < 15; i++ { + if i > 0 && i < 5 { + continue + } + pkt := rtp.Packet{ + Header: rtp.Header{SequenceNumber: uint16(i + 65533), Timestamp: uint32(i)}, + Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}, + } + b, err := pkt.Marshal() + assert.NoError(t, err) + _, err = buff.Write(b) + assert.NoError(t, err) + } + wg.Wait() + + }) +} + +func TestNewBuffer(t *testing.T) { + type args struct { + options Options + ssrc uint32 + } + tests := []struct { + name string + args args + }{ + { + name: "Must not be nil and add packets in sequence", + args: args{ + options: Options{ + MaxBitRate: 1e6, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var TestPackets = []*rtp.Packet{ + { + Header: rtp.Header{ + SequenceNumber: 65533, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 65534, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 2, + }, + }, + { + Header: rtp.Header{ + SequenceNumber: 65535, + }, + }, + } + pool := &sync.Pool{ + New: func() interface{} { + b := make([]byte, 1500) + return &b + }, + } + logger.SetGlobalOptions(logger.GlobalConfig{V: 2}) // 2 - TRACE + logger := logger.New() + buff := NewBuffer(123, pool, pool, logger) + buff.codecType = webrtc.RTPCodecTypeVideo + assert.NotNil(t, buff) + assert.NotNil(t, TestPackets) + buff.OnFeedback(func(_ []rtcp.Packet) { + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: "video/vp8", + ClockRate: 9600, + RTCPFeedback: nil, + }, + PayloadType: 0, + }}, + }, Options{}) + + for _, p := range TestPackets { + buf, _ := p.Marshal() + buff.Write(buf) + } + // assert.Equal(t, 6, buff.PacketQueue.size) + assert.Equal(t, uint32(1<<16), buff.seqHdlr.Cycles()) + assert.Equal(t, uint16(2), uint16(buff.seqHdlr.MaxSeqNo())) + }) + } +} + +func TestFractionLostReport(t *testing.T) { + pool := &sync.Pool{ + New: func() interface{} { + b := make([]byte, 1500) + return &b + }, + } + logger.SetGlobalOptions(logger.GlobalConfig{V: 1}) // 2 - TRACE + buff := NewBuffer(123, pool, pool, logger.New()) + buff.codecType = webrtc.RTPCodecTypeVideo + assert.NotNil(t, buff) + var wg sync.WaitGroup + wg.Add(1) + buff.SetLastFractionLostReport(55) + buff.OnFeedback(func(fb []rtcp.Packet) { + for _, pkt := range fb { + switch p := pkt.(type) { + case *rtcp.ReceiverReport: + for _, v := range p.Reports { + assert.EqualValues(t, 55, v.FractionLost) + } + wg.Done() + } + } + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{ + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: "audio/opus", + ClockRate: 48000, + }, + PayloadType: 96, + }, + }, + }, Options{}) + for i := 0; i < 15; i++ { + pkt := rtp.Packet{ + Header: rtp.Header{SequenceNumber: uint16(i), Timestamp: uint32(i)}, + Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}, + } + b, err := pkt.Marshal() + assert.NoError(t, err) + if i == 1 { + time.Sleep(1 * time.Second) + } + _, err = buff.Write(b) + assert.NoError(t, err) + } + wg.Wait() +} + +func TestSeqWrapHandler(t *testing.T) { + s := SeqWrapHandler{} + s.UpdateMaxSeq(1) + assert.Equal(t, uint32(1), s.MaxSeqNo()) + + type caseInfo struct { + seqs []uint32 //{seq1, seq2, unwrap of seq2} + newer bool // seq2 is newer than seq1 + } + // test normal case, name -> {seq1, seq2, unwrap of seq2} + cases := map[string]caseInfo{ + "no wrap": {[]uint32{1, 4, 4}, true}, + "no wrap backward": {[]uint32{4, 1, 1}, false}, + "wrap around forward to zero": {[]uint32{65534, 0, 65536}, true}, + "wrap around forward": {[]uint32{65534, 10, 65546}, true}, + "wrap around forward 2": {[]uint32{65535 + 65536*2, 1, 1 + 65536*3}, true}, + "wrap around backward ": {[]uint32{5, 65534, 65534}, false}, + "wrap around backward less than zero": {[]uint32{5, 65534, 65534}, false}, + } + + for k, v := range cases { + t.Run(k, func(t *testing.T) { + s := SeqWrapHandler{} + s.UpdateMaxSeq(v.seqs[0]) + extsn, newer := s.Unwrap(uint16(v.seqs[1])) + assert.Equal(t, v.newer, newer) + assert.Equal(t, v.seqs[2], extsn) + }) + } + +} + +func TestIsTimestampWrap(t *testing.T) { + type caseInfo struct { + name string + ts1 uint32 + ts2 uint32 + later bool + } + + cases := []caseInfo{ + {"normal case 1 timestamp later ", 2, 1, true}, + {"normal case 2 timestamp later", 0x1c000000, 0x10000000, true}, + {"wrap case timestamp later", 0xffff, 0xfc000000, true}, + {"wrap case timestamp early", 0xfc000000, 0xffff, false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.later, IsLaterTimestamp(c.ts1, c.ts2)) + }) + } +} diff --git a/pkg/sfu/buffer/errors.go b/pkg/sfu/buffer/errors.go new file mode 100644 index 000000000..8c273feaf --- /dev/null +++ b/pkg/sfu/buffer/errors.go @@ -0,0 +1,10 @@ +package buffer + +import "errors" + +var ( + errPacketNotFound = errors.New("packet not found in cache") + errBufferTooSmall = errors.New("buffer too small") + errPacketTooOld = errors.New("received packet too old") + errRTXPacket = errors.New("packet already received") +) diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go new file mode 100644 index 000000000..627c76072 --- /dev/null +++ b/pkg/sfu/buffer/factory.go @@ -0,0 +1,97 @@ +package buffer + +import ( + "io" + "sync" + + "github.com/go-logr/logr" + "github.com/pion/transport/packetio" +) + +type Factory struct { + sync.RWMutex + videoPool *sync.Pool + audioPool *sync.Pool + rtpBuffers map[uint32]*Buffer + rtcpReaders map[uint32]*RTCPReader + logger logr.Logger +} + +func NewBufferFactory(trackingPackets int, logger logr.Logger) *Factory { + // Enable package wide logging for non-method functions. + // If logger is empty - use default Logger. + // Logger is a public variable in buffer package. + if logger == (logr.Logger{}) { + logger = Logger + } else { + Logger = logger + } + + return &Factory{ + videoPool: &sync.Pool{ + New: func() interface{} { + b := make([]byte, trackingPackets*maxPktSize) + return &b + }, + }, + audioPool: &sync.Pool{ + New: func() interface{} { + b := make([]byte, maxPktSize*25) + return &b + }, + }, + rtpBuffers: make(map[uint32]*Buffer), + rtcpReaders: make(map[uint32]*RTCPReader), + logger: logger, + } +} + +func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser { + f.Lock() + defer f.Unlock() + switch packetType { + case packetio.RTCPBufferPacket: + if reader, ok := f.rtcpReaders[ssrc]; ok { + return reader + } + reader := NewRTCPReader(ssrc) + f.rtcpReaders[ssrc] = reader + reader.OnClose(func() { + f.Lock() + delete(f.rtcpReaders, ssrc) + f.Unlock() + }) + return reader + case packetio.RTPBufferPacket: + if reader, ok := f.rtpBuffers[ssrc]; ok { + return reader + } + buffer := NewBuffer(ssrc, f.videoPool, f.audioPool, f.logger) + f.rtpBuffers[ssrc] = buffer + buffer.OnClose(func() { + f.Lock() + delete(f.rtpBuffers, ssrc) + f.Unlock() + }) + return buffer + } + return nil +} + +func (f *Factory) GetBufferPair(ssrc uint32) (*Buffer, *RTCPReader) { + f.RLock() + defer f.RUnlock() + return f.rtpBuffers[ssrc], f.rtcpReaders[ssrc] +} + +func (f *Factory) GetBuffer(ssrc uint32) *Buffer { + f.RLock() + defer f.RUnlock() + return f.rtpBuffers[ssrc] +} + +func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader { + f.RLock() + defer f.RUnlock() + return f.rtcpReaders[ssrc] +} diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go new file mode 100644 index 000000000..aff3efa88 --- /dev/null +++ b/pkg/sfu/buffer/helpers.go @@ -0,0 +1,297 @@ +package buffer + +import ( + "encoding/binary" + "errors" + "sync/atomic" +) + +var ( + errShortPacket = errors.New("packet is not large enough") + errNilPacket = errors.New("invalid nil packet") + errInvalidPacket = errors.New("invalid packet") +) + +type atomicBool int32 + +func (a *atomicBool) set(value bool) { + var i int32 + if value { + i = 1 + } + atomic.StoreInt32((*int32)(a), i) +} + +func (a *atomicBool) get() bool { + return atomic.LoadInt32((*int32)(a)) != 0 +} + +// VP8 is a helper to get temporal data from VP8 packet header +/* + VP8 Payload Descriptor + 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + |X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED) + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL) + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL) + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + L: | TL0PICIDX | (OPTIONAL) | PictureID | + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + T/K:|TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL) + +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ + T/K:|TID|Y| KEYIDX | (OPTIONAL) + +-+-+-+-+-+-+-+-+ +*/ +type VP8 struct { + TemporalSupported bool // LK-TODO: CLEANUP-REMOVE + FirstByte byte + + PictureIDPresent int + PictureID uint16 /* 8 or 16 bits, picture ID */ + PicIDIdx int // LK-TODO: CLEANUP-REMOVE + MBit bool + + TL0PICIDXPresent int + TL0PICIDX uint8 /* 8 bits temporal level zero index */ + TlzIdx int // LK-TODO: CLEANUP-REMOVE + + // Optional Header If either of the T or K bits are set to 1, + // the TID/Y/KEYIDX extension field MUST be present. + TIDPresent int + TID uint8 /* 2 bits temporal layer idx */ + Y uint8 + + KEYIDXPresent int + KEYIDX uint8 /* 5 bits of key frame idx */ + + HeaderSize int + + // IsKeyFrame is a helper to detect if current packet is a keyframe + IsKeyFrame bool +} + +// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon +func (p *VP8) Unmarshal(payload []byte) error { + if payload == nil { + return errNilPacket + } + + payloadLen := len(payload) + + if payloadLen < 1 { + return errShortPacket + } + + idx := 0 + p.FirstByte = payload[idx] + S := payload[idx]&0x10 > 0 + // Check for extended bit control + if payload[idx]&0x80 > 0 { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + I := payload[idx]&0x80 > 0 + L := payload[idx]&0x40 > 0 + T := payload[idx]&0x20 > 0 + K := payload[idx]&0x10 > 0 + if L && !T { + return errInvalidPacket + } + // Check if T is present, if not, no temporal layer is available + p.TemporalSupported = payload[idx]&0x20 > 0 + // Check for PictureID + if I { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + p.PicIDIdx = idx + p.PictureIDPresent = 1 + pid := payload[idx] & 0x7f + // Check if m is 1, then Picture ID is 15 bits + if payload[idx]&0x80 > 0 { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + p.MBit = true + p.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]}) + } else { + p.PictureID = uint16(pid) + } + } + // Check if TL0PICIDX is present + if L { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + p.TlzIdx = idx + p.TL0PICIDXPresent = 1 + + if int(idx) >= payloadLen { + return errShortPacket + } + p.TL0PICIDX = payload[idx] + } + if T || K { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + if T { + p.TIDPresent = 1 + p.TID = (payload[idx] & 0xc0) >> 6 + p.Y = (payload[idx] & 0x20) >> 5 + } + if K { + p.KEYIDXPresent = 1 + p.KEYIDX = payload[idx] & 0x1f + } + } + if idx >= payloadLen { + return errShortPacket + } + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + // Check is packet is a keyframe by looking at P bit in vp8 payload + p.IsKeyFrame = payload[idx]&0x01 == 0 && S + } else { + idx++ + if payloadLen < idx+1 { + return errShortPacket + } + // Check is packet is a keyframe by looking at P bit in vp8 payload + p.IsKeyFrame = payload[idx]&0x01 == 0 && S + } + p.HeaderSize = idx + return nil +} + +func (v *VP8) MarshalTo(buf []byte) error { + if len(buf) < v.HeaderSize { + return errShortPacket + } + + idx := 0 + buf[idx] = v.FirstByte + if (v.PictureIDPresent + v.TL0PICIDXPresent + v.TIDPresent + v.KEYIDXPresent) != 0 { + buf[idx] |= 0x80 // X bit + idx++ + buf[idx] = byte(v.PictureIDPresent<<7) | byte(v.TL0PICIDXPresent<<6) | byte(v.TIDPresent<<5) | byte(v.KEYIDXPresent<<4) + idx++ + if v.PictureIDPresent == 1 { + if v.MBit { + buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f) + buf[idx+1] = byte(v.PictureID & 0xff) + idx += 2 + } else { + buf[idx] = byte(v.PictureID) + idx++ + } + } + if v.TL0PICIDXPresent == 1 { + buf[idx] = byte(v.TL0PICIDX) + idx++ + } + if v.TIDPresent == 1 || v.KEYIDXPresent == 1 { + buf[idx] = 0 + if v.TIDPresent == 1 { + buf[idx] = byte(v.TID<<6) | byte(v.Y<<5) + } + if v.KEYIDXPresent == 1 { + buf[idx] |= byte(v.KEYIDX & 0x1f) + } + idx++ + } + } else { + buf[idx] &^= 0x80 // X bit + idx++ + } + + return nil +} + +func VP8PictureIdSizeDiff(mBit1 bool, mBit2 bool) int { + if mBit1 == mBit2 { + return 0 + } + + if mBit1 { + return 1 + } + + return -1 +} + +// isH264Keyframe detects if h264 payload is a keyframe +// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45 +// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU +func isH264Keyframe(payload []byte) bool { + if len(payload) < 1 { + return false + } + nalu := payload[0] & 0x1F + if nalu == 0 { + // reserved + return false + } else if nalu <= 23 { + // simple NALU + return nalu == 5 + } else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 { + // STAP-A, STAP-B, MTAP16 or MTAP24 + i := 1 + if nalu == 25 || nalu == 26 || nalu == 27 { + // skip DON + i += 2 + } + for i < len(payload) { + if i+2 > len(payload) { + return false + } + length := uint16(payload[i])<<8 | + uint16(payload[i+1]) + i += 2 + if i+int(length) > len(payload) { + return false + } + offset := 0 + if nalu == 26 { + offset = 3 + } else if nalu == 27 { + offset = 4 + } + if offset >= int(length) { + return false + } + n := payload[i+offset] & 0x1F + if n == 7 { + return true + } else if n >= 24 { + // is this legal? + Logger.V(0).Info("Non-simple NALU within a STAP") + } + i += int(length) + } + if i == len(payload) { + return false + } + return false + } else if nalu == 28 || nalu == 29 { + // FU-A or FU-B + if len(payload) < 2 { + return false + } + if (payload[1] & 0x80) == 0 { + // not a starting fragment + return false + } + return payload[1]&0x1F == 7 + } + return false +} diff --git a/pkg/sfu/buffer/helpers_test.go b/pkg/sfu/buffer/helpers_test.go new file mode 100644 index 000000000..5cdff9359 --- /dev/null +++ b/pkg/sfu/buffer/helpers_test.go @@ -0,0 +1,94 @@ +package buffer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVP8Helper_Unmarshal(t *testing.T) { + type args struct { + payload []byte + } + tests := []struct { + name string + args args + wantErr bool + checkTemporal bool + temporalSupport bool + checkKeyFrame bool + keyFrame bool + checkPictureID bool + pictureID uint16 + checkTlzIdx bool + tlzIdx uint8 + checkTempID bool + temporalID uint8 + }{ + { + name: "Empty or nil payload must return error", + args: args{payload: []byte{}}, + wantErr: true, + }, + { + name: "Temporal must be supported by setting T bit to 1", + args: args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}}, + checkTemporal: true, + temporalSupport: true, + }, + { + name: "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1", + args: args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}}, + checkPictureID: true, + pictureID: 17, + }, + { + name: "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1", + args: args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}}, + checkPictureID: true, + pictureID: 4711, + }, + { + name: "Temporal level zero index must be present if L set to 1", + args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}}, + checkTlzIdx: true, + tlzIdx: 180, + }, + { + name: "Temporal index must be present and used if T bit set to 1", + args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}}, + checkTempID: true, + temporalID: 2, + }, + { + name: "Check if packet is a keyframe by looking at P bit set to 0", + args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}}, + checkKeyFrame: true, + keyFrame: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + p := &VP8{} + if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr { + t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.checkTemporal { + assert.Equal(t, tt.temporalSupport, p.TemporalSupported) + } + if tt.checkKeyFrame { + assert.Equal(t, tt.keyFrame, p.IsKeyFrame) + } + if tt.checkPictureID { + assert.Equal(t, tt.pictureID, p.PictureID) + } + if tt.checkTlzIdx { + assert.Equal(t, tt.tlzIdx, p.TL0PICIDX) + } + if tt.checkTempID { + assert.Equal(t, tt.temporalID, p.TID) + } + }) + } +} diff --git a/pkg/sfu/buffer/nack.go b/pkg/sfu/buffer/nack.go new file mode 100644 index 000000000..1ce891c95 --- /dev/null +++ b/pkg/sfu/buffer/nack.go @@ -0,0 +1,106 @@ +package buffer + +import ( + "sort" + + "github.com/pion/rtcp" +) + +const maxNackTimes = 3 // Max number of times a packet will be NACKed +const maxNackCache = 100 // Max NACK sn the sfu will keep reference + +type nack struct { + sn uint32 + nacked uint8 +} + +type nackQueue struct { + nacks []nack + kfSN uint32 +} + +func newNACKQueue() *nackQueue { + return &nackQueue{ + nacks: make([]nack, 0, maxNackCache+1), + } +} + +func (n *nackQueue) remove(extSN uint32) { + i := sort.Search(len(n.nacks), func(i int) bool { return n.nacks[i].sn >= extSN }) + if i >= len(n.nacks) || n.nacks[i].sn != extSN { + return + } + copy(n.nacks[i:], n.nacks[i+1:]) + n.nacks = n.nacks[:len(n.nacks)-1] +} + +func (n *nackQueue) push(extSN uint32) { + i := sort.Search(len(n.nacks), func(i int) bool { return n.nacks[i].sn >= extSN }) + if i < len(n.nacks) && n.nacks[i].sn == extSN { + return + } + + nck := nack{ + sn: extSN, + nacked: 0, + } + if i == len(n.nacks) { + n.nacks = append(n.nacks, nck) + } else { + n.nacks = append(n.nacks[:i+1], n.nacks[i:]...) + n.nacks[i] = nck + } + + if len(n.nacks) >= maxNackCache { + copy(n.nacks, n.nacks[1:]) + } +} + +func (n *nackQueue) pairs(headSN uint32) ([]rtcp.NackPair, bool) { + if len(n.nacks) == 0 { + return nil, false + } + i := 0 + askKF := false + var np rtcp.NackPair + var nps []rtcp.NackPair + lostIdx := -1 + for _, nck := range n.nacks { + if nck.nacked >= maxNackTimes { + if nck.sn > n.kfSN { + n.kfSN = nck.sn + askKF = true + } + continue + } + if nck.sn >= headSN-2 { + n.nacks[i] = nck + i++ + continue + } + n.nacks[i] = nack{ + sn: nck.sn, + nacked: nck.nacked + 1, + } + i++ + + // first nackpair or need a new nackpair + if lostIdx < 0 || nck.sn > n.nacks[lostIdx].sn+16 { + if lostIdx >= 0 { + nps = append(nps, np) + } + np.PacketID = uint16(nck.sn) + np.LostPackets = 0 + lostIdx = i - 1 + continue + } + np.LostPackets |= 1 << ((nck.sn) - n.nacks[lostIdx].sn - 1) + } + + // append last nackpair + if lostIdx != -1 { + nps = append(nps, np) + } + n.nacks = n.nacks[:i] + return nps, askKF +} diff --git a/pkg/sfu/buffer/nack_test.go b/pkg/sfu/buffer/nack_test.go new file mode 100644 index 000000000..97d9f4e5a --- /dev/null +++ b/pkg/sfu/buffer/nack_test.go @@ -0,0 +1,196 @@ +package buffer + +import ( + "math/rand" + "reflect" + "testing" + "time" + + "github.com/pion/rtcp" + "github.com/stretchr/testify/assert" +) + +func Test_nackQueue_pairs(t *testing.T) { + type fields struct { + nacks []nack + } + tests := []struct { + name string + fields fields + args []uint32 + want []rtcp.NackPair + }{ + { + name: "Must return correct single pairs pair", + fields: fields{ + nacks: nil, + }, + args: []uint32{1, 2, 4, 5}, + want: []rtcp.NackPair{{ + PacketID: 1, + LostPackets: 13, + }}, + }, + { + name: "Must return correct pair wrap", + fields: fields{ + nacks: nil, + }, + args: []uint32{65536, 65538, 65540, 65541, 65566, 65568}, // wrap around 65533,2,4,5 + want: []rtcp.NackPair{{ + PacketID: 0, // 65536 + LostPackets: 1<<4 + 1<<3 + 1<<1, + }, + { + PacketID: 30, // 65566 + LostPackets: 1 << 1, + }}, + }, + { + name: "Must return 2 pairs pair", + fields: fields{ + nacks: nil, + }, + args: []uint32{1, 2, 4, 5, 20, 22, 24, 27}, + want: []rtcp.NackPair{ + { + PacketID: 1, + LostPackets: 13, + }, + { + PacketID: 20, + LostPackets: 74, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n := &nackQueue{ + nacks: tt.fields.nacks, + } + for _, sn := range tt.args { + n.push(sn) + } + got, _ := n.pairs(75530) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("pairs() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_nackQueue_push(t *testing.T) { + type fields struct { + nacks []nack + } + type args struct { + sn []uint32 + } + tests := []struct { + name string + fields fields + args args + want []uint32 + }{ + { + name: "Must keep packet order", + fields: fields{ + nacks: make([]nack, 0, 10), + }, + args: args{ + sn: []uint32{3, 4, 1, 5, 8, 7, 5}, + }, + want: []uint32{1, 3, 4, 5, 7, 8}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n := &nackQueue{ + nacks: tt.fields.nacks, + } + for _, sn := range tt.args.sn { + n.push(sn) + } + var newSN []uint32 + for _, sn := range n.nacks { + newSN = append(newSN, sn.sn) + } + assert.Equal(t, tt.want, newSN) + }) + } +} + +func Test_nackQueue(t *testing.T) { + type fields struct { + nacks []nack + } + type args struct { + sn []uint32 + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "Must keep packet order", + fields: fields{ + nacks: make([]nack, 0, 10), + }, + args: args{ + sn: []uint32{3, 4, 1, 5, 8, 7, 5}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n := nackQueue{} + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := 0; i < 100; i++ { + assert.NotPanics(t, func() { + n.push(uint32(r.Intn(60000))) + n.remove(uint32(r.Intn(60000))) + n.pairs(60001) + }) + } + }) + } +} + +func Test_nackQueue_remove(t *testing.T) { + type args struct { + sn []uint32 + } + tests := []struct { + name string + args args + want []uint32 + }{ + { + name: "Must keep packet order", + args: args{ + sn: []uint32{3, 4, 1, 5, 8, 7, 5}, + }, + want: []uint32{1, 3, 4, 7, 8}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n := nackQueue{} + for _, sn := range tt.args.sn { + n.push(sn) + } + n.remove(5) + var newSN []uint32 + for _, sn := range n.nacks { + newSN = append(newSN, sn.sn) + } + assert.Equal(t, tt.want, newSN) + }) + } +} diff --git a/pkg/sfu/buffer/rtcpreader.go b/pkg/sfu/buffer/rtcpreader.go new file mode 100644 index 000000000..7472a2e96 --- /dev/null +++ b/pkg/sfu/buffer/rtcpreader.go @@ -0,0 +1,44 @@ +package buffer + +import ( + "io" + "sync/atomic" +) + +type RTCPReader struct { + ssrc uint32 + closed atomicBool + onPacket atomic.Value //func([]byte) + onClose func() +} + +func NewRTCPReader(ssrc uint32) *RTCPReader { + return &RTCPReader{ssrc: ssrc} +} + +func (r *RTCPReader) Write(p []byte) (n int, err error) { + if r.closed.get() { + err = io.EOF + return + } + if f, ok := r.onPacket.Load().(func([]byte)); ok { + f(p) + } + return +} + +func (r *RTCPReader) OnClose(fn func()) { + r.onClose = fn +} + +func (r *RTCPReader) Close() error { + r.closed.set(true) + r.onClose() + return nil +} + +func (r *RTCPReader) OnPacket(f func([]byte)) { + r.onPacket.Store(f) +} + +func (r *RTCPReader) Read(_ []byte) (n int, err error) { return } diff --git a/pkg/sfu/datachannel.go b/pkg/sfu/datachannel.go new file mode 100644 index 000000000..f292a81f4 --- /dev/null +++ b/pkg/sfu/datachannel.go @@ -0,0 +1,81 @@ +package sfu + +import ( + "context" + + "github.com/pion/webrtc/v3" +) + +type ( + // Datachannel is a wrapper to define middlewares executed on defined label. + // The datachannels created will be negotiated on join to all peers that joins + // the SFU. + Datachannel struct { + Label string + middlewares []func(MessageProcessor) MessageProcessor + onMessage func(ctx context.Context, args ProcessArgs) + } + + ProcessArgs struct { + Peer Peer + Message webrtc.DataChannelMessage + DataChannel *webrtc.DataChannel + } + + Middlewares []func(MessageProcessor) MessageProcessor + + MessageProcessor interface { + Process(ctx context.Context, args ProcessArgs) + } + + ProcessFunc func(ctx context.Context, args ProcessArgs) + + chainHandler struct { + middlewares Middlewares + Last MessageProcessor + current MessageProcessor + } +) + +// Use adds the middlewares to the current Datachannel. +// The middlewares are going to be executed before the OnMessage event fires. +func (dc *Datachannel) Use(middlewares ...func(MessageProcessor) MessageProcessor) { + dc.middlewares = append(dc.middlewares, middlewares...) +} + +// OnMessage sets the message callback for the datachannel, the event is fired +// after all the middlewares have processed the message. +func (dc *Datachannel) OnMessage(fn func(ctx context.Context, args ProcessArgs)) { + dc.onMessage = fn +} + +func (p ProcessFunc) Process(ctx context.Context, args ProcessArgs) { + p(ctx, args) +} + +func (mws Middlewares) Process(h MessageProcessor) MessageProcessor { + return &chainHandler{mws, h, chain(mws, h)} +} + +func (mws Middlewares) ProcessFunc(h MessageProcessor) MessageProcessor { + return &chainHandler{mws, h, chain(mws, h)} +} + +func newDCChain(m []func(p MessageProcessor) MessageProcessor) Middlewares { + return Middlewares(m) +} + +func (c *chainHandler) Process(ctx context.Context, args ProcessArgs) { + c.current.Process(ctx, args) +} + +func chain(mws []func(processor MessageProcessor) MessageProcessor, last MessageProcessor) MessageProcessor { + if len(mws) == 0 { + return last + } + h := mws[len(mws)-1](last) + for i := len(mws) - 2; i >= 0; i-- { + h = mws[i](h) + } + return h +} diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go new file mode 100644 index 000000000..c7174d69a --- /dev/null +++ b/pkg/sfu/downtrack.go @@ -0,0 +1,1668 @@ +package sfu + +import ( + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/sdp/v3" + "github.com/pion/transport/packetio" + "github.com/pion/webrtc/v3" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +// DownTrackType determines the type of track +type DownTrackType int + +const ( + SimpleDownTrack DownTrackType = iota + 1 + SimulcastDownTrack +) + +const ( + RTPPaddingMaxPayloadSize = 255 + RTPPaddingEstimatedHeaderSize = 20 +) + +var ( + ErrOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") + ErrPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") + ErrDuplicatePacket = errors.New("duplicate packet") + ErrPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") + ErrNotVP8 = errors.New("not VP8") + ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache") + ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer") +) + +type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) + +// DownTrack implements TrackLocal, is the track used to write packets +// to SFU Subscriber, the track handle the packets for simple, simulcast +// and SVC Publisher. +type DownTrack struct { + id string + peerID string + bound atomicBool + mime string + ssrc uint32 + streamID string + maxTrack int + payloadType uint8 + sequencer *sequencer + trackType DownTrackType + bufferFactory *buffer.Factory + payload *[]byte + + currentSpatialLayer atomicInt32 + targetSpatialLayer atomicInt32 + temporalLayer atomicInt32 + + enabled atomicBool + reSync atomicBool + lastSSRC atomicUint32 + + munger *Munger + vp8Munger *VP8Munger + + simulcast simulcastTrackHelpers + maxSpatialLayer atomicInt32 + maxTemporalLayer atomicInt32 + + codec webrtc.RTPCodecCapability + rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter + receiver Receiver + transceiver *webrtc.RTPTransceiver + writeStream webrtc.TrackLocalWriter + onCloseHandler func() + onBind func() + receiverReportListeners []ReceiverReportListener + listenerLock sync.RWMutex + closeOnce sync.Once + + // Report helpers + octetCount atomicUint32 + packetCount atomicUint32 + lossFraction atomicUint8 + + // Debug info + lastPli atomicInt64 + lastRTP atomicInt64 + pktsMuted atomicUint32 + pktsDropped atomicUint32 + pktsBandwidthConstrainedDropped atomicUint32 + + maxPacketTs uint32 + + bandwidthConstrainedMuted atomicBool + + // RTCP callbacks + onREMB func(dt *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) + + // simulcast layer availability change callback + onAvailableLayersChanged func(dt *DownTrack, layerAdded bool) + + // subscription change callback + onSubscriptionChanged func(dt *DownTrack) + + // packet sent callback + onPacketSent func(dt *DownTrack, size int) +} + +// NewDownTrack returns a DownTrack. +func NewDownTrack(c webrtc.RTPCodecCapability, r Receiver, bf *buffer.Factory, peerID string, mt int) (*DownTrack, error) { + return &DownTrack{ + id: r.TrackID(), + peerID: peerID, + maxTrack: mt, + streamID: r.StreamID(), + bufferFactory: bf, + receiver: r, + codec: c, + munger: NewMunger(), + }, nil +} + +// Bind is called by the PeerConnection after negotiation is complete +// This asserts that the code requested is supported by the remote peer. +// If so it setups all the state (SSRC and PayloadType) to have a call +func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { + parameters := webrtc.RTPCodecParameters{RTPCodecCapability: d.codec} + if codec, err := codecParametersFuzzySearch(parameters, t.CodecParameters()); err == nil { + d.ssrc = uint32(t.SSRC()) + d.payloadType = uint8(codec.PayloadType) + d.writeStream = t.WriteStream() + d.mime = strings.ToLower(codec.MimeType) + d.reSync.set(true) + d.enabled.set(true) + d.bandwidthConstrainedMute(false) + if rr := d.bufferFactory.GetOrNew(packetio.RTCPBufferPacket, uint32(t.SSRC())).(*buffer.RTCPReader); rr != nil { + rr.OnPacket(func(pkt []byte) { + d.handleRTCP(pkt) + }) + } + if strings.HasPrefix(d.codec.MimeType, "video/") { + d.sequencer = newSequencer(d.maxTrack) + } + if d.onBind != nil { + d.onBind() + } + d.bound.set(true) + return codec, nil + } + return webrtc.RTPCodecParameters{}, webrtc.ErrUnsupportedCodec +} + +// Unbind implements the teardown logic when the track is no longer needed. This happens +// because a track has been stopped. +func (d *DownTrack) Unbind(_ webrtc.TrackLocalContext) error { + d.bound.set(false) + d.receiver.DeleteDownTrack(d.peerID) + return nil +} + +// ID is the unique identifier for this Track. This should be unique for the +// stream, but doesn't have to globally unique. A common example would be 'audio' or 'video' +// and StreamID would be 'desktop' or 'webcam' +func (d *DownTrack) ID() string { return d.id } + +// Codec returns current track codec capability +func (d *DownTrack) Codec() webrtc.RTPCodecCapability { return d.codec } + +// StreamID is the group this track belongs too. This must be unique +func (d *DownTrack) StreamID() string { return d.streamID } + +// Sets RTP header extensions for this track +func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { + d.rtpHeaderExtensions = rtpHeaderExtensions +} + +// Kind controls if this TrackLocal is audio or video +func (d *DownTrack) Kind() webrtc.RTPCodecType { + switch { + case strings.HasPrefix(d.codec.MimeType, "audio/"): + return webrtc.RTPCodecTypeAudio + case strings.HasPrefix(d.codec.MimeType, "video/"): + return webrtc.RTPCodecTypeVideo + default: + return webrtc.RTPCodecType(0) + } +} + +func (d *DownTrack) SSRC() uint32 { + return d.ssrc +} + +func (d *DownTrack) Stop() error { + if d.transceiver != nil { + return d.transceiver.Stop() + } + return fmt.Errorf("d.transceiver not exists") +} + +func (d *DownTrack) SetTransceiver(transceiver *webrtc.RTPTransceiver) { + d.transceiver = transceiver +} + +func (d *DownTrack) MaybeTranslateVP8(pkt *rtp.Packet, meta packetMeta) error { + if d.vp8Munger == nil { + return nil + } + + var incomingVP8 buffer.VP8 + if err := incomingVP8.Unmarshal(pkt.Payload); err != nil { + return err + } + + translatedVP8 := meta.unpackVP8() + payload, err := d.translateVP8Packet(pkt, &incomingVP8, translatedVP8) + if err != nil { + return err + } + + pkt.Payload = payload + return nil +} + +// Writes RTP header extensions of track +func (d *DownTrack) WriteRTPHeaderExtensions(hdr *rtp.Header) error { + // clear out extensions that may have been in the forwarded header + hdr.Extension = false + hdr.ExtensionProfile = 0 + hdr.Extensions = []rtp.Extension{} + + for _, ext := range d.rtpHeaderExtensions { + if ext.URI != sdp.ABSSendTimeURI { + // supporting only abs-send-time + continue + } + + sendTime := rtp.NewAbsSendTimeExtension(time.Now()) + b, err := sendTime.Marshal() + if err != nil { + return err + } + + err = hdr.SetExtension(uint8(ext.ID), b) + if err != nil { + return err + } + } + + return nil +} + +// WriteRTP writes a RTP Packet to the DownTrack +func (d *DownTrack) WriteRTP(p *buffer.ExtPacket, layer int32) error { + d.lastRTP.set(time.Now().UnixNano()) + + if !d.bound.get() { + return nil + } + if !d.enabled.get() { + d.pktsMuted.add(1) + return nil + } + + if d.bandwidthConstrainedMuted.get() { + d.pktsBandwidthConstrainedDropped.add(1) + return nil + } + + switch d.trackType { + case SimpleDownTrack: + return d.writeSimpleRTP(p) + case SimulcastDownTrack: + return d.writeSimulcastRTP(p, layer) + } + return nil +} + +// WritePaddingRTP tries to write as many padding only RTP packets as necessary +// to satisfy given size to the DownTrack +func (d *DownTrack) WritePaddingRTP(bytesToSend int) int { + // LK-TODO-START + // Potentially write padding even if muted. Given that padding + // can be sent only on frame boudaries, writing on disabled tracks + // will give more options. But, it is possible that forwarding stopped + // on a non-frame boundary when the track is muted. + // LK-TODO-END + if !d.enabled.get() || d.packetCount.get() == 0 { + return 0 + } + + // LK-TODO-START + // Ideally should look at header extensions negotiated for + // track and decide if padding can be sent. But, browsers behave + // in unexpected ways when using audio for bandwidth estimation and + // padding is mainly used to probe for excess available bandwidth. + // So, to be safe, limit to video tracks + // LK-TODO-END + if d.Kind() == webrtc.RTPCodecTypeAudio { + return 0 + } + + // LK-TODO Look at load balancing a la sfu.Receiver to spread across available CPUs + bytesSent := 0 + for { + size := bytesToSend + // RTP padding maximum is 255 bytes. Break it up. + // Use 20 byte as estimate of RTP header size (12 byte header + 8 byte extension) + if size > RTPPaddingMaxPayloadSize+RTPPaddingEstimatedHeaderSize { + size = RTPPaddingMaxPayloadSize + RTPPaddingEstimatedHeaderSize + } + + sn, ts, err := d.munger.UpdateAndGetPaddingSnTs() + if err != nil { + return bytesSent + } + + // LK-TODO-START + // Hold sending padding packets till first RTCP-RR is received for this RTP stream. + // That is definitive proof that the remote side knows about this RTP stream. + // The packet count check at the beginning of this function gates sending padding + // on as yet unstarted streams which is a reasonble check. + // LK-TODO-END + + // intentionally ignoring check for bandwidth constrained mute + // as padding is typically used to probe for channel capacity + // and sending it on any track achieves the purpose of probing + // the channel. + + hdr := rtp.Header{ + Version: 2, + Padding: true, + Marker: false, + PayloadType: d.payloadType, + SequenceNumber: sn, + Timestamp: ts, + SSRC: d.ssrc, + CSRC: []uint32{}, + } + + err = d.WriteRTPHeaderExtensions(&hdr) + if err != nil { + return bytesSent + } + + payloadSize := size - RTPPaddingEstimatedHeaderSize + payload := make([]byte, payloadSize) + // last byte of padding has padding size including that byte + payload[payloadSize-1] = byte(payloadSize) + + _, err = d.writeStream.WriteRTP(&hdr, payload) + if err != nil { + return bytesSent + } + + // LK-TODO - check if we should keep separate padding stats + size = hdr.MarshalSize() + len(payload) + d.UpdateStats(uint32(size)) + + // LK-TODO-START + // NACK buffer for these probe packets. + // Probably okay to absorb the NACKs for these and ignore them. + // Retransmssion is probably a sign of network congestion/badness. + // So, retransmitting padding packets is only going to make matters worse. + // LK-TODO-END + + bytesSent += size + bytesToSend -= size + if bytesToSend <= 0 { + break + } + } + + return bytesSent +} + +func (d *DownTrack) Enabled() bool { + return d.enabled.get() +} + +// Mute enables or disables media forwarding +func (d *DownTrack) Mute(val bool) { + if d.enabled.get() != val { + return + } + d.enabled.set(!val) + if val { + d.lossFraction.set(0) + d.reSync.set(val) + } + + if d.onSubscriptionChanged != nil { + d.onSubscriptionChanged(d) + } +} + +// Close track +func (d *DownTrack) Close() { + d.enabled.set(false) + d.closeOnce.Do(func() { + Logger.V(1).Info("Closing sender", "peer_id", d.peerID) + if d.payload != nil { + packetFactory.Put(d.payload) + } + if d.onCloseHandler != nil { + d.onCloseHandler() + } + }) +} + +func (d *DownTrack) SetInitialLayers(spatialLayer, temporalLayer int32) { + d.currentSpatialLayer.set(spatialLayer) + d.targetSpatialLayer.set(spatialLayer) + d.temporalLayer.set((temporalLayer << 16) | temporalLayer) +} + +func (d *DownTrack) CurrentSpatialLayer() int32 { + return d.currentSpatialLayer.get() +} + +func (d *DownTrack) TargetSpatialLayer() int32 { + return d.targetSpatialLayer.get() +} + +func (d *DownTrack) MaxSpatialLayer() int32 { + return d.maxSpatialLayer.get() +} + +// SwitchSpatialLayer switches the current layer +func (d *DownTrack) SwitchSpatialLayer(targetLayer int32, setAsMax bool) error { + // LK-TODO-START + // This gets called directly from two places outside. + // 1. From livekit-server when suscriber updates TrackSetting + // 2. From sfu.Receiver when a new up track is added + // Make a method in this struct which allows setting subscriber controls. + // StreamAllocator needs to know about subscriber setting changes. + // Only the StreamAllocator should be changing forwarded layers. + // Make a method `SetMaxSpatialLayer()` and remove `setAsMax` from this. + // Trigger callback to StreamAllocator to notify subscription change + // in that method. + // LK-TODO-END + if d.trackType != SimulcastDownTrack { + return ErrSpatialNotSupported + } + if !d.receiver.HasSpatialLayer(targetLayer) { + return ErrSpatialLayerNotFound + } + // already set + if d.CurrentSpatialLayer() == targetLayer { + return nil + } + + d.targetSpatialLayer.set(targetLayer) + if setAsMax { + d.maxSpatialLayer.set(targetLayer) + + if d.onSubscriptionChanged != nil { + d.onSubscriptionChanged(d) + } + } + return nil +} + +func (d *DownTrack) SwitchSpatialLayerDone(layer int32) { + d.currentSpatialLayer.set(layer) +} + +func (d *DownTrack) UptrackLayersChange(availableLayers []uint16, layerAdded bool) (int32, error) { + if d.trackType == SimulcastDownTrack { + currentLayer := uint16(d.CurrentSpatialLayer()) + maxLayer := uint16(d.maxSpatialLayer.get()) + + var maxFound uint16 = 0 + layerFound := false + var minFound uint16 = 0 + for _, target := range availableLayers { + if target <= maxLayer { + if target > maxFound { + maxFound = target + layerFound = true + } + } else { + if minFound > target { + minFound = target + } + } + } + var targetLayer uint16 + if layerFound { + targetLayer = maxFound + } else { + targetLayer = minFound + } + if currentLayer != targetLayer { + // LK-TODO-START + // This layer switch should be removed when StreamAllocator is used. + // Available layers change should be signalled to StreamAllocator + // and StreamAllocator will take care of adjusting allocations. + // LK-TODO-END + if err := d.SwitchSpatialLayer(int32(targetLayer), false); err != nil { + return int32(targetLayer), err + } + } + + if d.onAvailableLayersChanged != nil { + d.onAvailableLayersChanged(d, layerAdded) + } + + return int32(targetLayer), nil + } + return -1, fmt.Errorf("downtrack %s does not support simulcast", d.id) +} + +func (d *DownTrack) SwitchTemporalLayer(targetLayer int32, setAsMax bool) { + // LK-TODO-START + // See note in SwitchSpatialLayer to split out setting max layer + // into a separate method and triggering notification to StreamAllocator. + // BTW, looks like `setAsMax` is not set to true at any callsite of this API + // LK-TODO-END + if d.trackType == SimulcastDownTrack { + layer := d.temporalLayer.get() + currentLayer := uint16(layer) + currentTargetLayer := uint16(layer >> 16) + + // Don't switch until previous switch is done or canceled + if currentLayer != currentTargetLayer { + return + } + + d.temporalLayer.set((targetLayer << 16) | int32(currentLayer)) + if setAsMax { + d.maxTemporalLayer.set(targetLayer) + + if d.onSubscriptionChanged != nil { + d.onSubscriptionChanged(d) + } + } + } +} + +// OnCloseHandler method to be called on remote tracked removed +func (d *DownTrack) OnCloseHandler(fn func()) { + d.onCloseHandler = fn +} + +func (d *DownTrack) OnBind(fn func()) { + d.onBind = fn +} + +func (d *DownTrack) OnREMB(fn func(dt *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate)) { + d.onREMB = fn +} + +func (d *DownTrack) CurrentMaxLossFraction() uint8 { + return d.lossFraction.get() +} + +func (d *DownTrack) AddReceiverReportListener(listener ReceiverReportListener) { + d.listenerLock.Lock() + defer d.listenerLock.Unlock() + d.receiverReportListeners = append(d.receiverReportListeners, listener) +} + +func (d *DownTrack) OnAvailableLayersChanged(fn func(dt *DownTrack, layerAdded bool)) { + d.onAvailableLayersChanged = fn +} + +func (d *DownTrack) OnSubscriptionChanged(fn func(dt *DownTrack)) { + d.onSubscriptionChanged = fn +} + +func (d *DownTrack) OnPacketSent(fn func(dt *DownTrack, size int)) { + d.onPacketSent = fn +} + +func (d *DownTrack) AdjustAllocation(availableChannelCapacity uint64) (uint64, uint64) { + if d.Kind() == webrtc.RTPCodecTypeAudio || !d.enabled.get() { + return 0, 0 + } + + // LK-TODO for temporal preference, traverse the bitrates array the other way + optimalBandwidthNeeded := uint64(0) + brs := d.receiver.GetBitrateTemporalCumulative() + for i := d.maxSpatialLayer.get(); i >= 0; i-- { + for j := d.maxTemporalLayer.get(); j >= 0; j-- { + if brs[i][j] == 0 { + continue + } + if optimalBandwidthNeeded == 0 { + optimalBandwidthNeeded = brs[i][j] + } + if brs[i][j] < availableChannelCapacity { + d.bandwidthConstrainedMute(false) // just in case it was muted + d.SwitchSpatialLayer(int32(i), false) + d.SwitchTemporalLayer(int32(j), false) + + return brs[i][j], optimalBandwidthNeeded + } + } + } + + // no layer fits in the available channel capacity, disable the track + // LK-TODO - this should fire some callback to notify an observer that a + // track has been disabled/muted due to bandwidth constraints + d.bandwidthConstrainedMute(true) + + return 0, optimalBandwidthNeeded +} + +func (d *DownTrack) IncreaseAllocation() (bool, uint64, uint64) { + // LK-TODO-START + // This is mainly used in probing to try a slightly higher layer. + // But, if down track is not a simulcast track, then the next + // available layer (i. e. the only layer of simple track) may boost + // things by a lot (it could happen in simulcast jumps too). + // May need to take in a layer increase threshold as an argument + // (in terms of bps) and increase layer only if the jump is within + // that threshold. + // LK-TODO-END + if d.Kind() == webrtc.RTPCodecTypeAudio || !d.enabled.get() { + return false, 0, 0 + } + + currentSpatialLayer := d.CurrentSpatialLayer() + targetSpatialLayer := d.TargetSpatialLayer() + + temporalLayer := d.temporalLayer.get() + currentTemporalLayer := temporalLayer & 0x0f + targetTemporalLayer := temporalLayer >> 16 + + // if targets are still pending, don't increase + if targetSpatialLayer != currentSpatialLayer || targetTemporalLayer != currentTemporalLayer { + return false, 0, 0 + } + + // move to the next available layer + optimalBandwidthNeeded := uint64(0) + brs := d.receiver.GetBitrateTemporalCumulative() + for i := d.maxSpatialLayer.get(); i >= 0; i-- { + for j := d.maxTemporalLayer.get(); j >= 0; j-- { + if brs[i][j] == 0 { + continue + } + if optimalBandwidthNeeded == 0 { + optimalBandwidthNeeded = brs[i][j] + break + } + } + + if optimalBandwidthNeeded != 0 { + break + } + } + + if d.bandwidthConstrainedMuted.get() { + // try the lowest spatial and temporal layer if available + // LK-TODO-START + // note that this will never be zero because we do not track + // layer 0 in available layers. So, this will need fixing. + // LK-TODO-END + if brs[0][0] == 0 { + // no feed available + return false, 0, 0 + } + + d.bandwidthConstrainedMute(false) + d.SwitchSpatialLayer(int32(0), false) + d.SwitchTemporalLayer(int32(0), false) + return true, brs[0][0], optimalBandwidthNeeded + } + + // try moving temporal layer up in the current spatial layer + // LK-TODO currentTemporalLayer may be outside available range because of inital value being out of range, fix it + nextTemporalLayer := currentTemporalLayer + 1 + if nextTemporalLayer <= d.maxTemporalLayer.get() && brs[currentSpatialLayer][nextTemporalLayer] > 0 { + d.SwitchTemporalLayer(nextTemporalLayer, false) + return true, brs[currentSpatialLayer][nextTemporalLayer], optimalBandwidthNeeded + } + + // try moving spatial layer up if already at max temporal layer of current spatial layer + // LK-TODO currentTemporalLayer may be outside available range because of inital value being out of range, fix it + nextSpatialLayer := currentSpatialLayer + 1 + if nextSpatialLayer <= d.maxSpatialLayer.get() && brs[nextSpatialLayer][0] > 0 { + d.SwitchSpatialLayer(nextSpatialLayer, false) + d.SwitchTemporalLayer(0, false) + return true, brs[nextSpatialLayer][0], optimalBandwidthNeeded + } + + return false, 0, 0 +} + +func (d *DownTrack) CreateSourceDescriptionChunks() []rtcp.SourceDescriptionChunk { + if !d.bound.get() { + return nil + } + return []rtcp.SourceDescriptionChunk{ + { + Source: d.ssrc, + Items: []rtcp.SourceDescriptionItem{{ + Type: rtcp.SDESCNAME, + Text: d.streamID, + }}, + }, { + Source: d.ssrc, + Items: []rtcp.SourceDescriptionItem{{ + Type: rtcp.SDESType(15), + Text: d.transceiver.Mid(), + }}, + }, + } +} + +func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { + if !d.bound.get() { + return nil + } + + srRTP, srNTP := d.receiver.GetSenderReportTime(d.CurrentSpatialLayer()) + if srRTP == 0 { + return nil + } + + now := time.Now() + nowNTP := toNtpTime(now) + + diff := (uint64(now.Sub(ntpTime(srNTP).Time())) * uint64(d.codec.ClockRate)) / uint64(time.Second) + if diff < 0 { + diff = 0 + } + octets, packets := d.getSRStats() + + return &rtcp.SenderReport{ + SSRC: d.ssrc, + NTPTime: uint64(nowNTP), + RTPTime: srRTP + uint32(diff), + PacketCount: packets, + OctetCount: octets, + } +} + +func (d *DownTrack) UpdateStats(packetLen uint32) { + d.octetCount.add(packetLen) + d.packetCount.add(1) +} + +// bandwidthConstrainedMute enables or disables media forwarding dictated by channel bandwidth constraints +func (d *DownTrack) bandwidthConstrainedMute(val bool) { + if d.bandwidthConstrainedMuted.get() == val { + return + } + d.bandwidthConstrainedMuted.set(val) + if val { + d.reSync.set(val) + } +} + +func (d *DownTrack) writeSimpleRTP(extPkt *buffer.ExtPacket) error { + if d.reSync.get() { + if d.Kind() == webrtc.RTPCodecTypeVideo { + if !extPkt.KeyFrame { + d.receiver.SendRTCP([]rtcp.Packet{ + &rtcp.PictureLossIndication{SenderSSRC: d.ssrc, MediaSSRC: extPkt.Packet.SSRC}, + }) + d.lastPli.set(time.Now().UnixNano()) + d.pktsDropped.add(1) + return nil + } + } + if d.packetCount.get() > 0 { + // LK-TODO-START + // TS offset of 1 is not accurate. It should ideally + // be driven by packetization of the incoming track. + // But, this handles track switch on a simple track scenario. + // It is not a supported use case. So, it is okay. But, if + // we support switch track (i. e. same down track switches + // to a different up track), this needs to be looked at. + // LK-TODO-END + d.munger.UpdateSnTsOffsets(extPkt, 1, 1) + } else { + d.munger.SetLastSnTs(extPkt) + if d.mime == "video/vp8" { + if vp8, ok := extPkt.Payload.(buffer.VP8); ok { + if vp8.TIDPresent == 1 { + d.vp8Munger = NewVP8Munger() + } + } + } + if d.vp8Munger != nil { + d.vp8Munger.SetLast(extPkt) + } + } + d.lastSSRC.set(extPkt.Packet.SSRC) + d.reSync.set(false) + } + + payload := extPkt.Packet.Payload + + var ( + translatedVP8 *buffer.VP8 + err error + ) + if d.vp8Munger != nil && len(payload) > 0 { + // LK-TODO-START + // Errors below do not update sequence number. That is a problem if the stream + // is expected to continue past the error. The translation should not error out. + // But, if there is a legitimate error case and the stream can continue beyond + // that, the sequence numbers should be updated to ensure that subsequent packet + // translations works fine and produce proper translated sequence numbers. + // LK-TODO-END + translatedVP8, err = d.vp8Munger.UpdateAndGet(extPkt, d.temporalLayer.get()>>16) + if err != nil { + if err == ErrFilteredVP8TemporalLayer || err == ErrOutOfOrderVP8PictureIdCacheMiss { + if err == ErrFilteredVP8TemporalLayer { + // filtered temporal layer, update sequence number offset to prevent holes + d.munger.PacketDropped(extPkt) + } + d.pktsDropped.add(1) + return nil + } + + d.pktsDropped.add(1) + return err + } + + incomingVP8, ok := extPkt.Payload.(buffer.VP8) + if !ok { + d.pktsDropped.add(1) + return ErrNotVP8 + } + + payload, err = d.translateVP8Packet(&extPkt.Packet, &incomingVP8, translatedVP8) + if err != nil { + d.pktsDropped.add(1) + return err + } + } + + newSN, newTS, err := d.munger.UpdateAndGetSnTs(extPkt) + if err != nil { + if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { + return nil + } + + d.pktsDropped.add(1) + return err + } + if d.sequencer != nil { + meta := d.sequencer.push(extPkt.Packet.SequenceNumber, newSN, newTS, 0, extPkt.Head) + if meta != nil && d.vp8Munger != nil { + meta.packVP8(translatedVP8) + } + } + + // LK-TODO maybe include RTP header size also + d.UpdateStats(uint32(len(payload))) + + hdr := extPkt.Packet.Header + hdr.PayloadType = d.payloadType + hdr.Timestamp = newTS + hdr.SequenceNumber = newSN + hdr.SSRC = d.ssrc + + err = d.WriteRTPHeaderExtensions(&hdr) + if err != nil { + return err + } + + _, err = d.writeStream.WriteRTP(&hdr, payload) + if err == nil && d.onPacketSent != nil { + d.onPacketSent(d, hdr.MarshalSize()+len(payload)) + } + + return err +} + +func (d *DownTrack) writeSimulcastRTP(extPkt *buffer.ExtPacket, layer int32) error { + // Check if packet SSRC is different from before + // if true, the video source changed + csl := d.CurrentSpatialLayer() + if csl != layer { + return nil + } + + lastSSRC := d.lastSSRC.get() + reSync := d.reSync.get() + if lastSSRC != extPkt.Packet.SSRC || reSync { + // Wait for a keyframe to sync new source + if reSync && !extPkt.KeyFrame { + // Packet is not a keyframe, discard it + // LK-TODO-START + // Some of this happens is happening in sfu.Receiver also. + // If performance is not a concern, sfu.Receiver should send + // all the packets to down tracks and down track should be + // the only one deciding whether to switch/forward/drop + // LK-TODO-END + d.receiver.SendRTCP([]rtcp.Packet{ + &rtcp.PictureLossIndication{SenderSSRC: d.ssrc, MediaSSRC: extPkt.Packet.SSRC}, + }) + d.lastPli.set(time.Now().UnixNano()) + d.pktsDropped.add(1) + return nil + } + + if reSync && d.simulcast.lTSCalc.get() != 0 { + d.simulcast.lTSCalc.set(extPkt.Arrival) + } + + d.lastSSRC.set(extPkt.Packet.SSRC) + d.reSync.set(false) + } + + // LK-TODO-START + // The below offset calculation is not technically correct. + // Timestamps based on the system time of an intermediate box like + // SFU is not going to be accurate. Packets arrival/processing + // are subject to vagaries of network delays, SFU processing etc. + // But, the correct way is a lot harder. Will have to + // look at RTCP SR to get timestamps and figure out alignment + // of layers and use that during layer switch. That can + // get tricky. Given the complexity of that approach, maybe + // this is just fine till it is not :-). + // LK-TODO-END + + // Compute how much time passed between the old RTP extPkt + // and the current packet, and fix timestamp on source change + lTSCalc := d.simulcast.lTSCalc.get() + if lTSCalc != 0 && lastSSRC != extPkt.Packet.SSRC { + tDiff := (extPkt.Arrival - lTSCalc) / 1e6 + // LK-TODO-START + // this is assuming clock rate of 90000. + // Should be fine for video, but ideally should use ClockRate of the track + // LK-TODO-END + td := uint32((tDiff * 90) / 1000) + if td == 0 { + td = 1 + } + d.munger.UpdateSnTsOffsets(extPkt, 1, td) + if d.vp8Munger != nil { + d.vp8Munger.UpdateOffsets(extPkt) + } + } else if lTSCalc == 0 { + d.munger.SetLastSnTs(extPkt) + if d.mime == "video/vp8" { + if _, ok := extPkt.Payload.(buffer.VP8); ok { + // need a munger for simulcast with or without temporal filtering + d.vp8Munger = NewVP8Munger() + } + } + if d.vp8Munger != nil { + d.vp8Munger.SetLast(extPkt) + } + } + + payload := extPkt.Packet.Payload + + var ( + translatedVP8 *buffer.VP8 + err error + ) + if d.vp8Munger != nil && len(payload) > 0 { + // LK-TODO-START + // Errors below do not update sequence number. That is a problem if the stream + // is expected to continue past the error. The translation should not error out. + // But, if there is a legitimate error case and the stream can continue beyond + // that, the sequence numbers should be updated to ensure that subsequent packet + // translations works fine and produce proper translated sequence numbers. + // LK-TODO-END + translatedVP8, err = d.vp8Munger.UpdateAndGet(extPkt, d.temporalLayer.get()>>16) + if err != nil { + if err == ErrFilteredVP8TemporalLayer || err == ErrOutOfOrderVP8PictureIdCacheMiss { + if err == ErrFilteredVP8TemporalLayer { + // filtered temporal layer, update sequence number offset to prevent holes + d.munger.PacketDropped(extPkt) + } + d.pktsDropped.add(1) + return nil + } + + d.pktsDropped.add(1) + return err + } + + incomingVP8, ok := extPkt.Payload.(buffer.VP8) + if !ok { + d.pktsDropped.add(1) + return ErrNotVP8 + } + + payload, err = d.translateVP8Packet(&extPkt.Packet, &incomingVP8, translatedVP8) + if err != nil { + d.pktsDropped.add(1) + return err + } + } + + newSN, newTS, err := d.munger.UpdateAndGetSnTs(extPkt) + if err != nil { + if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { + return nil + } + + d.pktsDropped.add(1) + return err + } + if d.sequencer != nil { + meta := d.sequencer.push(extPkt.Packet.SequenceNumber, newSN, newTS, uint8(csl), extPkt.Head) + if meta != nil && d.vp8Munger != nil { + meta.packVP8(translatedVP8) + } + } + + // LK-TODO - maybe include RTP header? + d.UpdateStats(uint32(len(payload))) + + // Update base + d.simulcast.lTSCalc.set(extPkt.Arrival) + + // Update extPkt headers + hdr := extPkt.Packet.Header + hdr.SequenceNumber = newSN + hdr.Timestamp = newTS + hdr.SSRC = d.ssrc + hdr.PayloadType = d.payloadType + + err = d.WriteRTPHeaderExtensions(&hdr) + if err != nil { + return err + } + + _, err = d.writeStream.WriteRTP(&hdr, payload) + if err == nil && d.onPacketSent != nil { + d.onPacketSent(d, hdr.MarshalSize()+len(payload)) + } + + return err +} + +func (d *DownTrack) handleRTCP(bytes []byte) { + // LK-TODO - should probably handle RTCP even if muted + if !d.enabled.get() { + return + } + + pkts, err := rtcp.Unmarshal(bytes) + if err != nil { + Logger.Error(err, "Unmarshal rtcp receiver packets err") + } + + var fwdPkts []rtcp.Packet + pliOnce := true + firOnce := true + + var ( + maxRatePacketLoss uint8 + expectedMinBitrate uint64 + ) + + ssrc := d.lastSSRC.get() + if ssrc == 0 { + return + } + + for _, pkt := range pkts { + switch p := pkt.(type) { + case *rtcp.PictureLossIndication: + if pliOnce { + d.lastPli.set(time.Now().UnixNano()) + p.MediaSSRC = ssrc + p.SenderSSRC = d.ssrc + fwdPkts = append(fwdPkts, p) + pliOnce = false + } + case *rtcp.FullIntraRequest: + if firOnce { + p.MediaSSRC = ssrc + p.SenderSSRC = d.ssrc + fwdPkts = append(fwdPkts, p) + firOnce = false + } + case *rtcp.ReceiverEstimatedMaximumBitrate: + // LK-TODO-START + // Remove expectedMinBitrate calculation when switching code + // to StreamAllocator based layer control + // LK-TODO-END + if expectedMinBitrate == 0 || expectedMinBitrate > uint64(p.Bitrate) { + expectedMinBitrate = uint64(p.Bitrate) + } + if d.onREMB != nil { + d.onREMB(d, p) + } + case *rtcp.ReceiverReport: + // create new receiver report w/ only valid reception reports + rr := &rtcp.ReceiverReport{ + SSRC: p.SSRC, + ProfileExtensions: p.ProfileExtensions, + } + for _, r := range p.Reports { + if r.SSRC != d.ssrc { + continue + } + rr.Reports = append(rr.Reports, r) + if maxRatePacketLoss == 0 || maxRatePacketLoss < r.FractionLost { + maxRatePacketLoss = r.FractionLost + } + } + d.lossFraction.set(maxRatePacketLoss) + if len(rr.Reports) > 0 { + d.listenerLock.RLock() + for _, l := range d.receiverReportListeners { + l(d, rr) + } + d.listenerLock.RUnlock() + } + case *rtcp.TransportLayerNack: + var nackedPackets []packetMeta + for _, pair := range p.Nacks { + nackedPackets = append(nackedPackets, d.sequencer.getSeqNoPairs(pair.PacketList())...) + } + if err = d.receiver.RetransmitPackets(d, nackedPackets); err != nil { + return + } + } + } + + // LK-TODO: Remove when switching to StreamAllocator based layer control + if d.trackType == SimulcastDownTrack && (maxRatePacketLoss != 0 || expectedMinBitrate != 0) { + d.handleLayerChange(maxRatePacketLoss, expectedMinBitrate) + } + + if len(fwdPkts) > 0 { + d.receiver.SendRTCP(fwdPkts) + } +} + +// LK-TODO: Remove when switching to StreamAllocator based layer control +func (d *DownTrack) handleLayerChange(maxRatePacketLoss uint8, expectedMinBitrate uint64) { + currentSpatialLayer := d.CurrentSpatialLayer() + targetSpatialLayer := d.TargetSpatialLayer() + + temporalLayer := d.temporalLayer.get() + currentTemporalLayer := temporalLayer & 0x0f + targetTemporalLayer := temporalLayer >> 16 + + if targetSpatialLayer == currentSpatialLayer && currentTemporalLayer == targetTemporalLayer { + if time.Now().After(d.simulcast.switchDelay) { + brs := d.receiver.GetBitrate() + cbr := brs[currentSpatialLayer] + mtl := d.receiver.GetMaxTemporalLayer() + mctl := mtl[currentSpatialLayer] + + if maxRatePacketLoss <= 5 { + if currentTemporalLayer < mctl && currentTemporalLayer+1 <= d.maxTemporalLayer.get() && + expectedMinBitrate >= 3*cbr/4 { + d.SwitchTemporalLayer(currentTemporalLayer+1, false) + d.simulcast.switchDelay = time.Now().Add(3 * time.Second) + } + if currentTemporalLayer >= mctl && expectedMinBitrate >= 3*cbr/2 && currentSpatialLayer+1 <= d.maxSpatialLayer.get() && + currentSpatialLayer+1 <= 2 { + if err := d.SwitchSpatialLayer(currentSpatialLayer+1, false); err == nil { + d.SwitchTemporalLayer(0, false) + } + d.simulcast.switchDelay = time.Now().Add(5 * time.Second) + } + } + if maxRatePacketLoss >= 25 { + if (expectedMinBitrate <= 5*cbr/8 || currentTemporalLayer == 0) && + currentSpatialLayer > 0 && + brs[currentSpatialLayer-1] != 0 { + if err := d.SwitchSpatialLayer(currentSpatialLayer-1, false); err != nil { + d.SwitchTemporalLayer(mtl[currentSpatialLayer-1], false) + } + d.simulcast.switchDelay = time.Now().Add(10 * time.Second) + } else { + d.SwitchTemporalLayer(currentTemporalLayer-1, false) + d.simulcast.switchDelay = time.Now().Add(5 * time.Second) + } + } + } + } +} + +func (d *DownTrack) getSRStats() (octets, packets uint32) { + return d.octetCount.get(), d.packetCount.get() +} + +func (d *DownTrack) translateVP8Packet(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 *buffer.VP8) (buf []byte, err error) { + temporalLayer := d.temporalLayer.get() + currentLayer := uint16(temporalLayer) + currentTargetLayer := uint16(temporalLayer >> 16) + // catch up temporal layer if necessary + if currentTargetLayer != currentLayer { + if incomingVP8.TIDPresent == 1 && incomingVP8.TID <= uint8(currentTargetLayer) { + d.temporalLayer.set(int32(currentTargetLayer)<<16 | int32(currentTargetLayer)) + } + } + + buf = *d.payload + buf = buf[:len(pkt.Payload)+translatedVP8.HeaderSize-incomingVP8.HeaderSize] + + srcPayload := pkt.Payload[incomingVP8.HeaderSize:] + dstPayload := buf[translatedVP8.HeaderSize:] + copy(dstPayload, srcPayload) + + hdr := buf[:translatedVP8.HeaderSize] + err = translatedVP8.MarshalTo(hdr) + + return +} + +func (d *DownTrack) DebugInfo() map[string]interface{} { + mungerParams := d.munger.getParams() + stats := map[string]interface{}{ + "HighestIncomingSN": mungerParams.highestIncomingSN, + "LastSN": mungerParams.lastSN, + "SNOffset": mungerParams.snOffset, + "LastTS": mungerParams.lastTS, + "TSOffset": mungerParams.tsOffset, + "LastMarker": mungerParams.lastMarker, + "LastRTP": d.lastRTP.get(), + "LastPli": d.lastPli.get(), + "PacketsDropped": d.pktsDropped.get(), + "PacketsMuted": d.pktsMuted.get(), + } + + senderReport := d.CreateSenderReport() + if senderReport != nil { + stats["NTPTime"] = senderReport.NTPTime + stats["RTPTime"] = senderReport.RTPTime + stats["PacketCount"] = senderReport.PacketCount + } + + return map[string]interface{}{ + "PeerID": d.peerID, + "TrackID": d.id, + "StreamID": d.streamID, + "SSRC": d.ssrc, + "MimeType": d.codec.MimeType, + "Bound": d.bound.get(), + "Enabled": d.enabled.get(), + "Resync": d.reSync.get(), + "CurrentSpatialLayer": d.CurrentSpatialLayer(), + "Stats": stats, + } +} + +// +// munger +// +type MungerParams struct { + highestIncomingSN uint16 + lastSN uint16 + snOffset uint16 + lastTS uint32 + tsOffset uint32 + lastMarker bool + + missingSNs map[uint16]uint16 +} + +type Munger struct { + lock sync.RWMutex + + MungerParams +} + +func NewMunger() *Munger { + return &Munger{MungerParams: MungerParams{ + missingSNs: make(map[uint16]uint16, 10), + }} +} + +func (m *Munger) getParams() MungerParams { + m.lock.RLock() + defer m.lock.RUnlock() + + return MungerParams{ + highestIncomingSN: m.highestIncomingSN, + lastSN: m.lastSN, + snOffset: m.snOffset, + lastTS: m.lastTS, + tsOffset: m.tsOffset, + lastMarker: m.lastMarker, + } +} + +func (m *Munger) SetLastSnTs(extPkt *buffer.ExtPacket) { + m.lock.Lock() + defer m.lock.Unlock() + + m.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 + m.lastSN = extPkt.Packet.SequenceNumber + m.lastTS = extPkt.Packet.Timestamp +} + +func (m *Munger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint16, tsAdjust uint32) { + m.lock.Lock() + defer m.lock.Unlock() + + m.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 + m.snOffset = extPkt.Packet.SequenceNumber - m.lastSN - snAdjust + m.tsOffset = extPkt.Packet.Timestamp - m.lastTS - tsAdjust +} + +func (m *Munger) PacketDropped(extPkt *buffer.ExtPacket) { + m.lock.Lock() + defer m.lock.Unlock() + + if !extPkt.Head { + return + } + + m.highestIncomingSN = extPkt.Packet.SequenceNumber + m.snOffset += 1 +} + +func (m *Munger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (uint16, uint32, error) { + m.lock.Lock() + defer m.lock.Unlock() + + // if out-of-order, look up missing sequence number cache + if !extPkt.Head { + snOffset, ok := m.missingSNs[extPkt.Packet.SequenceNumber] + if !ok { + return 0, 0, ErrOutOfOrderSequenceNumberCacheMiss + } + + delete(m.missingSNs, extPkt.Packet.SequenceNumber) + return extPkt.Packet.SequenceNumber - snOffset, extPkt.Packet.Timestamp - m.tsOffset, nil + } + + // if there are gaps, record it in missing sequence number cache + diff := extPkt.Packet.SequenceNumber - m.highestIncomingSN + if diff > 1 { + if extPkt.Packet.SequenceNumber > m.highestIncomingSN { + for lostSN := m.highestIncomingSN + 1; lostSN < extPkt.Packet.SequenceNumber; lostSN++ { + m.missingSNs[lostSN] = m.snOffset + } + } else { + for lostSN := m.highestIncomingSN + 1; lostSN <= uint16(buffer.MaxSN-1); lostSN++ { + m.missingSNs[lostSN] = m.snOffset + } + for lostSN := uint16(0); lostSN < extPkt.Packet.SequenceNumber; lostSN++ { + m.missingSNs[lostSN] = m.snOffset + } + } + } else { + // can get duplicate packet due to FEC + if diff == 0 { + return 0, 0, ErrDuplicatePacket + } + + // if padding only packet, can be dropped and sequence number adjusted + // as it is contiguous and in order. That means this is the highest + // incoming sequence number and it is a good point to adjust + // sequence number offset. + if len(extPkt.Packet.Payload) == 0 { + m.highestIncomingSN = extPkt.Packet.SequenceNumber + m.snOffset += 1 + return 0, 0, ErrPaddingOnlyPacket + } + } + + // in-order incoming packet, may or may not be contiguous. + // In the case of loss (i. e. incoming sequence number is not contiguous), + // forward even if it is a padding only packet. With temporal scalability, + // it is unclear if the current packet should be dropped if it is not + // contiguous. Hence forward anything that is not contiguous. + // Reference: http://www.rtcbits.com/2017/04/howto-implement-temporal-scalability.html + mungedSN := extPkt.Packet.SequenceNumber - m.snOffset + mungedTS := extPkt.Packet.Timestamp - m.tsOffset + + m.highestIncomingSN = extPkt.Packet.SequenceNumber + m.lastSN = mungedSN + m.lastTS = mungedTS + m.lastMarker = extPkt.Packet.Marker + + return mungedSN, mungedTS, nil +} + +func (m *Munger) UpdateAndGetPaddingSnTs() (uint16, uint32, error) { + m.lock.Lock() + defer m.lock.Unlock() + + if !m.lastMarker { + return 0, 0, ErrPaddingNotOnFrameBoundary + } + + sn := m.lastSN + 1 + ts := m.lastTS + + m.lastSN = sn + m.snOffset -= 1 + + return sn, ts, nil +} + +// +// VP8 munger +// +type VP8MungerParams struct { + pictureIdWrapHandler VP8PictureIdWrapHandler + extLastPictureId int32 + pictureIdOffset int32 + lastTl0PicIdx uint8 + tl0PicIdxOffset uint8 + lastKeyIdx uint8 + keyIdxOffset uint8 + + missingPictureIds map[int32]int32 +} + +type VP8Munger struct { + lock sync.Mutex + + VP8MungerParams +} + +func NewVP8Munger() *VP8Munger { + return &VP8Munger{VP8MungerParams: VP8MungerParams{ + missingPictureIds: make(map[int32]int32, 10), + }} +} + +func (v *VP8Munger) SetLast(extPkt *buffer.ExtPacket) { + v.lock.Lock() + defer v.lock.Unlock() + + vp8, ok := extPkt.Payload.(buffer.VP8) + if !ok { + return + } + + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + v.extLastPictureId = int32(vp8.PictureID) + v.lastTl0PicIdx = vp8.TL0PICIDX + v.lastKeyIdx = vp8.KEYIDX +} + +func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { + v.lock.Lock() + defer v.lock.Unlock() + + vp8, ok := extPkt.Payload.(buffer.VP8) + if !ok { + return + } + + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + v.pictureIdOffset = int32(vp8.PictureID) - v.extLastPictureId - 1 + v.tl0PicIdxOffset = vp8.TL0PICIDX - v.lastTl0PicIdx - 1 + v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f +} + +func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, maxTemporalLayer int32) (*buffer.VP8, error) { + v.lock.Lock() + defer v.lock.Unlock() + + vp8, ok := extPkt.Payload.(buffer.VP8) + if !ok { + return nil, ErrNotVP8 + } + + extPictureId, newer := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.MBit) + + // if out-of-order, look up missing picture id cache + if !newer { + pictureIdOffset, ok := v.missingPictureIds[extPictureId] + if !ok { + return nil, ErrOutOfOrderVP8PictureIdCacheMiss + } + + delete(v.missingPictureIds, extPictureId) + + mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff) + vp8Packet := &buffer.VP8{ + FirstByte: vp8.FirstByte, + PictureIDPresent: vp8.PictureIDPresent, + PictureID: mungedPictureId, + MBit: mungedPictureId > 127, + TL0PICIDXPresent: vp8.TL0PICIDXPresent, + TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset, + TIDPresent: vp8.TIDPresent, + TID: vp8.TID, + Y: vp8.Y, + KEYIDXPresent: vp8.KEYIDXPresent, + KEYIDX: vp8.KEYIDX - v.keyIdxOffset, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + } + return vp8Packet, nil + } + + prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() + v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.MBit) + + // if there are gaps, record it in missing picture id cache + // check for > 1 as consecutive packets can have the same picture ID, + // i. e. one picture composed of multiple packets + if extPictureId-prevMaxPictureId > 1 { + for lostPictureId := prevMaxPictureId + 1; lostPictureId < extPictureId; lostPictureId++ { + v.missingPictureIds[lostPictureId] = v.pictureIdOffset + } + } else { + if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { + // adjust only once per picture as a picture could have multiple packets + if vp8.PictureIDPresent == 1 && prevMaxPictureId != extPictureId { + v.pictureIdOffset += 1 + } + return nil, ErrFilteredVP8TemporalLayer + } + } + + // in-order incoming picture, may or may not be contiguous. + // In the case of loss (i. e. incoming picture number is not contiguous), + // forward even if it is a filtered layer. With temporal scalability, + // it is unclear if the current packet should be dropped if it is not + // contiguous. Hence forward anything that is not contiguous. + // Reference: http://www.rtcbits.com/2017/04/howto-implement-temporal-scalability.html + extMungedPictureId := extPictureId - v.pictureIdOffset + mungedPictureId := uint16(extMungedPictureId & 0x7fff) + mungedTl0PicIdx := vp8.TL0PICIDX - v.tl0PicIdxOffset + mungedKeyIdx := (vp8.KEYIDX - v.keyIdxOffset) & 0x1f + + v.extLastPictureId = extMungedPictureId + v.lastTl0PicIdx = mungedTl0PicIdx + v.lastKeyIdx = mungedKeyIdx + + vp8Packet := &buffer.VP8{ + FirstByte: vp8.FirstByte, + PictureIDPresent: vp8.PictureIDPresent, + PictureID: mungedPictureId, + MBit: mungedPictureId > 127, + TL0PICIDXPresent: vp8.TL0PICIDXPresent, + TL0PICIDX: mungedTl0PicIdx, + TIDPresent: vp8.TIDPresent, + TID: vp8.TID, + Y: vp8.Y, + KEYIDXPresent: vp8.KEYIDXPresent, + KEYIDX: mungedKeyIdx, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + } + return vp8Packet, nil +} + +func isWrapping7Bit(val1 int32, val2 int32) bool { + return val2 < val1 && (val1-val2) > (1<<6) +} + +func isWrapping15Bit(val1 int32, val2 int32) bool { + return val2 < val1 && (val1-val2) > (1<<14) +} + +type VP8PictureIdWrapHandler struct { + maxPictureId int32 + maxMBit bool + totalWrap int32 + lastWrap int32 +} + +func (v *VP8PictureIdWrapHandler) Init(extPictureId int32, mBit bool) { + v.maxPictureId = extPictureId + v.maxMBit = mBit + v.totalWrap = 0 + v.lastWrap = 0 +} + +func (v *VP8PictureIdWrapHandler) MaxPictureId() int32 { + return v.maxPictureId +} + +// unwrap picture id and update the maxPictureId. return unwrapped value, and whether picture id is newer +func (v *VP8PictureIdWrapHandler) Unwrap(pictureId uint16, mBit bool) (int32, bool) { + // + // VP8 Picture ID is specified very flexibly. + // + // Reference: https://datatracker.ietf.org/doc/html/draft-ietf-payload-vp8 + // + // Quoting from the RFC + // ---------------------------- + // PictureID: 7 or 15 bits (shown left and right, respectively, in + // Figure 2) not including the M bit. This is a running index of + // the frames, which MAY start at a random value, MUST increase by + // 1 for each subsequent frame, and MUST wrap to 0 after reaching + // the maximum ID (all bits set). The 7 or 15 bits of the + // PictureID go from most significant to least significant, + // beginning with the first bit after the M bit. The sender + // chooses a 7 or 15 bit index and sets the M bit accordingly. + // The receiver MUST NOT assume that the number of bits in + // PictureID stay the same through the session. Having sent a + // 7-bit PictureID with all bits set to 1, the sender may either + // wrap the PictureID to 0, or extend to 15 bits and continue + // incrementing + // ---------------------------- + // + // While in practice, senders may not switch between modes indiscriminately, + // it is possible that small picture ids are sent in 7 bits and then switch + // to 15 bits. But, to ensure correctness, this code keeps track of how much + // quantity has wrapped and uses that to figure out if the incoming picture id + // is newer OR out-of-order. + // + maxPictureId := v.maxPictureId + // maxPictureId can be -1 at the start + if maxPictureId > 0 { + if v.maxMBit { + maxPictureId = v.maxPictureId & 0x7fff + } else { + maxPictureId = v.maxPictureId & 0x7f + } + } + + var newPictureId int32 + if mBit { + newPictureId = int32(pictureId & 0x7fff) + } else { + newPictureId = int32(pictureId & 0x7f) + } + + // + // if the new picture id is too far ahead of max, i.e. more than half of last wrap, + // it is out-of-order, unwrap backwards + // + if v.totalWrap > 0 { + if (v.maxPictureId + (v.lastWrap >> 1)) < (newPictureId + v.totalWrap) { + return newPictureId + v.totalWrap - v.lastWrap, false + } + } + + // + // check for wrap around based on mode of previous picture id. + // There are three cases here + // 1. Wrapping from 15-bit -> 8-bit (32767 -> 0) + // 2. Wrapping from 15-bit -> 15-bit (32767 -> 0) + // 3. Wrapping from 8-bit -> 8-bit (127 -> 0) + // In all cases, looking at the mode of previous picture id will + // ensure that we are calculating the rap properly. + // + wrap := int32(0) + if v.maxMBit { + if isWrapping15Bit(maxPictureId, newPictureId) { + wrap = 1 << 15 + } + } else { + if isWrapping7Bit(maxPictureId, newPictureId) { + wrap = 1 << 7 + } + } + + v.totalWrap += wrap + if wrap != 0 { + v.lastWrap = wrap + } + newPictureId += v.totalWrap + + // >= in the below check as there could be multiple packets per picture + return newPictureId, newPictureId >= v.maxPictureId +} + +func (v *VP8PictureIdWrapHandler) UpdateMaxPictureId(extPictureId int32, mBit bool) { + v.maxPictureId = extPictureId + v.maxMBit = mBit +} diff --git a/pkg/sfu/errors.go b/pkg/sfu/errors.go new file mode 100644 index 000000000..1f7132efd --- /dev/null +++ b/pkg/sfu/errors.go @@ -0,0 +1,18 @@ +package sfu + +import "errors" + +var ( + // PeerLocal erors + errPeerConnectionInitFailed = errors.New("pc init failed") + errCreatingDataChannel = errors.New("failed to create data channel") + // router errors + errNoReceiverFound = errors.New("no receiver found") + // Helpers errors + errShortPacket = errors.New("packet is not large enough") + errNilPacket = errors.New("invalid nil packet") + + ErrSpatialNotSupported = errors.New("current track does not support simulcast/SVC") + ErrSpatialLayerNotFound = errors.New("the requested layer does not exist") + ErrSpatialLayerBusy = errors.New("a spatial layer change is in progress, try later") +) diff --git a/pkg/sfu/helpers.go b/pkg/sfu/helpers.go new file mode 100644 index 000000000..a9961f510 --- /dev/null +++ b/pkg/sfu/helpers.go @@ -0,0 +1,131 @@ +package sfu + +import ( + "encoding/binary" + "strings" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/pion/webrtc/v3" +) + +var ( + ntpEpoch = time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC) +) + +type ntpTime uint64 + +// LK-TODO: CLEANUP-REMOVE-FUNCTION +// setVp8TemporalLayer is a helper to detect and modify accordingly the vp8 payload to reflect +// temporal changes in the SFU. +// VP8 temporal layers implemented according https://tools.ietf.org/html/rfc7741 +func setVP8TemporalLayer(p *buffer.ExtPacket, d *DownTrack) (buf []byte, picID uint16, tlz0Idx uint8, drop bool) { + pkt, ok := p.Payload.(buffer.VP8) + if !ok { + return + } + + layer := d.temporalLayer.get() + currentLayer := uint16(layer) + currentTargetLayer := uint16(layer >> 16) + // Check if temporal getLayer is requested + if currentTargetLayer != currentLayer { + if pkt.TID <= uint8(currentTargetLayer) { + d.temporalLayer.set(int32(currentTargetLayer)<<16 | int32(currentTargetLayer)) + } + } else if pkt.TID > uint8(currentLayer) { + drop = true + return + } + + buf = *d.payload + buf = buf[:len(p.Packet.Payload)] + copy(buf, p.Packet.Payload) + + picID = pkt.PictureID - d.simulcast.refPicID.get() + d.simulcast.pRefPicID.get() + 1 + tlz0Idx = pkt.TL0PICIDX - d.simulcast.refTlZIdx.get() + d.simulcast.pRefTlZIdx.get() + 1 + + if p.Head { + d.simulcast.lPicID.set(picID) + d.simulcast.lTlZIdx.set(tlz0Idx) + } + + modifyVP8TemporalPayload(buf, pkt.PicIDIdx, pkt.TlzIdx, picID, tlz0Idx, pkt.MBit) + + return +} + +// LK-TODO: CLEANUP-REMOVE-FUNCTION +func modifyVP8TemporalPayload(payload []byte, picIDIdx, tlz0Idx int, picID uint16, tlz0ID uint8, mBit bool) { + pid := make([]byte, 2) + binary.BigEndian.PutUint16(pid, picID) + payload[picIDIdx] = pid[0] + if mBit { + payload[picIDIdx] |= 0x80 + payload[picIDIdx+1] = pid[1] + } + payload[tlz0Idx] = tlz0ID +} + +// Do a fuzzy find for a codec in the list of codecs +// Used for lookup up a codec in an existing list to find a match +func codecParametersFuzzySearch(needle webrtc.RTPCodecParameters, haystack []webrtc.RTPCodecParameters) (webrtc.RTPCodecParameters, error) { + // First attempt to match on MimeType + SDPFmtpLine + for _, c := range haystack { + if strings.EqualFold(c.RTPCodecCapability.MimeType, needle.RTPCodecCapability.MimeType) && + c.RTPCodecCapability.SDPFmtpLine == needle.RTPCodecCapability.SDPFmtpLine { + return c, nil + } + } + + // Fallback to just MimeType + for _, c := range haystack { + if strings.EqualFold(c.RTPCodecCapability.MimeType, needle.RTPCodecCapability.MimeType) { + return c, nil + } + } + + return webrtc.RTPCodecParameters{}, webrtc.ErrCodecNotFound +} + +func ntpToMillisSinceEpoch(ntp uint64) uint64 { + // ntp time since epoch calculate fractional ntp as milliseconds + // (lower 32 bits stored as 1/2^32 seconds) and add + // ntp seconds (stored in higher 32 bits) as milliseconds + return (((ntp & 0xFFFFFFFF) * 1000) >> 32) + ((ntp >> 32) * 1000) +} + +func fastForwardTimestampAmount(newestTimestamp uint32, referenceTimestamp uint32) uint32 { + if buffer.IsTimestampWrapAround(newestTimestamp, referenceTimestamp) { + return uint32(uint64(newestTimestamp) + 0x100000000 - uint64(referenceTimestamp)) + } + if newestTimestamp < referenceTimestamp { + return 0 + } + return newestTimestamp - referenceTimestamp +} + +func (t ntpTime) Duration() time.Duration { + sec := (t >> 32) * 1e9 + frac := (t & 0xffffffff) * 1e9 + nsec := frac >> 32 + if uint32(frac) >= 0x80000000 { + nsec++ + } + return time.Duration(sec + nsec) +} + +func (t ntpTime) Time() time.Time { + return ntpEpoch.Add(t.Duration()) +} + +func toNtpTime(t time.Time) ntpTime { + nsec := uint64(t.Sub(ntpEpoch)) + sec := nsec / 1e9 + nsec = (nsec - sec*1e9) << 32 + frac := nsec / 1e9 + if nsec%1e9 >= 1e9/2 { + frac++ + } + return ntpTime(sec<<32 | frac) +} diff --git a/pkg/sfu/helpers_test.go b/pkg/sfu/helpers_test.go new file mode 100644 index 000000000..f21bc84ac --- /dev/null +++ b/pkg/sfu/helpers_test.go @@ -0,0 +1,34 @@ +package sfu + +import ( + "testing" + "time" +) + +func Test_timeToNtp(t *testing.T) { + type args struct { + ns time.Time + } + tests := []struct { + name string + args args + wantNTP uint64 + }{ + { + name: "Must return correct NTP time", + args: args{ + ns: time.Unix(1602391458, 1234), + }, + wantNTP: 16369753560730047668, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + gotNTP := uint64(toNtpTime(tt.args.ns)) + if gotNTP != tt.wantNTP { + t.Errorf("timeToNtp() gotFraction = %v, want %v", gotNTP, tt.wantNTP) + } + }) + } +} diff --git a/pkg/sfu/logger/zerologr.go b/pkg/sfu/logger/zerologr.go new file mode 100644 index 000000000..2e6deb9b7 --- /dev/null +++ b/pkg/sfu/logger/zerologr.go @@ -0,0 +1,133 @@ +// Copyright 2019 Jorn Friedrich Dreyer +// Modified 2021 Serhii Mikhno +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logger defines a default implementation of the github.com/go-logr/logr +// interfaces built on top of zerolog (github.com/rs/zerolog) and is the default +// implementation for ion-sfu released binaries. + +// This package separates log level into two different concepts: +// - V-Level - verbosity level, number, that every logger has. +// A higher value that means more logs will be written. +// - Log-level - usual log level (TRACE|DEBUG|INFO). +// Every log row combines those two values. +// You can set log level to TRACE and see all general traces. +// To see more logs just add -v + +package logger + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/go-logr/logr" + "github.com/go-logr/zerologr" + "github.com/rs/zerolog" +) + +const ( + timeFormat = "2006-01-02 15:04:05.000" +) + +// GlobalConfig config contains global options +type GlobalConfig struct { + V int `mapstructure:"v"` +} + +// SetGlobalOptions sets the global options, like level against which all info logs will be +// compared. If this is greater than or equal to the "V" of the logger, the +// message will be logged. Concurrent-safe. +func SetGlobalOptions(config GlobalConfig) { + lvl := 1 - config.V + if v := int(zerolog.TraceLevel); lvl < v { + lvl = v + } else if v := int(zerolog.InfoLevel); lvl > v { + lvl = v + } + zerolog.SetGlobalLevel(zerolog.Level(lvl)) +} + +// SetVLevelByStringGlobal does the same as SetGlobalOptions but +// trying to expose verbosity level as more familiar "word-based" log levels +func SetVLevelByStringGlobal(level string) { + if v, err := zerolog.ParseLevel(level); err == nil { + zerolog.SetGlobalLevel(v) + } +} + +// Options that can be passed to NewWithOptions +type Options struct { + // Name is an optional name of the logger + Name string + TimeFormat string + Output io.Writer + // Logger is an instance of zerolog, if nil a default logger is used + Logger *zerolog.Logger +} + +// New returns a logr.Logger, LogSink is implemented by zerolog. +func New() logr.Logger { + return NewWithOptions(Options{}) +} + +// NewWithOptions returns a logr.Logger, LogSink is implemented by zerolog. +func NewWithOptions(opts Options) logr.Logger { + if opts.TimeFormat != "" { + zerolog.TimeFieldFormat = opts.TimeFormat + } else { + zerolog.TimeFieldFormat = timeFormat + } + + var out io.Writer + if opts.Output != nil { + out = opts.Output + } else { + out = getOutputFormat() + } + + if opts.Logger == nil { + l := zerolog.New(out).With().Timestamp().Logger() + opts.Logger = &l + } + + ls := zerologr.NewLogSink(opts.Logger) + if zerolog.LevelFieldName == "" { + // Restore field removed by Zerologr + zerolog.LevelFieldName = "level" + } + l := logr.New(ls) + if opts.Name != "" { + l = l.WithName(opts.Name) + } + return l +} + +func getOutputFormat() zerolog.ConsoleWriter { + output := zerolog.ConsoleWriter{Out: os.Stdout, NoColor: false} + output.FormatTimestamp = func(i interface{}) string { + return "[" + i.(string) + "]" + } + output.FormatLevel = func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("[%-3s]", i)) + } + output.FormatMessage = func(i interface{}) string { + _, file, line, _ := runtime.Caller(10) + return fmt.Sprintf("[%s:%d] => %s", filepath.Base(file), line, i) + } + return output +} diff --git a/pkg/sfu/logger/zerologr_test.go b/pkg/sfu/logger/zerologr_test.go new file mode 100644 index 000000000..8f4b85b8e --- /dev/null +++ b/pkg/sfu/logger/zerologr_test.go @@ -0,0 +1,144 @@ +package logger + +import ( + "bytes" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultVerbosityLevel(t *testing.T) { + + t.Run("info", func(t *testing.T) { + SetGlobalOptions(GlobalConfig{V: 0}) + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("info") + assert.Equal(t, `{"level":"info","v":0,"time":"NOTIME","message":"info"}`+"\n", out.String()) + }) + + t.Run("info-add-fields", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("", "test_field", 123) + assert.Equal(t, `{"level":"info","v":0,"test_field":123,"time":"NOTIME"}`+"\n", out.String()) + }) + + t.Run("empty", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("") + assert.Equal(t, `{"level":"info","v":0,"time":"NOTIME"}`+"\n", out.String()) + }) + + t.Run("disabled", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.V(1).Info("You should not see this") + assert.Equal(t, ``, out.String()) + }) +} + +func TestVerbosityLevel2(t *testing.T) { + + t.Run("info", func(t *testing.T) { + SetGlobalOptions(GlobalConfig{V: 2}) + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("info") + assert.Equal(t, `{"level":"info","v":0,"time":"NOTIME","message":"info"}`+"\n", out.String()) + }) + + t.Run("info-add-fields", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("", "test_field", 123) + assert.Equal(t, `{"level":"info","v":0,"test_field":123,"time":"NOTIME"}`+"\n", out.String()) + }) + + t.Run("empty", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.Info("") + assert.Equal(t, `{"level":"info","v":0,"time":"NOTIME"}`+"\n", out.String()) + }) + + t.Run("disabled", func(t *testing.T) { + out := &bytes.Buffer{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: out}) + log.V(1).Info("") + assert.Equal(t, `{"level":"debug","v":1,"time":"NOTIME"}`+"\n", out.String()) + }) +} + +type blackholeStream struct { + writeCount uint64 +} + +func (s *blackholeStream) WriteCount() uint64 { + return atomic.LoadUint64(&s.writeCount) +} + +func (s *blackholeStream) Write(p []byte) (int, error) { + atomic.AddUint64(&s.writeCount, 1) + return len(p), nil +} + +func BenchmarkLoggerLogs(b *testing.B) { + stream := &blackholeStream{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: stream}) + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + log.Info("The quick brown fox jumps over the lazy dog") + } + }) + + if stream.WriteCount() != uint64(b.N) { + b.Fatalf("Log write count") + } +} + +func BenchmarLoggerLog(b *testing.B) { + stream := &blackholeStream{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: stream}) + b.ResetTimer() + SetGlobalOptions(GlobalConfig{V: 1}) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + log.V(1).Info("The quick brown fox jumps over the lazy dog") + } + }) + + if stream.WriteCount() != uint64(b.N) { + b.Fatalf("Log write count") + } +} + +func BenchmarkLoggerLogWith10Fields(b *testing.B) { + stream := &blackholeStream{} + log := NewWithOptions(Options{TimeFormat: "NOTIME", Output: stream}) + b.ResetTimer() + SetGlobalOptions(GlobalConfig{V: 1}) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + log.V(1).Info("The quick brown fox jumps over the lazy dog", + "test1", 1, + "test2", 2, + "test3", 3, + "test4", 4, + "test5", 5, + "test6", 6, + "test7", 7, + "test8", 8, + "test9", 9, + "test10", 10) + } + }) + + if stream.WriteCount() != uint64(b.N) { + b.Fatalf("Log write count") + } +} diff --git a/pkg/sfu/mediaengine.go b/pkg/sfu/mediaengine.go new file mode 100644 index 000000000..b5fe40b59 --- /dev/null +++ b/pkg/sfu/mediaengine.go @@ -0,0 +1,92 @@ +package sfu + +import ( + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v3" +) + +const ( + mimeTypeH264 = "video/h264" + mimeTypeOpus = "audio/opus" + mimeTypeVP8 = "video/vp8" + mimeTypeVP9 = "video/vp9" +) + +const frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" + +func getPublisherMediaEngine() (*webrtc.MediaEngine, error) { + me := &webrtc.MediaEngine{} + if err := me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1", RTCPFeedback: nil}, + PayloadType: 111, + }, webrtc.RTPCodecTypeAudio); err != nil { + return nil, err + } + + videoRTCPFeedback := []webrtc.RTCPFeedback{{"goog-remb", ""}, {"ccm", "fir"}, {"nack", ""}, {"nack", "pli"}} + for _, codec := range []webrtc.RTPCodecParameters{ + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeVP8, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback}, + PayloadType: 96, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 98, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 100, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 102, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 127, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 125, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 108, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: mimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032", RTCPFeedback: videoRTCPFeedback}, + PayloadType: 123, + }, + } { + if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + } + + for _, extension := range []string{ + sdp.SDESMidURI, + sdp.SDESRTPStreamIDURI, + sdp.TransportCCURI, + frameMarking, + } { + if err := me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: extension}, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + } + for _, extension := range []string{ + sdp.SDESMidURI, + sdp.SDESRTPStreamIDURI, + sdp.AudioLevelURI, + } { + if err := me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: extension}, webrtc.RTPCodecTypeAudio); err != nil { + return nil, err + } + } + + return me, nil +} + +func getSubscriberMediaEngine() (*webrtc.MediaEngine, error) { + me := &webrtc.MediaEngine{} + return me, nil +} diff --git a/pkg/sfu/peer.go b/pkg/sfu/peer.go new file mode 100644 index 000000000..7fa2432a1 --- /dev/null +++ b/pkg/sfu/peer.go @@ -0,0 +1,311 @@ +package sfu + +import ( + "errors" + "fmt" + "sync" + + "github.com/lucsky/cuid" + + "github.com/pion/webrtc/v3" +) + +const ( + publisher = 0 + subscriber = 1 +) + +var ( + // ErrTransportExists join is called after a peerconnection is established + ErrTransportExists = errors.New("rtc transport already exists for this connection") + // ErrNoTransportEstablished cannot signal before join + ErrNoTransportEstablished = errors.New("no rtc transport exists for this Peer") + // ErrOfferIgnored if offer received in unstable state + ErrOfferIgnored = errors.New("offered ignored") +) + +type Peer interface { + ID() string + Session() Session + Publisher() *Publisher + Subscriber() *Subscriber + Close() error + SendDCMessage(label string, msg []byte) error +} + +// JoinConfig allow adding more control to the peers joining a SessionLocal. +type JoinConfig struct { + // If true the peer will not be allowed to publish tracks to SessionLocal. + NoPublish bool + // If true the peer will not be allowed to subscribe to other peers in SessionLocal. + NoSubscribe bool + // If true the peer will not automatically subscribe all tracks, + // and then the peer can use peer.Subscriber().AddDownTrack/RemoveDownTrack + // to customize the subscrbe stream combination as needed. + // this parameter depends on NoSubscribe=false. + NoAutoSubscribe bool +} + +// SessionProvider provides the SessionLocal to the sfu.Peer +// This allows the sfu.SFU{} implementation to be customized / wrapped by another package +type SessionProvider interface { + GetSession(sid string) (Session, WebRTCTransportConfig) +} + +type ChannelAPIMessage struct { + Method string `json:"method"` + Params interface{} `json:"params,omitempty"` +} + +// PeerLocal represents a pair peer connection +type PeerLocal struct { + sync.Mutex + id string + closed atomicBool + session Session + provider SessionProvider + + publisher *Publisher + subscriber *Subscriber + + OnOffer func(*webrtc.SessionDescription) + OnIceCandidate func(*webrtc.ICECandidateInit, int) + OnICEConnectionStateChange func(webrtc.ICEConnectionState) + + remoteAnswerPending bool + negotiationPending bool +} + +// NewPeer creates a new PeerLocal for signaling with the given SFU +func NewPeer(provider SessionProvider) *PeerLocal { + return &PeerLocal{ + provider: provider, + } +} + +// Join initializes this peer for a given sessionID +func (p *PeerLocal) Join(sid, uid string, config ...JoinConfig) error { + var conf JoinConfig + if len(config) > 0 { + conf = config[0] + } + + if p.session != nil { + Logger.V(1).Info("peer already exists", "session_id", sid, "peer_id", p.id, "publisher_id", p.publisher.id) + return ErrTransportExists + } + + if uid == "" { + uid = cuid.New() + } + p.id = uid + var err error + + s, cfg := p.provider.GetSession(sid) + p.session = s + + if !conf.NoSubscribe { + p.subscriber, err = NewSubscriber(uid, cfg) + if err != nil { + return fmt.Errorf("error creating transport: %v", err) + } + + p.subscriber.noAutoSubscribe = conf.NoAutoSubscribe + + p.subscriber.OnNegotiationNeeded(func() { + p.Lock() + defer p.Unlock() + + if p.remoteAnswerPending { + p.negotiationPending = true + return + } + + Logger.V(1).Info("Negotiation needed", "peer_id", p.id) + offer, err := p.subscriber.CreateOffer() + if err != nil { + Logger.Error(err, "CreateOffer error") + return + } + + p.remoteAnswerPending = true + if p.OnOffer != nil && !p.closed.get() { + Logger.V(0).Info("Send offer", "peer_id", p.id) + p.OnOffer(&offer) + } + }) + + p.subscriber.OnICECandidate(func(c *webrtc.ICECandidate) { + Logger.V(1).Info("On subscriber ice candidate called for peer", "peer_id", p.id) + if c == nil { + return + } + + if p.OnIceCandidate != nil && !p.closed.get() { + json := c.ToJSON() + p.OnIceCandidate(&json, subscriber) + } + }) + } + + if !conf.NoPublish { + p.publisher, err = NewPublisher(uid, p.session, &cfg) + if err != nil { + return fmt.Errorf("error creating transport: %v", err) + } + if !conf.NoSubscribe { + for _, dc := range p.session.GetDCMiddlewares() { + if err := p.subscriber.AddDatachannel(p, dc); err != nil { + return fmt.Errorf("setting subscriber default dc datachannel: %w", err) + } + } + } + + p.publisher.OnICECandidate(func(c *webrtc.ICECandidate) { + Logger.V(1).Info("on publisher ice candidate called for peer", "peer_id", p.id) + if c == nil { + return + } + + if p.OnIceCandidate != nil && !p.closed.get() { + json := c.ToJSON() + p.OnIceCandidate(&json, publisher) + } + }) + + p.publisher.OnICEConnectionStateChange(func(s webrtc.ICEConnectionState) { + if p.OnICEConnectionStateChange != nil && !p.closed.get() { + p.OnICEConnectionStateChange(s) + } + }) + } + + p.session.AddPeer(p) + + Logger.V(0).Info("PeerLocal join SessionLocal", "peer_id", p.id, "session_id", sid) + + if !conf.NoSubscribe { + p.session.Subscribe(p) + } + return nil +} + +// Answer an offer from remote +func (p *PeerLocal) Answer(sdp webrtc.SessionDescription) (*webrtc.SessionDescription, error) { + if p.publisher == nil { + return nil, ErrNoTransportEstablished + } + + Logger.V(0).Info("PeerLocal got offer", "peer_id", p.id) + + if p.publisher.SignalingState() != webrtc.SignalingStateStable { + return nil, ErrOfferIgnored + } + + answer, err := p.publisher.Answer(sdp) + if err != nil { + return nil, fmt.Errorf("error creating answer: %v", err) + } + + Logger.V(0).Info("PeerLocal send answer", "peer_id", p.id) + + return &answer, nil +} + +// SetRemoteDescription when receiving an answer from remote +func (p *PeerLocal) SetRemoteDescription(sdp webrtc.SessionDescription) error { + if p.subscriber == nil { + return ErrNoTransportEstablished + } + p.Lock() + defer p.Unlock() + + Logger.V(0).Info("PeerLocal got answer", "peer_id", p.id) + if err := p.subscriber.SetRemoteDescription(sdp); err != nil { + return fmt.Errorf("setting remote description: %w", err) + } + + p.remoteAnswerPending = false + + if p.negotiationPending { + p.negotiationPending = false + p.subscriber.negotiate() + } + + return nil +} + +// Trickle candidates available for this peer +func (p *PeerLocal) Trickle(candidate webrtc.ICECandidateInit, target int) error { + if p.subscriber == nil || p.publisher == nil { + return ErrNoTransportEstablished + } + Logger.V(0).Info("PeerLocal trickle", "peer_id", p.id) + switch target { + case publisher: + if err := p.publisher.AddICECandidate(candidate); err != nil { + return fmt.Errorf("setting ice candidate: %w", err) + } + case subscriber: + if err := p.subscriber.AddICECandidate(candidate); err != nil { + return fmt.Errorf("setting ice candidate: %w", err) + } + } + return nil +} + +func (p *PeerLocal) SendDCMessage(label string, msg []byte) error { + if p.subscriber == nil { + return fmt.Errorf("no subscriber for this peer") + } + dc := p.subscriber.DataChannel(label) + + if dc == nil { + return fmt.Errorf("data channel %s doesn't exist", label) + } + + if err := dc.SendText(string(msg)); err != nil { + return fmt.Errorf("failed to send message: %v", err) + } + return nil +} + +// Close shuts down the peer connection and sends true to the done channel +func (p *PeerLocal) Close() error { + p.Lock() + defer p.Unlock() + + if !p.closed.set(true) { + return nil + } + + if p.session != nil { + p.session.RemovePeer(p) + } + if p.publisher != nil { + p.publisher.Close() + } + if p.subscriber != nil { + if err := p.subscriber.Close(); err != nil { + return err + } + } + return nil +} + +func (p *PeerLocal) Subscriber() *Subscriber { + return p.subscriber +} + +func (p *PeerLocal) Publisher() *Publisher { + return p.publisher +} + +func (p *PeerLocal) Session() Session { + return p.session +} + +// ID return the peer id +func (p *PeerLocal) ID() string { + return p.id +} diff --git a/pkg/sfu/prober.go b/pkg/sfu/prober.go new file mode 100644 index 000000000..388ba3285 --- /dev/null +++ b/pkg/sfu/prober.go @@ -0,0 +1,352 @@ +// +// Design of Prober +// +// Probing is to used to check for existence of excess channel capacity. +// This is especially useful in the downstream direction of SFU. +// SFU forwards audio/video streams from one or more publishers to +// all the subscribers. But, the downstream channel of a subscriber +// may not be big enough to carry all the streams. It is also a time +// varying quantity. +// +// When there is not enough capacity, some streams will be paused. +// To resume a stream, SFU would need to know that the channel has +// enough capacity. That's where probing comes in. When conditions +// are favorable, SFU can send probe packets so that the bandwidth +// estimator has more data to estimate available channel capacity +// better. +// NOTE: What defines `favorable conditions` is implementation dependent. +// +// There are two options for probing +// - Use padding only RTP packets: This one is preferable as +// probe rate can be controlled more tightly. +// - Resume a paused stream or forward a higher spatial layer: +// Have to find a stream at probing rate. Also, a stream could +// get a key frame unexpectedly boosting rate in the probing +// window. +// +// The strategy used depends on stream allocator implementation. +// This module can be used if the stream allocator decides to use +// padding only RTP packets for probing purposes. +// +// Implementation: +// There are a couple of options +// - Check prober in the forwarding path (pull from prober). +// This is preferred for scalability reasons. But, this +// suffers from not being able to probe when all streams +// are paused (could be due to downstream bandwidth +// constraints or the corresponding upstream tracks may +// have paused due to upstream bandwidth constraints). +// Another issue is not being to have tight control on +// probing window boundary as the packet forwarding path +// may not have a packet to forward. But, it should not +// be a major concern as long as some stream(s) is/are +// forwarded as there should be a packet atleast every +// 60 ms or so (forwarding only one stream at 15 fps). +// Usually, it will be serviced much more frequently when +// there are multiple streams getting forwarded. +// - Run it a go routine. But, that would have to wake up +// very often to prevent bunching up of probe +// packets. So, a scalability concern as there is one prober +// per subscriber peer connection. But, probe windows +// should be very short (of the order of 100s of ms). +// So, this approach might be fine. +// +// The implementation here follows the second approach of using a +// go routine. +// +// Pacing: +// ------ +// Ideally, the subscriber peer connection should have a pacer which +// trickles data out at the estimated channel capacity rate (and +// estimated channel capacity + probing rate when actively probing). +// +// But, there a few significant challenges +// 1. Pacer will require buffering of forwarded packets. That means +// more memory, more CPU (have to make copy of packets) and +// more latency in the media stream. +// 2. Scalability concern as SFU may be handling hundreds of +// subscriber peer connections and each one processing the pacing +// loop at 5ms interval will add up. +// +// So, this module assumes that pacing is inherently provided by the +// publishers for media streams. That is a reasonable assumption given +// that publishing clients will run their own pacer and pacing data out +// at a steady rate. +// +// A further assumption is that if there are multiple publishers for +// a subscriber peer connection, all the publishers are not pacing +// in sync, i. e. each publisher's pacer is completely independent +// and SFU will be receiving the media packets with a good spread and +// not clumped together. +// +// Given those assumptions, this module monitors media send rate and +// adjusts probing packet sends accordingly. Although the probing may +// have a high enough wake up frequency, it is for short windows. +// For example, probing at 5 Mbps for 1/2 second and sending 1000 byte +// probe per iteration will wake up every 1.6 ms. That is very high, +// but should last for 1/2 second or so. +// 5 Mbps over 1/2 second = 2.5 Mbps +// 2.5 Mbps = 312500 bytes = 313 probes at 1000 byte probes +// 313 probes over 1/2 second = 1.6 ms between probes +// +// A few things to note +// 1. When a probe cluster is added, the expected media rate is provided. +// So, the wake up interval takes that into account. For example, +// if probing at 5 Mbps for 1/2 second and if 4 Mbps of it is expected +// to be provided by media traffic, the wake up interval becomes 8 ms. +// 2. The amount of probing should actually be capped at some value to +// avoid too much self-induced congestion. It maybe something like 500 kbps. +// That will increase the wake up interval to 16 ms in the above example. +// 3. In practice, the probing interval may also be shorter. Typically, +// it can be run for 2 - 3 RTTs to get a good measurement. For +// the longest hauls, RTT could be 250 ms or so leading to the probing +// window being long(ish). But, RTT should be much shorter especially if +// the subscriber peer connection of the client is able to connect to +// the nearest data center. +// +package sfu + +import ( + "sync" + "time" + + "github.com/gammazero/deque" +) + +type Prober struct { + clustersMu sync.RWMutex + clusters deque.Deque + activeCluster *Cluster + + onSendProbe func(bytesToSend int) int +} + +func NewProber() *Prober { + p := &Prober{} + p.clusters.SetMinCapacity(2) + return p +} + +func (p *Prober) IsRunning() bool { + p.clustersMu.RLock() + defer p.clustersMu.RUnlock() + + return p.clusters.Len() > 0 +} + +func (p *Prober) Reset() { + p.clustersMu.Lock() + // LK-TODO - log if active cluster is getting reset, maybe log state of all clusters + defer p.clustersMu.Unlock() + p.clusters.Clear() +} + +func (p *Prober) OnSendProbe(f func(bytesToSend int) int) { + p.onSendProbe = f +} + +func (p *Prober) AddCluster(desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) { + if desiredRateBps <= 0 { + return + } + + cluster := NewCluster(desiredRateBps, expectedRateBps, minDuration, maxDuration) + // LK-TODO - log information about added cluster + + p.pushBackClusterAndMaybeStart(cluster) +} + +func (p *Prober) PacketSent(size int) { + cluster := p.getFrontCluster() + if cluster == nil { + return + } + + cluster.PacketSent(size) +} + +func (p *Prober) getFrontCluster() *Cluster { + p.clustersMu.RLock() + defer p.clustersMu.RUnlock() + + if p.activeCluster != nil { + return p.activeCluster + } + + if p.clusters.Len() == 0 { + p.activeCluster = nil + } else { + p.activeCluster = p.clusters.Front().(*Cluster) + p.activeCluster.Start() + } + return p.activeCluster +} + +func (p *Prober) popFrontCluster(cluster *Cluster) { + p.clustersMu.Lock() + defer p.clustersMu.Unlock() + + if p.clusters.Len() == 0 { + p.activeCluster = nil + return + } + + if p.clusters.Front().(*Cluster) == cluster { + p.clusters.PopFront() + } + + if cluster == p.activeCluster { + p.activeCluster = nil + } +} + +func (p *Prober) pushBackClusterAndMaybeStart(cluster *Cluster) { + p.clustersMu.Lock() + defer p.clustersMu.Unlock() + + p.clusters.PushBack(cluster) + + if p.clusters.Len() == 1 { + go p.run() + } +} + +func (p *Prober) run() { + for { + // determine how long to sleep + cluster := p.getFrontCluster() + if cluster == nil { + return + } + + time.Sleep(cluster.GetSleepDuration()) + + // wake up and check for probes to send + cluster = p.getFrontCluster() + if cluster == nil { + return + } + + if !cluster.Process(p) { + p.popFrontCluster(cluster) + continue + } + } +} + +type Cluster struct { + // LK-TODO-START + // Check if we can operate at cluster level without a lock. + // The quantities that are updated in a different thread are + // bytesSentNonProbe - maybe make this an atomic value + // Lock contention time should be very minimal though. + // LK-TODO-END + lock sync.RWMutex + + desiredBytes int + minDuration time.Duration + maxDuration time.Duration + + sleepDuration time.Duration + + bytesSentProbe int + bytesSentNonProbe int + startTime time.Time +} + +func NewCluster(desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) *Cluster { + minDurationMs := minDuration.Milliseconds() + desiredBytes := int((int64(desiredRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) + expectedBytes := int((int64(expectedRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) + + // pace based on sending approximately 1000 bytes per probe + numProbes := int((desiredBytes - expectedBytes + 999) / 1000) + sleepDurationMicroSeconds := int(float64(minDurationMs*1000)/float64(numProbes) + 0.5) + c := &Cluster{ + desiredBytes: desiredBytes, + minDuration: minDuration, + maxDuration: maxDuration, + sleepDuration: time.Duration(sleepDurationMicroSeconds) * time.Microsecond, + } + return c +} + +func (c *Cluster) Start() { + c.lock.Lock() + defer c.lock.Unlock() + + if c.startTime.IsZero() { + c.startTime = time.Now() + } +} + +func (c *Cluster) GetSleepDuration() time.Duration { + c.lock.RLock() + defer c.lock.RUnlock() + + return c.sleepDuration +} + +func (c *Cluster) PacketSent(size int) { + c.lock.Lock() + defer c.lock.Unlock() + + c.bytesSentNonProbe += size +} + +func (c *Cluster) Process(p *Prober) bool { + c.lock.RLock() + + // if already past deadline, end the cluster + timeElapsed := time.Since(c.startTime) + if timeElapsed > c.maxDuration { + // LK-TODO log information about short fall in probing + c.lock.RUnlock() + return false + } + + // Calculate number of probe bytes that should have been sent since start. + // Overall goal is to send desired number of probe bytes in minDuration. + // However it is possible that timeElapsed is more than minDuration due + // to scheduling variance. When overshooting time budget, use a capped + // short fall if there a grace period given. + windowDone := float64(timeElapsed) / float64(c.minDuration) + if windowDone > 1.0 { + // cluster has been running for longer than minDuration + windowDone = 1.0 + } + + bytesShouldHaveBeenSent := int(windowDone * float64(c.desiredBytes)) + bytesShortFall := bytesShouldHaveBeenSent - c.bytesSentProbe - c.bytesSentNonProbe + if bytesShortFall < 0 { + bytesShortFall = 0 + } + // cap short fall to limit to 8 packets in an iteration + // 275 bytes per packet (255 max RTP padding payload + 20 bytes RTP header) + if bytesShortFall > (275 * 8) { + bytesShortFall = 275 * 8 + } + // round up to packet size + bytesShortFall = ((bytesShortFall + 274) / 275) * 275 + c.lock.RUnlock() + + bytesSent := 0 + if bytesShortFall > 0 && p.onSendProbe != nil { + bytesSent = p.onSendProbe(bytesShortFall) + } + + c.lock.Lock() + c.bytesSentProbe += bytesSent + + // do not end cluster until minDuration elapses even if rate is achieved. + // Ensures that the next cluster (if any) does not start early. + if (c.bytesSentProbe+c.bytesSentNonProbe) >= c.desiredBytes && timeElapsed >= c.minDuration { + // LK-TODO - log data about how much time the probe finished compared to min/max + c.lock.Unlock() + return false + } + + // LK-TODO look at adapting sleep time based on how many bytes and how much time is left + c.lock.Unlock() + return true +} diff --git a/pkg/sfu/publisher.go b/pkg/sfu/publisher.go new file mode 100644 index 000000000..39e846d5f --- /dev/null +++ b/pkg/sfu/publisher.go @@ -0,0 +1,448 @@ +package sfu + +import ( + "fmt" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/relay" + "github.com/pion/rtcp" + "github.com/pion/transport/packetio" + "github.com/pion/webrtc/v3" +) + +type Publisher struct { + mu sync.RWMutex + id string + pc *webrtc.PeerConnection + cfg *WebRTCTransportConfig + + router Router + session Session + tracks []PublisherTrack + relayed atomicBool + relayPeers []*relayPeer + candidates []webrtc.ICECandidateInit + + onICEConnectionStateChangeHandler atomic.Value // func(webrtc.ICEConnectionState) + onPublisherTrack atomic.Value // func(PublisherTrack) + + closeOnce sync.Once +} + +type relayPeer struct { + peer *relay.Peer + dcs []*webrtc.DataChannel + withSRReports bool + relayFanOutDataChannels bool +} + +type PublisherTrack struct { + Track *webrtc.TrackRemote + Receiver Receiver + // This will be used in the future for tracks that will be relayed as clients or servers + // This is for SVC and Simulcast where you will be able to chose if the relayed peer just + // want a single track (for recording/ processing) or get all the tracks (for load balancing) + clientRelay bool +} + +// NewPublisher creates a new Publisher +func NewPublisher(id string, session Session, cfg *WebRTCTransportConfig) (*Publisher, error) { + me, err := getPublisherMediaEngine() + if err != nil { + Logger.Error(err, "NewPeer error", "peer_id", id) + return nil, errPeerConnectionInitFailed + } + + api := webrtc.NewAPI(webrtc.WithMediaEngine(me), webrtc.WithSettingEngine(cfg.Setting)) + pc, err := api.NewPeerConnection(cfg.Configuration) + + if err != nil { + Logger.Error(err, "NewPeer error", "peer_id", id) + return nil, errPeerConnectionInitFailed + } + + p := &Publisher{ + id: id, + pc: pc, + cfg: cfg, + router: newRouter(id, session, cfg), + session: session, + } + + pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { + Logger.V(1).Info("Peer got remote track id", + "peer_id", p.id, + "track_id", track.ID(), + "mediaSSRC", track.SSRC(), + "rid", track.RID(), + "stream_id", track.StreamID(), + ) + + r, pub := p.router.AddReceiver(receiver, track, track.ID(), track.StreamID()) + if pub { + p.session.Publish(p.router, r) + p.mu.Lock() + publisherTrack := PublisherTrack{track, r, true} + p.tracks = append(p.tracks, publisherTrack) + for _, rp := range p.relayPeers { + if err = p.createRelayTrack(track, r, rp.peer); err != nil { + Logger.V(1).Error(err, "Creating relay track.", "peer_id", p.id) + } + } + p.mu.Unlock() + if handler, ok := p.onPublisherTrack.Load().(func(PublisherTrack)); ok && handler != nil { + handler(publisherTrack) + } + } else { + p.mu.Lock() + p.tracks = append(p.tracks, PublisherTrack{track, r, false}) + p.mu.Unlock() + } + }) + + pc.OnDataChannel(func(dc *webrtc.DataChannel) { + if dc.Label() == APIChannelLabel { + // terminate api data channel + return + } + p.session.AddDatachannel(id, dc) + }) + + pc.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + Logger.V(1).Info("ice connection status", "state", connectionState) + switch connectionState { + case webrtc.ICEConnectionStateFailed: + fallthrough + case webrtc.ICEConnectionStateClosed: + Logger.V(1).Info("webrtc ice closed", "peer_id", p.id) + p.Close() + } + + if handler, ok := p.onICEConnectionStateChangeHandler.Load().(func(webrtc.ICEConnectionState)); ok && handler != nil { + handler(connectionState) + } + }) + + p.router.SetRTCPWriter(p.pc.WriteRTCP) + + return p, nil +} + +func (p *Publisher) Answer(offer webrtc.SessionDescription) (webrtc.SessionDescription, error) { + if err := p.pc.SetRemoteDescription(offer); err != nil { + return webrtc.SessionDescription{}, err + } + + for _, c := range p.candidates { + if err := p.pc.AddICECandidate(c); err != nil { + Logger.Error(err, "Add publisher ice candidate to peer err", "peer_id", p.id) + } + } + p.candidates = nil + + answer, err := p.pc.CreateAnswer(nil) + if err != nil { + return webrtc.SessionDescription{}, err + } + if err := p.pc.SetLocalDescription(answer); err != nil { + return webrtc.SessionDescription{}, err + } + return answer, nil +} + +// GetRouter returns Router with mediaSSRC +func (p *Publisher) GetRouter() Router { + return p.router +} + +// Close peer +func (p *Publisher) Close() { + p.closeOnce.Do(func() { + if len(p.relayPeers) > 0 { + p.mu.Lock() + for _, rp := range p.relayPeers { + if err := rp.peer.Close(); err != nil { + Logger.Error(err, "Closing relay peer transport.") + } + } + p.mu.Unlock() + } + p.router.Stop() + if err := p.pc.Close(); err != nil { + Logger.Error(err, "webrtc transport close err") + } + }) +} + +func (p *Publisher) OnPublisherTrack(f func(track PublisherTrack)) { + p.onPublisherTrack.Store(f) +} + +// OnICECandidate handler +func (p *Publisher) OnICECandidate(f func(c *webrtc.ICECandidate)) { + p.pc.OnICECandidate(f) +} + +func (p *Publisher) OnICEConnectionStateChange(f func(connectionState webrtc.ICEConnectionState)) { + p.onICEConnectionStateChangeHandler.Store(f) +} + +func (p *Publisher) SignalingState() webrtc.SignalingState { + return p.pc.SignalingState() +} + +func (p *Publisher) PeerConnection() *webrtc.PeerConnection { + return p.pc +} + +// Relay will relay all current and future tracks from current Publisher +func (p *Publisher) Relay(signalFn func(meta relay.PeerMeta, signal []byte) ([]byte, error), + options ...func(r *relayPeer)) (*relay.Peer, error) { + lrp := &relayPeer{} + for _, o := range options { + o(lrp) + } + + rp, err := relay.NewPeer(relay.PeerMeta{ + PeerID: p.id, + SessionID: p.session.ID(), + }, &relay.PeerConfig{ + SettingEngine: p.cfg.Setting, + ICEServers: p.cfg.Configuration.ICEServers, + Logger: Logger, + }) + if err != nil { + return nil, fmt.Errorf("relay: %w", err) + } + lrp.peer = rp + + rp.OnReady(func() { + peer := p.session.GetPeer(p.id) + + p.relayed.set(true) + if lrp.relayFanOutDataChannels { + for _, lbl := range p.session.GetFanOutDataChannelLabels() { + lbl := lbl + dc, err := rp.CreateDataChannel(lbl) + if err != nil { + Logger.V(1).Error(err, "Creating data channels.", "peer_id", p.id) + } + dc.OnMessage(func(msg webrtc.DataChannelMessage) { + if peer == nil || peer.Subscriber() == nil { + return + } + if sdc := peer.Subscriber().DataChannel(lbl); sdc != nil { + if msg.IsString { + if err = sdc.SendText(string(msg.Data)); err != nil { + Logger.Error(err, "Sending dc message err") + } + } else { + if err = sdc.Send(msg.Data); err != nil { + Logger.Error(err, "Sending dc message err") + } + } + } + }) + } + } + + p.mu.Lock() + for _, tp := range p.tracks { + if !tp.clientRelay { + // simulcast will just relay client track for now + continue + } + if err = p.createRelayTrack(tp.Track, tp.Receiver, rp); err != nil { + Logger.V(1).Error(err, "Creating relay track.", "peer_id", p.id) + } + } + p.relayPeers = append(p.relayPeers, lrp) + p.mu.Unlock() + + if lrp.withSRReports { + go p.relayReports(rp) + } + }) + + rp.OnDataChannel(func(channel *webrtc.DataChannel) { + if !lrp.relayFanOutDataChannels { + return + } + p.mu.Lock() + lrp.dcs = append(lrp.dcs, channel) + p.mu.Unlock() + + p.session.AddDatachannel("", channel) + }) + + if err = rp.Offer(signalFn); err != nil { + return nil, fmt.Errorf("relay: %w", err) + } + + return rp, nil +} + +func (p *Publisher) PublisherTracks() []PublisherTrack { + p.mu.Lock() + defer p.mu.Unlock() + + tracks := make([]PublisherTrack, len(p.tracks)) + for idx, track := range p.tracks { + tracks[idx] = track + } + return tracks +} + +// AddRelayFanOutDataChannel adds fan out data channel to relayed peers +func (p *Publisher) AddRelayFanOutDataChannel(label string) { + p.mu.RLock() + defer p.mu.RUnlock() + + for _, rp := range p.relayPeers { + for _, dc := range rp.dcs { + if dc.Label() == label { + continue + } + } + + dc, err := rp.peer.CreateDataChannel(label) + if err != nil { + Logger.V(1).Error(err, "Creating data channels.", "peer_id", p.id) + } + dc.OnMessage(func(msg webrtc.DataChannelMessage) { + p.session.FanOutMessage("", label, msg) + }) + } +} + +// GetRelayedDataChannels Returns a slice of data channels that belongs to relayed +// peers +func (p *Publisher) GetRelayedDataChannels(label string) []*webrtc.DataChannel { + p.mu.RLock() + defer p.mu.RUnlock() + + dcs := make([]*webrtc.DataChannel, 0, len(p.relayPeers)) + for _, rp := range p.relayPeers { + for _, dc := range rp.dcs { + if dc.Label() == label { + dcs = append(dcs, dc) + break + } + } + } + return dcs +} + +// Relayed returns true if the publisher has been relayed at least once +func (p *Publisher) Relayed() bool { + return p.relayed.get() +} + +func (p *Publisher) Tracks() []*webrtc.TrackRemote { + p.mu.RLock() + defer p.mu.RUnlock() + + tracks := make([]*webrtc.TrackRemote, len(p.tracks)) + for idx, track := range p.tracks { + tracks[idx] = track.Track + } + return tracks +} + +// AddICECandidate to peer connection +func (p *Publisher) AddICECandidate(candidate webrtc.ICECandidateInit) error { + if p.pc.RemoteDescription() != nil { + return p.pc.AddICECandidate(candidate) + } + p.candidates = append(p.candidates, candidate) + return nil +} + +func (p *Publisher) createRelayTrack(track *webrtc.TrackRemote, receiver Receiver, rp *relay.Peer) error { + codec := track.Codec() + downTrack, err := NewDownTrack(webrtc.RTPCodecCapability{ + MimeType: codec.MimeType, + ClockRate: codec.ClockRate, + Channels: codec.Channels, + SDPFmtpLine: codec.SDPFmtpLine, + RTCPFeedback: []webrtc.RTCPFeedback{{"nack", ""}, {"nack", "pli"}}, + }, receiver, p.cfg.BufferFactory, p.id, p.cfg.Router.MaxPacketTrack) + if err != nil { + Logger.V(1).Error(err, "Create Relay downtrack err", "peer_id", p.id) + return err + } + + sdr, err := rp.AddTrack(receiver.(*WebRTCReceiver).receiver, track, downTrack) + if err != nil { + Logger.V(1).Error(err, "Relaying track.", "peer_id", p.id) + return fmt.Errorf("relay: %w", err) + } + + p.cfg.BufferFactory.GetOrNew(packetio.RTCPBufferPacket, + uint32(sdr.GetParameters().Encodings[0].SSRC)).(*buffer.RTCPReader).OnPacket(func(bytes []byte) { + pkts, err := rtcp.Unmarshal(bytes) + if err != nil { + Logger.V(1).Error(err, "Unmarshal rtcp reports", "peer_id", p.id) + return + } + var rpkts []rtcp.Packet + for _, pkt := range pkts { + switch pk := pkt.(type) { + case *rtcp.PictureLossIndication: + rpkts = append(rpkts, &rtcp.PictureLossIndication{ + SenderSSRC: pk.MediaSSRC, + MediaSSRC: uint32(track.SSRC()), + }) + } + } + + if len(rpkts) > 0 { + if err := p.pc.WriteRTCP(rpkts); err != nil { + Logger.V(1).Error(err, "Sending rtcp relay reports", "peer_id", p.id) + } + } + + }) + + downTrack.OnCloseHandler(func() { + if err = sdr.Stop(); err != nil { + Logger.V(1).Error(err, "Stopping relay sender.", "peer_id", p.id) + } + }) + + receiver.AddDownTrack(downTrack, true) + return nil +} + +func (p *Publisher) relayReports(rp *relay.Peer) { + for { + time.Sleep(5 * time.Second) + + var r []rtcp.Packet + for _, t := range rp.LocalTracks() { + if dt, ok := t.(*DownTrack); ok { + if !dt.bound.get() { + continue + } + if sr := dt.CreateSenderReport(); sr != nil { + r = append(r, sr) + } + } + } + + if len(r) == 0 { + continue + } + + if err := rp.WriteRTCP(r); err != nil { + if err == io.EOF || err == io.ErrClosedPipe { + return + } + Logger.Error(err, "Sending downtrack reports err") + } + } +} diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go new file mode 100644 index 000000000..055598669 --- /dev/null +++ b/pkg/sfu/receiver.go @@ -0,0 +1,708 @@ +package sfu + +import ( + "io" + "math/rand" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/gammazero/workerpool" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/webrtc/v3" + "github.com/rs/zerolog/log" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/stats" +) + +// Receiver defines a interface for a track receivers +type Receiver interface { + TrackID() string + StreamID() string + Codec() webrtc.RTPCodecParameters + Kind() webrtc.RTPCodecType + SSRC(layer int) uint32 + SetTrackMeta(trackID, streamID string) + AddUpTrack(track *webrtc.TrackRemote, buffer *buffer.Buffer, bestQualityFirst bool) + AddDownTrack(track *DownTrack, bestQualityFirst bool) + SetUpTrackPaused(paused bool) + HasSpatialLayer(layer int32) bool + GetBitrate() [3]uint64 + GetBitrateTemporal() [3][4]uint64 + GetBitrateTemporalCumulative() [3][4]uint64 + GetMaxTemporalLayer() [3]int32 + RetransmitPackets(track *DownTrack, packets []packetMeta) error + DeleteDownTrack(peerID string) + OnCloseHandler(fn func()) + SendRTCP(p []rtcp.Packet) + SetRTCPCh(ch chan []rtcp.Packet) + + GetSenderReportTime(layer int32) (rtpTS uint32, ntpTS uint64) + DebugInfo() map[string]interface{} +} + +const ( + lostUpdateDelta = 1e9 +) + +// WebRTCReceiver receives a video track +type WebRTCReceiver struct { + peerID string + trackID string + streamID string + kind webrtc.RTPCodecType + stream string + receiver *webrtc.RTPReceiver + codec webrtc.RTPCodecParameters + stats [3]*stats.Stream + nackWorker *workerpool.WorkerPool + isSimulcast bool + availableLayers atomic.Value + onCloseHandler func() + closeOnce sync.Once + closed atomicBool + trackers [3]*StreamTracker + useTrackers bool + + rtcpMu sync.Mutex + rtcpCh chan []rtcp.Packet + lastPli atomicInt64 + pliThrottle int64 + + bufferMu sync.RWMutex + buffers [3]*buffer.Buffer + + upTrackMu sync.RWMutex + upTracks [3]*webrtc.TrackRemote + + downTrackMu sync.RWMutex + downTracks []*DownTrack + index map[string]int + free map[int]struct{} + numProcs int + lbThreshold int + + fracLostMu sync.Mutex + maxDownFracLost uint8 + maxDownFracLostTs time.Time +} + +type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver + +// WithPliThrottle indicates minimum time(ms) between sending PLIs +func WithPliThrottle(period int64) ReceiverOpts { + return func(w *WebRTCReceiver) *WebRTCReceiver { + w.pliThrottle = period * 1e6 + return w + } +} + +// WithStreamTrackers enables StreamTracker use for simulcast +func WithStreamTrackers() ReceiverOpts { + return func(w *WebRTCReceiver) *WebRTCReceiver { + w.useTrackers = true + return w + } +} + +// WithLoadBalanceThreshold enables parallelization of packet writes when downTracks exceeds threshold +// Value should be between 3 and 150. +// For a server handling a few large rooms, use a smaller value (required to handle very large (250+ participant) rooms). +// For a server handling many small rooms, use a larger value or disable. +// Set to 0 (disabled) by default. +func WithLoadBalanceThreshold(downTracks int) ReceiverOpts { + return func(w *WebRTCReceiver) *WebRTCReceiver { + w.lbThreshold = downTracks + return w + } +} + +// NewWebRTCReceiver creates a new webrtc track receivers +func NewWebRTCReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, pid string, opts ...ReceiverOpts) Receiver { + w := &WebRTCReceiver{ + peerID: pid, + receiver: receiver, + trackID: track.ID(), + streamID: track.StreamID(), + codec: track.Codec(), + kind: track.Kind(), + nackWorker: workerpool.New(1), + isSimulcast: len(track.RID()) > 0, + pliThrottle: 500e6, + downTracks: make([]*DownTrack, 0), + index: make(map[string]int), + free: make(map[int]struct{}), + numProcs: runtime.NumCPU(), + } + if runtime.GOMAXPROCS(0) < w.numProcs { + w.numProcs = runtime.GOMAXPROCS(0) + } + for _, opt := range opts { + w = opt(w) + } + return w +} + +func (w *WebRTCReceiver) SetTrackMeta(trackID, streamID string) { + w.streamID = streamID + w.trackID = trackID +} + +func (w *WebRTCReceiver) StreamID() string { + return w.streamID +} + +func (w *WebRTCReceiver) TrackID() string { + return w.trackID +} + +func (w *WebRTCReceiver) SSRC(layer int) uint32 { + w.upTrackMu.RLock() + defer w.upTrackMu.RUnlock() + + if track := w.upTracks[layer]; track != nil { + return uint32(track.SSRC()) + } + return 0 +} + +func (w *WebRTCReceiver) Codec() webrtc.RTPCodecParameters { + return w.codec +} + +func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType { + return w.kind +} + +func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buffer, bestQualityFirst bool) { + if w.closed.get() { + return + } + + var layer int32 + switch track.RID() { + case fullResolution: + layer = 2 + case halfResolution: + layer = 1 + default: + layer = 0 + } + + w.upTrackMu.Lock() + w.upTracks[layer] = track + w.upTrackMu.Unlock() + + w.bufferMu.Lock() + w.buffers[layer] = buff + w.bufferMu.Unlock() + + if w.isSimulcast { + w.addAvailableLayer(uint16(layer), false) + + w.downTrackMu.RLock() + // LK-TODO-START + // DownTrack layer change should not happen directly from here. + // Layer switching should be controlled by StreamAllocator. So, this + // should call into DownTrack to notify availability of a new layer. + // One challenge to think about is that the layer bitrate is not available + // for a second after start up as sfu.Buffer reports at that cadence. + // One possibility is to initialize sfu.Buffer with default bitrate + // based on layer. + // LK-TODO-END + for _, dt := range w.downTracks { + if dt != nil { + if (bestQualityFirst && layer > dt.CurrentSpatialLayer()) || + (!bestQualityFirst && layer < dt.CurrentSpatialLayer()) { + _ = dt.SwitchSpatialLayer(layer, false) + } + } + } + w.downTrackMu.RUnlock() + + // always publish lowest layer + if layer != 0 && w.useTrackers { + tracker := NewStreamTracker() + w.trackers[layer] = tracker + tracker.OnStatusChanged = func(status StreamStatus) { + if status == StreamStatusStopped { + w.removeAvailableLayer(uint16(layer)) + } else { + w.addAvailableLayer(uint16(layer), true) + } + } + tracker.Start() + } + } + go w.forwardRTP(layer) +} + +// SetUpTrackPaused indicates upstream will not be sending any data. +// this will reflect the "muted" status and will pause streamtracker to ensure we don't turn off +// the layer +func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) { + if !w.isSimulcast { + return + } + w.upTrackMu.Lock() + defer w.upTrackMu.Unlock() + for _, tracker := range w.trackers { + if tracker != nil { + tracker.SetPaused(paused) + } + } +} + +func (w *WebRTCReceiver) AddDownTrack(track *DownTrack, bestQualityFirst bool) { + if w.closed.get() { + return + } + + layer := 0 + + w.downTrackMu.RLock() + _, ok := w.index[track.peerID] + w.downTrackMu.RUnlock() + if ok { + return + } + + if w.isSimulcast { + w.upTrackMu.RLock() + for i, t := range w.upTracks { + if t != nil && w.HasSpatialLayer(int32(i)) { + layer = i + if !bestQualityFirst { + break + } + } + } + w.upTrackMu.RUnlock() + + track.SetInitialLayers(int32(layer), 2) + track.maxSpatialLayer.set(2) + track.maxTemporalLayer.set(2) + track.lastSSRC.set(w.SSRC(layer)) + track.trackType = SimulcastDownTrack + track.payload = packetFactory.Get().(*[]byte) + } else { + // LK-TODO-START + // check if any webrtc client does more than one temporal layer when not simulcasting. + // Maybe okay to just set the max temporal layer to 2 even in this case. + // Don't think there is any harm is setting it at 2 if the upper layers are + // not going to be there. + // LK-TODO-END + track.SetInitialLayers(0, 0) + track.trackType = SimpleDownTrack + } + + w.storeDownTrack(track) +} + +func (w *WebRTCReceiver) HasSpatialLayer(layer int32) bool { + layers, ok := w.availableLayers.Load().([]uint16) + if !ok { + return false + } + desired := uint16(layer) + for _, l := range layers { + if l == desired { + return true + } + } + return false +} + +func (w *WebRTCReceiver) downtrackLayerChange(layers []uint16, layerAdded bool) { + w.downTrackMu.RLock() + defer w.downTrackMu.RUnlock() + for _, dt := range w.downTracks { + if dt != nil { + _, _ = dt.UptrackLayersChange(layers, layerAdded) + } + } +} + +func (w *WebRTCReceiver) addAvailableLayer(layer uint16, updateDownTrack bool) { + w.upTrackMu.Lock() + layers, ok := w.availableLayers.Load().([]uint16) + if !ok { + layers = []uint16{} + } + hasLayer := false + for _, l := range layers { + if l == layer { + hasLayer = true + break + } + } + if !hasLayer { + layers = append(layers, layer) + } + w.availableLayers.Store(layers) + w.upTrackMu.Unlock() + + if updateDownTrack { + w.downtrackLayerChange(layers, true) + } +} + +func (w *WebRTCReceiver) removeAvailableLayer(layer uint16) { + w.upTrackMu.Lock() + layers, ok := w.availableLayers.Load().([]uint16) + if !ok { + w.upTrackMu.Unlock() + return + } + newLayers := make([]uint16, 0, 3) + for _, l := range layers { + if l != layer { + newLayers = append(newLayers, l) + } + } + w.availableLayers.Store(newLayers) + w.upTrackMu.Unlock() + // need to immediately switch off unavailable layers + w.downtrackLayerChange(newLayers, false) +} + +func (w *WebRTCReceiver) GetBitrate() [3]uint64 { + var br [3]uint64 + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + for i, buff := range w.buffers { + if buff != nil { + if w.HasSpatialLayer(int32(i)) { + br[i] = buff.Bitrate() + } else { + br[i] = 0 + } + } + } + return br +} + +func (w *WebRTCReceiver) GetBitrateTemporal() [3][4]uint64 { + var br [3][4]uint64 + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + for i, buff := range w.buffers { + if buff != nil { + tls := make([]uint64, 4) + if w.HasSpatialLayer(int32(i)) { + tls = buff.BitrateTemporal() + } + + for j := 0; j < len(br[i]); j++ { + br[i][j] = tls[j] + } + } + } + return br +} + +func (w *WebRTCReceiver) GetBitrateTemporalCumulative() [3][4]uint64 { + // LK-TODO: For SVC tracks, need to accumulate across spatial layers also + var br [3][4]uint64 + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + for i, buff := range w.buffers { + if buff != nil { + tls := make([]uint64, 4) + if w.HasSpatialLayer(int32(i)) { + tls = buff.BitrateTemporalCumulative() + } + + for j := 0; j < len(br[i]); j++ { + br[i][j] = tls[j] + } + } + } + return br +} + +func (w *WebRTCReceiver) GetMaxTemporalLayer() [3]int32 { + var tls [3]int32 + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + for i, buff := range w.buffers { + if buff != nil { + tls[i] = buff.MaxTemporalLayer() + } + } + return tls +} + +// OnCloseHandler method to be called on remote tracked removed +func (w *WebRTCReceiver) OnCloseHandler(fn func()) { + w.onCloseHandler = fn +} + +// DeleteDownTrack removes a DownTrack from a Receiver +func (w *WebRTCReceiver) DeleteDownTrack(peerID string) { + if w.closed.get() { + return + } + + w.downTrackMu.Lock() + defer w.downTrackMu.Unlock() + + idx, ok := w.index[peerID] + if !ok { + return + } + delete(w.index, peerID) + w.downTracks[idx] = nil + w.free[idx] = struct{}{} +} + +func (w *WebRTCReceiver) SendRTCP(p []rtcp.Packet) { + if _, ok := p[0].(*rtcp.PictureLossIndication); ok { + w.rtcpMu.Lock() + defer w.rtcpMu.Unlock() + if time.Now().UnixNano()-w.lastPli.get() < w.pliThrottle { + return + } + w.lastPli.set(time.Now().UnixNano()) + } + + w.rtcpCh <- p +} + +func (w *WebRTCReceiver) SetRTCPCh(ch chan []rtcp.Packet) { + w.rtcpCh = ch +} + +func (w *WebRTCReceiver) GetSenderReportTime(layer int32) (rtpTS uint32, ntpTS uint64) { + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + if w.buffers[layer] != nil { + rtpTS, ntpTS, _ = w.buffers[layer].GetSenderReportData() + } + return +} + +func (w *WebRTCReceiver) RetransmitPackets(track *DownTrack, packets []packetMeta) error { + if w.nackWorker.Stopped() { + return io.ErrClosedPipe + } + // LK-TODO: should move down track specific bits into there + w.nackWorker.Submit(func() { + src := packetFactory.Get().(*[]byte) + for _, meta := range packets { + pktBuff := *src + w.bufferMu.RLock() + buff := w.buffers[meta.layer] + w.bufferMu.RUnlock() + if buff == nil { + break + } + i, err := buff.GetPacket(pktBuff, meta.sourceSeqNo) + if err != nil { + if err == io.EOF { + break + } + continue + } + var pkt rtp.Packet + if err = pkt.Unmarshal(pktBuff[:i]); err != nil { + continue + } + pkt.Header.SequenceNumber = meta.targetSeqNo + pkt.Header.Timestamp = meta.timestamp + pkt.Header.SSRC = track.ssrc + pkt.Header.PayloadType = track.payloadType + + err = track.MaybeTranslateVP8(&pkt, meta) + if err != nil { + Logger.Error(err, "translating VP8 packet err") + continue + } + + err = track.WriteRTPHeaderExtensions(&pkt.Header) + if err != nil { + Logger.Error(err, "writing rtp header extensions err") + continue + } + + if _, err = track.writeStream.WriteRTP(&pkt.Header, pkt.Payload); err != nil { + Logger.Error(err, "Writing rtx packet err") + } else { + track.UpdateStats(uint32(i)) + } + } + packetFactory.Put(src) + }) + return nil +} + +func (w *WebRTCReceiver) forwardRTP(layer int32) { + tracker := w.trackers[layer] + + defer func() { + w.closeOnce.Do(func() { + w.closed.set(true) + w.closeTracks() + }) + if tracker != nil { + tracker.Stop() + } + }() + + pli := []rtcp.Packet{ + &rtcp.PictureLossIndication{SenderSSRC: rand.Uint32(), MediaSSRC: w.SSRC(int(layer))}, + } + + for { + w.bufferMu.RLock() + pkt, err := w.buffers[layer].ReadExtended() + w.bufferMu.RUnlock() + if err == io.EOF { + return + } + + if tracker != nil { + tracker.Observe(pkt.Packet.SequenceNumber) + } + + w.downTrackMu.RLock() + if w.lbThreshold == 0 || len(w.downTracks)-len(w.free) < w.lbThreshold { + // serial - not enough down tracks for parallelization to outweigh overhead + for _, dt := range w.downTracks { + if dt != nil { + w.writeRTP(layer, dt, pkt, pli) + } + } + } else { + // parallel - enables much more efficient multi-core utilization + start := uint64(0) + end := uint64(len(w.downTracks)) + + // 100µs is enough to amortize the overhead and provide sufficient load balancing. + // WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop. + step := uint64(2) + + var wg sync.WaitGroup + wg.Add(w.numProcs) + for p := 0; p < w.numProcs; p++ { + go func() { + defer wg.Done() + for { + n := atomic.AddUint64(&start, step) + if n >= end+step { + return + } + + for i := n - step; i < n && i < end; i++ { + if dt := w.downTracks[i]; dt != nil { + w.writeRTP(layer, dt, pkt, pli) + } + } + } + }() + } + wg.Wait() + } + w.downTrackMu.RUnlock() + } +} + +func (w *WebRTCReceiver) writeRTP(layer int32, dt *DownTrack, pkt *buffer.ExtPacket, pli []rtcp.Packet) { + // LK-TODO-START + // Ideally this code should also be moved into the DownTrack + // structure to keep things modular. Let the down track code + // make decision on forwarding or not + // LK-TODO-END + if w.isSimulcast { + targetLayer := dt.TargetSpatialLayer() + currentLayer := dt.CurrentSpatialLayer() + if targetLayer == layer && currentLayer != targetLayer { + if pkt.KeyFrame { + dt.SwitchSpatialLayerDone(targetLayer) + currentLayer = targetLayer + } else { + dt.lastPli.set(time.Now().UnixNano()) + w.SendRTCP(pli) + } + } + // LK-TODO-START + // Probably need a control here to stop forwarding current layer + // if the current layer is higher than target layer, i. e. target layer + // could have been switched down due to bandwidth constraints and + // continuing to forward higher layer is only going to exacerbate the issue. + // Note that the client might have also requested a lower layer. So, it + // would nice to distinguish between client requested downgrade vs bandwidth + // constrained downgrade and stop higher layer only in the bandwidth + // constrained case. + // LK-TODO-END + if currentLayer != layer { + dt.pktsDropped.add(1) + return + } + } + + if err := dt.WriteRTP(pkt, layer); err != nil { + log.Error().Err(err).Str("id", dt.id).Msg("Error writing to down track") + } +} + +// closeTracks close all tracks from Receiver +func (w *WebRTCReceiver) closeTracks() { + w.downTrackMu.Lock() + for _, dt := range w.downTracks { + if dt != nil { + dt.Close() + } + } + w.downTracks = make([]*DownTrack, 0) + w.index = make(map[string]int) + w.free = make(map[int]struct{}) + w.downTrackMu.Unlock() + + w.nackWorker.StopWait() + if w.onCloseHandler != nil { + w.onCloseHandler() + } +} + +func (w *WebRTCReceiver) storeDownTrack(track *DownTrack) { + w.downTrackMu.Lock() + defer w.downTrackMu.Unlock() + + for idx := range w.free { + w.index[track.peerID] = idx + w.downTracks[idx] = track + delete(w.free, idx) + return + } + + w.index[track.peerID] = len(w.downTracks) + w.downTracks = append(w.downTracks, track) +} + +func (w *WebRTCReceiver) DebugInfo() map[string]interface{} { + info := map[string]interface{}{ + "Simulcast": w.isSimulcast, + "LastPli": w.lastPli, + } + + w.upTrackMu.RLock() + upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks)) + for layer, ut := range w.upTracks { + if ut != nil { + upTrackInfo = append(upTrackInfo, map[string]interface{}{ + "Layer": layer, + "SSRC": ut.SSRC(), + "Msid": ut.Msid(), + "RID": ut.RID(), + }) + } + } + w.upTrackMu.RUnlock() + info["UpTracks"] = upTrackInfo + + return info +} diff --git a/pkg/sfu/receiver_test.go b/pkg/sfu/receiver_test.go new file mode 100644 index 000000000..73d3956a3 --- /dev/null +++ b/pkg/sfu/receiver_test.go @@ -0,0 +1,199 @@ +package sfu + +import ( + "fmt" + "hash/fnv" + "math/rand" + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/gammazero/workerpool" + "github.com/stretchr/testify/assert" +) + +func TestWebRTCReceiver_OnCloseHandler(t *testing.T) { + type args struct { + fn func() + } + tests := []struct { + name string + args args + }{ + { + name: "Must set on close handler function", + args: args{ + fn: func() {}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + w := &WebRTCReceiver{} + w.OnCloseHandler(tt.args.fn) + assert.NotNil(t, w.onCloseHandler) + }) + } +} + +func BenchmarkWriteRTP(b *testing.B) { + cases := []int{1, 2, 5, 10, 100, 250, 500} + workers := runtime.NumCPU() + wp := workerpool.New(workers) + for _, c := range cases { + // fills each bucket with a max of 50, i.e. []int{50, 50} for c=100 + fill := make([]int, 0) + for i := 50; ; i += 50 { + if i > c { + fill = append(fill, c%50) + break + } + + fill = append(fill, 50) + if i == c { + break + } + } + + // splits c into numCPU buckets, i.e. []int{9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8} for 12 cpus and c=100 + split := make([]int, workers) + for i := range split { + split[i] = c / workers + } + for i := 0; i < c%workers; i++ { + split[i]++ + } + + b.Run(fmt.Sprintf("%d-Downtracks/Control", c), func(b *testing.B) { + benchmarkNoPool(b, c) + }) + b.Run(fmt.Sprintf("%d-Downtracks/Pool(Fill)", c), func(b *testing.B) { + benchmarkPool(b, wp, fill) + }) + b.Run(fmt.Sprintf("%d-Downtracks/Pool(Hash)", c), func(b *testing.B) { + benchmarkPool(b, wp, split) + }) + b.Run(fmt.Sprintf("%d-Downtracks/Goroutines", c), func(b *testing.B) { + benchmarkGoroutine(b, split) + }) + b.Run(fmt.Sprintf("%d-Downtracks/LoadBalanced", c), func(b *testing.B) { + benchmarkLoadBalanced(b, workers, 2, c) + }) + b.Run(fmt.Sprintf("%d-Downtracks/LBPool", c), func(b *testing.B) { + benchmarkLoadBalancedPool(b, wp, workers, 2, c) + }) + } +} + +func benchmarkNoPool(b *testing.B, downTracks int) { + for i := 0; i < b.N; i++ { + for dt := 0; dt < downTracks; dt++ { + writeRTP() + } + } +} + +func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) { + for i := 0; i < b.N; i++ { + var wg sync.WaitGroup + for j := range buckets { + downTracks := buckets[j] + if downTracks == 0 { + continue + } + wg.Add(1) + wp.Submit(func() { + defer wg.Done() + for dt := 0; dt < downTracks; dt++ { + writeRTP() + } + }) + } + wg.Wait() + } +} + +func benchmarkGoroutine(b *testing.B, buckets []int) { + for i := 0; i < b.N; i++ { + var wg sync.WaitGroup + for j := range buckets { + downTracks := buckets[j] + if downTracks == 0 { + continue + } + wg.Add(1) + go func() { + defer wg.Done() + for dt := 0; dt < downTracks; dt++ { + writeRTP() + } + }() + } + wg.Wait() + } +} + +func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) { + for i := 0; i < b.N; i++ { + start := uint64(0) + step := uint64(step) + end := uint64(downTracks) + + var wg sync.WaitGroup + wg.Add(numProcs) + for p := 0; p < numProcs; p++ { + go func() { + defer wg.Done() + for { + n := atomic.AddUint64(&start, step) + if n >= end+step { + return + } + + for i := n - step; i < n && i < end; i++ { + writeRTP() + } + } + }() + } + wg.Wait() + } +} + +func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) { + for i := 0; i < b.N; i++ { + start := uint64(0) + step := uint64(step) + end := uint64(downTracks) + + var wg sync.WaitGroup + wg.Add(numProcs) + for p := 0; p < numProcs; p++ { + wp.Submit(func() { + defer wg.Done() + for { + n := atomic.AddUint64(&start, step) + if n >= end+step { + return + } + + for i := n - step; i < n && i < end; i++ { + writeRTP() + } + } + }) + } + wg.Wait() + } +} + +func writeRTP() { + s := []byte("simulate some work") + stop := 1900 + rand.Intn(200) + for j := 0; j < stop; j++ { + h := fnv.New128() + s = h.Sum(s) + } +} diff --git a/pkg/sfu/relay.go b/pkg/sfu/relay.go new file mode 100644 index 000000000..aa3f0133f --- /dev/null +++ b/pkg/sfu/relay.go @@ -0,0 +1,13 @@ +package sfu + +func RelayWithFanOutDataChannels() func(r *relayPeer) { + return func(r *relayPeer) { + r.relayFanOutDataChannels = true + } +} + +func RelayWithSenderReports() func(r *relayPeer) { + return func(r *relayPeer) { + r.withSRReports = true + } +} diff --git a/pkg/sfu/relay/README.md b/pkg/sfu/relay/README.md new file mode 100644 index 000000000..87f4ade34 --- /dev/null +++ b/pkg/sfu/relay/README.md @@ -0,0 +1,81 @@ +# Relay + +`ion-sfu` supports relaying tracks to other ion-SFUs or other services using the ORTC API. + +Using this api allows to quickly send the stream to other services by signaling a single request, after that all the following negotiations are handled internally. + +## API + +### Relay Peer + +The relay peer shares common methods with the Webrtc PeerConnection, so it should be straight forward to use. To create a new relay peer follow below example +: +```go + // Meta holds all the related information of the peer you want to relay. + meta := PeerMeta{ + PeerID : "super-villain-1", + SessionID : "world-domination", + } + // config will hold pion/webrtc related structs required for the connection. + // you should fill according your requirements or leave the defaults. + config := &PeerConfig{} + peer, err := NewPeer(meta, config) + handleErr(err) + + // Now before working with the peer you need to signal the peer to + // your remote sever, the signaling can be whatever method you want (gRPC, RESt, pubsub, etc..) + signalFunc= func (meta PeerMeta, signal []byte) ([]byte, error){ + if meta.session== "world-domination"{ + return RelayToLegionOfDoom(meta, signal) + } + return nil, errors.New("not supported") + } + + // The remote peer should create a new Relay Peer with the metadata and call Answer. + if err:= peer.Offer(signalFunc); err!=nil{ + handleErr(err) + } + + // If there are no errors, relay peer offer some convenience methods to communicate with + // Relayed peer. + + // Emit will fire and forget to the request event + peer.Emit("evil-plan-1", data) + // Request will wait for a remote answer, use a time cancelled + // context to not block forever if peer does not answer + ans,err:= peer.Request(ctx, "evil-plan-2", data) + // To listen to remote event just attach the callback to peer + peer.OnRequest( func (event string, msg Message){ + // to access to request data + msg.Paylod() + // to reply the request + msg.Reply(...) + }) + + // The Relay Peer also has some convenience callbacks to manage the peer lifespan. + + // Peer OnClose is called when the remote peer connection is closed, or the Close method is called + peer.OnClose(func()) + // Peer OnReady is called when the relay peer is ready to start negotiating tracks, data channels and request + // is highly recommended to attach all the initialization logic to this callback + peer.OnReady(func()) + + // To add or receive tracks or data channels the API is similar to webrtc Peer Connection, just listen + // to the required callbacks + peer.OnDataChannel(f func(channel *webrtc.DataChannel)) + peer.OnTrack(f func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver)) + // Make sure to call below methods after the OnReady callback fired. + peer.CreateDataChannel(label string) + peer.AddTrack(receiver *webrtc.RTPReceiver, remoteTrack *webrtc.TrackRemote, +localTrack webrtc.TrackLocal) (*webrtc.RTPSender, error) +``` + +### ION-SFU integration + +ION-SFU offers some convenience methods for relaying peers in a very simple way. + +To relay a peer just call `Peer.Publisher().Relay(...)` then signal the data to the remote SFU and ingest the data using: + +`session.AddRelayPeer(peerID string, signalData []byte) ([]byte, error)` + +set the []byte response from the method as the response of the signaling. And is ready, everytime a peer joins to the new SFU will negotiate the relayed stream. diff --git a/pkg/sfu/relay/relay.go b/pkg/sfu/relay/relay.go new file mode 100644 index 000000000..eafc15d7d --- /dev/null +++ b/pkg/sfu/relay/relay.go @@ -0,0 +1,657 @@ +package relay + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" +) + +const ( + signalerLabel = "ion_sfu_relay_signaler" + signalerRequestEvent = "ion_relay_request" +) + +var ( + ErrRelayPeerNotReady = errors.New("relay Peer is not ready") + ErrRelayPeerSignalDone = errors.New("relay Peer signal already called") + ErrRelaySignalDCNotReady = errors.New("relay Peer data channel is not ready") +) + +type signal struct { + Encodings *webrtc.RTPCodingParameters `json:"encodings,omitempty"` + ICECandidates []webrtc.ICECandidate `json:"iceCandidates,omitempty"` + ICEParameters webrtc.ICEParameters `json:"iceParameters,omitempty"` + DTLSParameters webrtc.DTLSParameters `json:"dtlsParameters,omitempty"` + SCTPCapabilities *webrtc.SCTPCapabilities `json:"sctpCapabilities,omitempty"` + TrackMeta *TrackMeta `json:"trackInfo,omitempty"` +} + +type request struct { + ID uint64 `json:"id"` + IsReply bool `json:"reply"` + Event string `json:"event"` + Payload []byte `json:"payload"` +} + +type TrackMeta struct { + StreamID string `json:"streamId"` + TrackID string `json:"trackId"` + CodecParameters *webrtc.RTPCodecParameters `json:"codecParameters,omitempty"` +} + +type PeerConfig struct { + SettingEngine webrtc.SettingEngine + ICEServers []webrtc.ICEServer + Logger logr.Logger +} + +type PeerMeta struct { + PeerID string `json:"peerId"` + SessionID string `json:"sessionId"` +} + +type Options struct { + // RelayMiddlewareDC if set to true middleware data channels will be created and forwarded + // to the relayed peer + RelayMiddlewareDC bool + // RelaySessionDC if set to true fanout data channels will be created and forwarded to the + // relayed peer + RelaySessionDC bool +} + +type Peer struct { + mu sync.Mutex + rmu sync.Mutex + me *webrtc.MediaEngine + log logr.Logger + api *webrtc.API + ice *webrtc.ICETransport + rand *rand.Rand + meta PeerMeta + sctp *webrtc.SCTPTransport + dtls *webrtc.DTLSTransport + role *webrtc.ICERole + ready bool + senders []*webrtc.RTPSender + receivers []*webrtc.RTPReceiver + pendingRequests map[uint64]chan []byte + localTracks []webrtc.TrackLocal + signalingDC *webrtc.DataChannel + gatherer *webrtc.ICEGatherer + dcIndex uint16 + + onReady atomic.Value // func() + onClose atomic.Value // func() + onRequest atomic.Value // func(event string, message Message) + onDataChannel atomic.Value // func(channel *webrtc.DataChannel) + onTrack atomic.Value // func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver, meta *TrackMeta) +} + +func NewPeer(meta PeerMeta, conf *PeerConfig) (*Peer, error) { + // Prepare ICE gathering options + iceOptions := webrtc.ICEGatherOptions{ + ICEServers: conf.ICEServers, + } + me := webrtc.MediaEngine{} + // Create an API object + api := webrtc.NewAPI(webrtc.WithMediaEngine(&me), webrtc.WithSettingEngine(conf.SettingEngine)) + // Create the ICE gatherer + gatherer, err := api.NewICEGatherer(iceOptions) + if err != nil { + return nil, err + } + // Construct the ICE transport + i := api.NewICETransport(gatherer) + // Construct the DTLS transport + dtls, err := api.NewDTLSTransport(i, nil) + // Construct the SCTP transport + sctp := api.NewSCTPTransport(dtls) + if err != nil { + return nil, err + } + + p := &Peer{ + me: &me, + api: api, + log: conf.Logger, + ice: i, + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + meta: meta, + sctp: sctp, + dtls: dtls, + gatherer: gatherer, + pendingRequests: make(map[uint64]chan []byte), + } + + sctp.OnDataChannel(func(channel *webrtc.DataChannel) { + if channel.Label() == signalerLabel { + p.signalingDC = channel + channel.OnMessage(p.handleRequest) + channel.OnOpen(func() { + if f := p.onReady.Load(); f != nil { + f.(func())() + } + }) + return + } + + if f := p.onDataChannel.Load(); f != nil { + f.(func(dataChannel *webrtc.DataChannel))(channel) + } + }) + + i.OnConnectionStateChange(func(state webrtc.ICETransportState) { + if state == webrtc.ICETransportStateFailed || state == webrtc.ICETransportStateDisconnected { + if err = p.Close(); err != nil { + p.log.Error(err, "Closing relayed p error") + } + } + }) + + return p, nil +} + +func (p *Peer) ID() string { + return p.meta.PeerID +} + +// Offer is used for establish the connection of the local relay Peer +// with the remote relay Peer. +// +// If connection is successful OnReady handler will be called +func (p *Peer) Offer(signalFn func(meta PeerMeta, signal []byte) ([]byte, error)) error { + if p.gatherer.State() != webrtc.ICEGathererStateNew { + return ErrRelayPeerSignalDone + } + + ls := &signal{} + gatherFinished := make(chan struct{}) + p.gatherer.OnLocalCandidate(func(i *webrtc.ICECandidate) { + if i == nil { + close(gatherFinished) + } + }) + // Gather candidates + if err := p.gatherer.Gather(); err != nil { + return err + } + <-gatherFinished + + var err error + + if ls.ICECandidates, err = p.gatherer.GetLocalCandidates(); err != nil { + return err + } + if ls.ICEParameters, err = p.gatherer.GetLocalParameters(); err != nil { + return err + } + if ls.DTLSParameters, err = p.dtls.GetLocalParameters(); err != nil { + return err + } + + sc := p.sctp.GetCapabilities() + ls.SCTPCapabilities = &sc + + role := webrtc.ICERoleControlling + p.role = &role + data, err := json.Marshal(ls) + + remoteSignal, err := signalFn(p.meta, data) + if err != nil { + return err + } + + rs := &signal{} + + if err = json.Unmarshal(remoteSignal, rs); err != nil { + return err + } + + if err = p.start(rs); err != nil { + return err + } + + if p.signalingDC, err = p.createDataChannel(signalerLabel); err != nil { + return err + } + + p.signalingDC.OnOpen(func() { + if f := p.onReady.Load(); f != nil { + f.(func())() + } + }) + p.signalingDC.OnMessage(p.handleRequest) + return nil +} + +// OnClose sets a callback that is called when relay Peer is closed. +func (p *Peer) OnClose(fn func()) { + p.onClose.Store(fn) +} + +// Answer answers the remote Peer signal signalRequest +func (p *Peer) Answer(request []byte) ([]byte, error) { + if p.gatherer.State() != webrtc.ICEGathererStateNew { + return nil, ErrRelayPeerSignalDone + } + + ls := &signal{} + gatherFinished := make(chan struct{}) + p.gatherer.OnLocalCandidate(func(i *webrtc.ICECandidate) { + if i == nil { + close(gatherFinished) + } + }) + // Gather candidates + if err := p.gatherer.Gather(); err != nil { + return nil, err + } + <-gatherFinished + + var err error + + if ls.ICECandidates, err = p.gatherer.GetLocalCandidates(); err != nil { + return nil, err + } + if ls.ICEParameters, err = p.gatherer.GetLocalParameters(); err != nil { + return nil, err + } + if ls.DTLSParameters, err = p.dtls.GetLocalParameters(); err != nil { + return nil, err + } + + sc := p.sctp.GetCapabilities() + ls.SCTPCapabilities = &sc + + role := webrtc.ICERoleControlled + p.role = &role + + rs := &signal{} + if err = json.Unmarshal(request, rs); err != nil { + return nil, err + } + + go func() { + if err = p.start(rs); err != nil { + p.log.Error(err, "Error starting relay") + } + }() + + return json.Marshal(ls) +} + +// WriteRTCP sends a user provided RTCP packet to the connected Peer. If no Peer is connected the +// packet is discarded. It also runs any configured interceptors. +func (p *Peer) WriteRTCP(pkts []rtcp.Packet) error { + _, err := p.dtls.WriteRTCP(pkts) + return err +} + +func (p *Peer) LocalTracks() []webrtc.TrackLocal { + return p.localTracks +} + +// OnReady calls the callback when relay Peer is ready to start sending/receiving and creating DC +func (p *Peer) OnReady(f func()) { + p.onReady.Store(f) +} + +// OnRequest calls the callback when Peer gets a request message from remote Peer +func (p *Peer) OnRequest(f func(event string, msg Message)) { + p.onRequest.Store(f) +} + +// OnDataChannel sets an event handler which is invoked when a data +// channel message arrives from a remote Peer. +func (p *Peer) OnDataChannel(f func(channel *webrtc.DataChannel)) { + p.onDataChannel.Store(f) +} + +// OnTrack sets an event handler which is called when remote track +// arrives from a remote Peer +func (p *Peer) OnTrack(f func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver, meta *TrackMeta)) { + p.onTrack.Store(f) +} + +// Close ends the relay Peer +func (p *Peer) Close() error { + closeErrs := make([]error, 3+len(p.senders)+len(p.receivers)) + for _, sdr := range p.senders { + closeErrs = append(closeErrs, sdr.Stop()) + } + for _, recv := range p.receivers { + closeErrs = append(closeErrs, recv.Stop()) + } + + closeErrs = append(closeErrs, p.sctp.Stop(), p.dtls.Stop(), p.ice.Stop()) + + if f := p.onClose.Load(); f != nil { + f.(func())() + } + + return joinErrs(closeErrs...) +} + +// CreateDataChannel creates a new DataChannel object with the given label +func (p *Peer) CreateDataChannel(label string) (*webrtc.DataChannel, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.ready { + return nil, ErrRelayPeerNotReady + } + + return p.createDataChannel(label) +} + +func (p *Peer) createDataChannel(label string) (*webrtc.DataChannel, error) { + idx := p.dcIndex + p.dcIndex = +1 + dcParams := &webrtc.DataChannelParameters{ + Label: label, + ID: &idx, + Ordered: true, + } + return p.api.NewDataChannel(p.sctp, dcParams) +} + +func (p *Peer) start(s *signal) error { + if err := p.ice.SetRemoteCandidates(s.ICECandidates); err != nil { + return err + } + + if err := p.ice.Start(p.gatherer, s.ICEParameters, p.role); err != nil { + return err + } + + if err := p.dtls.Start(s.DTLSParameters); err != nil { + return err + } + + if s.SCTPCapabilities != nil { + if err := p.sctp.Start(*s.SCTPCapabilities); err != nil { + return err + } + } + p.ready = true + return nil +} + +func (p *Peer) receive(s *signal) error { + var k webrtc.RTPCodecType + switch { + case strings.HasPrefix(s.TrackMeta.CodecParameters.MimeType, "audio/"): + k = webrtc.RTPCodecTypeAudio + case strings.HasPrefix(s.TrackMeta.CodecParameters.MimeType, "video/"): + k = webrtc.RTPCodecTypeVideo + default: + k = webrtc.RTPCodecType(0) + } + if err := p.me.RegisterCodec(*s.TrackMeta.CodecParameters, k); err != nil { + return err + } + + recv, err := p.api.NewRTPReceiver(k, p.dtls) + if err != nil { + return err + } + + if err = recv.Receive(webrtc.RTPReceiveParameters{Encodings: []webrtc.RTPDecodingParameters{ + { + webrtc.RTPCodingParameters{ + RID: s.Encodings.RID, + SSRC: s.Encodings.SSRC, + PayloadType: s.Encodings.PayloadType, + }, + }, + }}); err != nil { + return err + } + + recv.SetRTPParameters(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{*s.TrackMeta.CodecParameters}, + }) + + track := recv.Track() + + if f := p.onTrack.Load(); f != nil { + f.(func(remote *webrtc.TrackRemote, receiver *webrtc.RTPReceiver, meta *TrackMeta))(track, recv, s.TrackMeta) + } + + p.receivers = append(p.receivers, recv) + + return nil +} + +// AddTrack is used to negotiate a track to the remote peer +func (p *Peer) AddTrack(receiver *webrtc.RTPReceiver, remoteTrack *webrtc.TrackRemote, + localTrack webrtc.TrackLocal) (*webrtc.RTPSender, error) { + p.mu.Lock() + defer p.mu.Unlock() + + codec := remoteTrack.Codec() + sdr, err := p.api.NewRTPSender(localTrack, p.dtls) + if err != nil { + return nil, err + } + if err = p.me.RegisterCodec(codec, remoteTrack.Kind()); err != nil { + return nil, err + } + + s := &signal{} + + s.TrackMeta = &TrackMeta{ + StreamID: remoteTrack.StreamID(), + TrackID: remoteTrack.ID(), + CodecParameters: &codec, + } + + s.Encodings = &webrtc.RTPCodingParameters{ + SSRC: sdr.GetParameters().Encodings[0].SSRC, + PayloadType: remoteTrack.PayloadType(), + } + pld, err := json.Marshal(&s) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + if _, err = p.Request(ctx, signalerRequestEvent, pld); err != nil { + return nil, err + } + + params := receiver.GetParameters() + + if err = sdr.Send(webrtc.RTPSendParameters{ + RTPParameters: params, + Encodings: []webrtc.RTPEncodingParameters{ + { + webrtc.RTPCodingParameters{ + SSRC: s.Encodings.SSRC, + PayloadType: s.Encodings.PayloadType, + }, + }, + }, + }); err != nil { + p.log.Error(err, "Send RTPSender failed") + } + + p.localTracks = append(p.localTracks, localTrack) + p.senders = append(p.senders, sdr) + return sdr, nil +} + +// Emit emits the data argument to remote peer. +func (p *Peer) Emit(event string, data []byte) error { + req := request{ + ID: p.rand.Uint64(), + Event: event, + Payload: data, + } + + msg, err := json.Marshal(req) + if err != nil { + return err + } + + return p.signalingDC.Send(msg) +} + +func (p *Peer) Request(ctx context.Context, event string, data []byte) ([]byte, error) { + req := request{ + ID: p.rand.Uint64(), + Event: event, + Payload: data, + } + + msg, err := json.Marshal(req) + if err != nil { + return nil, err + } + + if err = p.signalingDC.Send(msg); err != nil { + return nil, err + } + + resp := make(chan []byte, 1) + + p.rmu.Lock() + p.pendingRequests[req.ID] = resp + p.rmu.Unlock() + + defer func() { + p.rmu.Lock() + delete(p.pendingRequests, req.ID) + p.rmu.Unlock() + }() + + select { + case r := <-resp: + return r, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (p *Peer) handleRequest(msg webrtc.DataChannelMessage) { + mr := &request{} + if err := json.Unmarshal(msg.Data, mr); err != nil { + p.log.Error(err, "Error marshaling remote message", "peer_id", p.meta.PeerID, "session_id", p.meta.SessionID) + return + } + + if mr.Event == signalerRequestEvent && !mr.IsReply { + p.mu.Lock() + defer p.mu.Unlock() + + r := &signal{} + if err := json.Unmarshal(mr.Payload, r); err != nil { + p.log.Error(err, "Error marshaling remote message", "peer_id", p.meta.PeerID, "session_id", p.meta.SessionID) + return + } + if err := p.receive(r); err != nil { + p.log.Error(err, "Error receiving remote track", "peer_id", p.meta.PeerID, "session_id", p.meta.SessionID) + return + } + if err := p.reply(mr.ID, mr.Event, nil); err != nil { + p.log.Error(err, "Error replying message", "peer_id", p.meta.PeerID, "session_id", p.meta.SessionID) + return + } + + return + } + + if mr.IsReply { + p.rmu.Lock() + if c, ok := p.pendingRequests[mr.ID]; ok { + c <- mr.Payload + delete(p.pendingRequests, mr.ID) + } + p.rmu.Unlock() + return + } + + if mr.Event != signalerRequestEvent { + if f := p.onRequest.Load(); f != nil { + f.(func(string, Message))(mr.Event, Message{ + p: p, + event: mr.Event, + id: mr.ID, + msg: mr.Payload, + }) + } + return + } + +} + +func (p *Peer) reply(id uint64, event string, payload []byte) error { + req := request{ + ID: id, + Event: event, + Payload: payload, + IsReply: true, + } + + msg, err := json.Marshal(req) + if err != nil { + return err + } + + if err = p.signalingDC.Send(msg); err != nil { + return err + } + return nil +} + +func joinErrs(errs ...error) error { + var joinErrsR func(string, int, ...error) error + joinErrsR = func(soFar string, count int, errs ...error) error { + if len(errs) == 0 { + if count == 0 { + return nil + } + return fmt.Errorf(soFar) + } + current := errs[0] + next := errs[1:] + if current == nil { + return joinErrsR(soFar, count, next...) + } + count++ + if count == 1 { + return joinErrsR(fmt.Sprintf("%s", current), count, next...) + } else if count == 2 { + return joinErrsR(fmt.Sprintf("1: %s\n2: %s", soFar, current), count, next...) + } + return joinErrsR(fmt.Sprintf("%s\n%d: %s", soFar, count, current), count, next...) + } + return joinErrsR("", 0, errs...) +} + +type Message struct { + p *Peer + event string + id uint64 + msg []byte +} + +func (m *Message) Payload() []byte { + return m.msg +} + +func (m *Message) Reply(msg []byte) error { + return m.p.reply(m.id, m.event, msg) +} diff --git a/pkg/sfu/relaypeer.go b/pkg/sfu/relaypeer.go new file mode 100644 index 000000000..25281e7c4 --- /dev/null +++ b/pkg/sfu/relaypeer.go @@ -0,0 +1,205 @@ +package sfu + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/relay" + "github.com/pion/rtcp" + "github.com/pion/transport/packetio" + "github.com/pion/webrtc/v3" +) + +type RelayPeer struct { + mu sync.RWMutex + + peer *relay.Peer + session Session + router Router + config *WebRTCTransportConfig + tracks []PublisherTrack + relayPeers []*relay.Peer + dataChannels []*webrtc.DataChannel +} + +func NewRelayPeer(peer *relay.Peer, session Session, config *WebRTCTransportConfig) *RelayPeer { + r := newRouter(peer.ID(), session, config) + r.SetRTCPWriter(peer.WriteRTCP) + + rp := &RelayPeer{ + peer: peer, + router: r, + config: config, + session: session, + } + + peer.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver, meta *relay.TrackMeta) { + if recv, pub := r.AddReceiver(receiver, track, meta.TrackID, meta.StreamID); pub { + recv.SetTrackMeta(meta.TrackID, meta.StreamID) + session.Publish(r, recv) + rp.mu.Lock() + rp.tracks = append(rp.tracks, PublisherTrack{track, recv, true}) + for _, lrp := range rp.relayPeers { + if err := rp.createRelayTrack(track, recv, lrp); err != nil { + Logger.V(1).Error(err, "Creating relay track.", "peer_id", peer.ID()) + } + } + rp.mu.Unlock() + } else { + rp.mu.Lock() + rp.tracks = append(rp.tracks, PublisherTrack{track, recv, false}) + rp.mu.Unlock() + } + }) + + return rp +} + +func (r *RelayPeer) GetRouter() Router { + return r.router +} + +func (r *RelayPeer) ID() string { + return r.peer.ID() +} + +func (r *RelayPeer) Relay(signalFn func(meta relay.PeerMeta, signal []byte) ([]byte, error)) (*relay.Peer, error) { + rp, err := relay.NewPeer(relay.PeerMeta{ + PeerID: r.peer.ID(), + SessionID: r.session.ID(), + }, &relay.PeerConfig{ + SettingEngine: r.config.Setting, + ICEServers: r.config.Configuration.ICEServers, + Logger: Logger, + }) + if err != nil { + return nil, fmt.Errorf("relay: %w", err) + } + + rp.OnReady(func() { + r.mu.Lock() + for _, tp := range r.tracks { + if !tp.clientRelay { + // simulcast will just relay client track for now + continue + } + if err = r.createRelayTrack(tp.Track, tp.Receiver, rp); err != nil { + Logger.V(1).Error(err, "Creating relay track.", "peer_id", r.ID()) + } + } + r.relayPeers = append(r.relayPeers, rp) + r.mu.Unlock() + go r.relayReports(rp) + }) + + rp.OnDataChannel(func(channel *webrtc.DataChannel) { + r.mu.Lock() + r.dataChannels = append(r.dataChannels, channel) + r.mu.Unlock() + r.session.AddDatachannel("", channel) + }) + + if err = rp.Offer(signalFn); err != nil { + return nil, fmt.Errorf("relay: %w", err) + } + + return rp, nil +} + +func (r *RelayPeer) DataChannel(label string) *webrtc.DataChannel { + r.mu.RLock() + defer r.mu.RUnlock() + for _, dc := range r.dataChannels { + if dc.Label() == label { + return dc + } + } + return nil +} + +func (r *RelayPeer) createRelayTrack(track *webrtc.TrackRemote, receiver Receiver, rp *relay.Peer) error { + codec := track.Codec() + downTrack, err := NewDownTrack(webrtc.RTPCodecCapability{ + MimeType: codec.MimeType, + ClockRate: codec.ClockRate, + Channels: codec.Channels, + SDPFmtpLine: codec.SDPFmtpLine, + RTCPFeedback: []webrtc.RTCPFeedback{{"nack", ""}, {"nack", "pli"}}, + }, receiver, r.config.BufferFactory, r.ID(), r.config.Router.MaxPacketTrack) + if err != nil { + Logger.V(1).Error(err, "Create Relay downtrack err", "peer_id", r.ID()) + return err + } + + sdr, err := rp.AddTrack(receiver.(*WebRTCReceiver).receiver, track, downTrack) + if err != nil { + Logger.V(1).Error(err, "Relaying track.", "peer_id", r.ID()) + return fmt.Errorf("relay: %w", err) + } + + r.config.BufferFactory.GetOrNew(packetio.RTCPBufferPacket, + uint32(sdr.GetParameters().Encodings[0].SSRC)).(*buffer.RTCPReader).OnPacket(func(bytes []byte) { + pkts, err := rtcp.Unmarshal(bytes) + if err != nil { + Logger.V(1).Error(err, "Unmarshal rtcp reports", "peer_id", r.ID()) + return + } + var rpkts []rtcp.Packet + for _, pkt := range pkts { + switch pk := pkt.(type) { + case *rtcp.PictureLossIndication: + rpkts = append(rpkts, &rtcp.PictureLossIndication{ + SenderSSRC: pk.MediaSSRC, + MediaSSRC: uint32(track.SSRC()), + }) + } + } + + if len(rpkts) > 0 { + if err := r.peer.WriteRTCP(rpkts); err != nil { + Logger.V(1).Error(err, "Sending rtcp relay reports", "peer_id", r.ID()) + } + } + }) + + downTrack.OnCloseHandler(func() { + if err = sdr.Stop(); err != nil { + Logger.V(1).Error(err, "Stopping relay sender.", "peer_id", r.ID()) + } + }) + + receiver.AddDownTrack(downTrack, true) + return nil +} + +func (r *RelayPeer) relayReports(rp *relay.Peer) { + for { + time.Sleep(5 * time.Second) + + var packets []rtcp.Packet + for _, t := range rp.LocalTracks() { + if dt, ok := t.(*DownTrack); ok { + if !dt.bound.get() { + continue + } + if sr := dt.CreateSenderReport(); sr != nil { + packets = append(packets, sr) + } + } + } + + if len(packets) == 0 { + continue + } + + if err := rp.WriteRTCP(packets); err != nil { + if err == io.EOF || err == io.ErrClosedPipe { + return + } + Logger.Error(err, "Sending downtrack reports err") + } + } +} diff --git a/pkg/sfu/router.go b/pkg/sfu/router.go new file mode 100644 index 000000000..ca477e835 --- /dev/null +++ b/pkg/sfu/router.go @@ -0,0 +1,347 @@ +package sfu + +import ( + "sync" + + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/stats" + "github.com/livekit/livekit-server/pkg/sfu/twcc" +) + +// Router defines a track rtp/rtcp Router +type Router interface { + ID() string + AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, trackID, streamID string) (Receiver, bool) + AddDownTracks(s *Subscriber, r Receiver) error + SetRTCPWriter(func([]rtcp.Packet) error) + AddDownTrack(s *Subscriber, r Receiver) (*DownTrack, error) + Stop() +} + +// RouterConfig defines Router configurations +type RouterConfig struct { + WithStats bool `mapstructure:"withstats"` + MaxBandwidth uint64 `mapstructure:"maxbandwidth"` + MaxPacketTrack int `mapstructure:"maxpackettrack"` + AudioLevelInterval int `mapstructure:"audiolevelinterval"` + AudioLevelThreshold uint8 `mapstructure:"audiolevelthreshold"` + AudioLevelFilter int `mapstructure:"audiolevelfilter"` + Simulcast SimulcastConfig `mapstructure:"simulcast"` + PliThrottle int64 `mapstructure:"plithrottle"` +} + +type router struct { + sync.RWMutex + id string + twcc *twcc.Responder + stats map[uint32]*stats.Stream + rtcpCh chan []rtcp.Packet + stopCh chan struct{} + config RouterConfig + session Session + receivers map[string]Receiver + bufferFactory *buffer.Factory + writeRTCP func([]rtcp.Packet) error +} + +// newRouter for routing rtp/rtcp packets +func newRouter(id string, session Session, config *WebRTCTransportConfig) Router { + ch := make(chan []rtcp.Packet, 10) + r := &router{ + id: id, + rtcpCh: ch, + stopCh: make(chan struct{}), + config: config.Router, + session: session, + receivers: make(map[string]Receiver), + stats: make(map[uint32]*stats.Stream), + bufferFactory: config.BufferFactory, + } + + if config.Router.WithStats { + stats.Peers.Inc() + } + + return r +} + +func (r *router) ID() string { + return r.id +} + +func (r *router) Stop() { + r.stopCh <- struct{}{} + + if r.config.WithStats { + stats.Peers.Dec() + } +} + +func (r *router) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.TrackRemote, trackID, streamID string) (Receiver, bool) { + r.Lock() + defer r.Unlock() + + publish := false + + buff, rtcpReader := r.bufferFactory.GetBufferPair(uint32(track.SSRC())) + + buff.OnFeedback(func(fb []rtcp.Packet) { + r.rtcpCh <- fb + }) + + if track.Kind() == webrtc.RTPCodecTypeAudio { + buff.OnAudioLevel(func(level uint8) { + r.session.AudioObserver().observe(streamID, level) + }) + r.session.AudioObserver().addStream(streamID) + + } else if track.Kind() == webrtc.RTPCodecTypeVideo { + if r.twcc == nil { + r.twcc = twcc.NewTransportWideCCResponder(uint32(track.SSRC())) + r.twcc.OnFeedback(func(p rtcp.RawPacket) { + r.rtcpCh <- []rtcp.Packet{&p} + }) + } + buff.OnTransportWideCC(func(sn uint16, timeNS int64, marker bool) { + r.twcc.Push(sn, timeNS, marker) + }) + } + + if r.config.WithStats { + r.stats[uint32(track.SSRC())] = stats.NewStream(buff) + } + + rtcpReader.OnPacket(func(bytes []byte) { + pkts, err := rtcp.Unmarshal(bytes) + if err != nil { + Logger.Error(err, "Unmarshal rtcp receiver packets err") + return + } + for _, pkt := range pkts { + switch pkt := pkt.(type) { + case *rtcp.SourceDescription: + if r.config.WithStats { + for _, chunk := range pkt.Chunks { + if s, ok := r.stats[chunk.Source]; ok { + for _, item := range chunk.Items { + if item.Type == rtcp.SDESCNAME { + s.SetCName(item.Text) + } + } + } + } + } + case *rtcp.SenderReport: + buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime) + if r.config.WithStats { + if st := r.stats[pkt.SSRC]; st != nil { + r.updateStats(st) + } + } + } + } + }) + + recv, ok := r.receivers[trackID] + if !ok { + recv = NewWebRTCReceiver(receiver, track, r.id, WithPliThrottle(r.config.PliThrottle*1e6)) + r.receivers[trackID] = recv + recv.SetRTCPCh(r.rtcpCh) + recv.OnCloseHandler(func() { + if r.config.WithStats { + if track.Kind() == webrtc.RTPCodecTypeVideo { + stats.VideoTracks.Dec() + } else { + stats.AudioTracks.Dec() + } + } + if recv.Kind() == webrtc.RTPCodecTypeAudio { + r.session.AudioObserver().removeStream(track.StreamID()) + } + r.deleteReceiver(trackID, uint32(track.SSRC())) + }) + publish = true + } + + recv.AddUpTrack(track, buff, r.config.Simulcast.BestQualityFirst) + + buff.Bind(receiver.GetParameters(), buffer.Options{ + MaxBitRate: r.config.MaxBandwidth, + }) + + if r.config.WithStats { + if track.Kind() == webrtc.RTPCodecTypeVideo { + stats.VideoTracks.Inc() + } else { + stats.AudioTracks.Inc() + } + } + + return recv, publish +} + +func (r *router) AddDownTracks(s *Subscriber, recv Receiver) error { + r.Lock() + defer r.Unlock() + + if s.noAutoSubscribe { + Logger.Info("peer turns off automatic subscription, skip tracks add") + return nil + } + + if recv != nil { + if _, err := r.AddDownTrack(s, recv); err != nil { + return err + } + s.negotiate() + return nil + } + + if len(r.receivers) > 0 { + for _, rcv := range r.receivers { + if _, err := r.AddDownTrack(s, rcv); err != nil { + return err + } + } + s.negotiate() + } + return nil +} + +func (r *router) SetRTCPWriter(fn func(packet []rtcp.Packet) error) { + r.writeRTCP = fn + go r.sendRTCP() +} + +func (r *router) AddDownTrack(sub *Subscriber, recv Receiver) (*DownTrack, error) { + for _, dt := range sub.GetDownTracks(recv.StreamID()) { + if dt.ID() == recv.TrackID() { + return dt, nil + } + } + + codec := recv.Codec() + if err := sub.me.RegisterCodec(codec, recv.Kind()); err != nil { + return nil, err + } + + downTrack, err := NewDownTrack(webrtc.RTPCodecCapability{ + MimeType: codec.MimeType, + ClockRate: codec.ClockRate, + Channels: codec.Channels, + SDPFmtpLine: codec.SDPFmtpLine, + RTCPFeedback: []webrtc.RTCPFeedback{{"goog-remb", ""}, {"nack", ""}, {"nack", "pli"}}, + }, recv, r.bufferFactory, sub.id, r.config.MaxPacketTrack) + if err != nil { + return nil, err + } + // Create webrtc sender for the peer we are sending track to + if downTrack.transceiver, err = sub.pc.AddTransceiverFromTrack(downTrack, webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + }); err != nil { + return nil, err + } + + // nolint:scopelint + downTrack.OnCloseHandler(func() { + if sub.pc.ConnectionState() != webrtc.PeerConnectionStateClosed { + if err := sub.pc.RemoveTrack(downTrack.transceiver.Sender()); err != nil { + if err == webrtc.ErrConnectionClosed { + return + } + Logger.Error(err, "Error closing down track") + } else { + sub.RemoveDownTrack(recv.StreamID(), downTrack) + sub.negotiate() + } + } + }) + + downTrack.OnBind(func() { + go sub.sendStreamDownTracksReports(recv.StreamID()) + }) + + sub.AddDownTrack(recv.StreamID(), downTrack) + recv.AddDownTrack(downTrack, r.config.Simulcast.BestQualityFirst) + return downTrack, nil +} + +func (r *router) deleteReceiver(track string, ssrc uint32) { + r.Lock() + delete(r.receivers, track) + delete(r.stats, ssrc) + r.Unlock() +} + +func (r *router) sendRTCP() { + for { + select { + case pkts := <-r.rtcpCh: + if err := r.writeRTCP(pkts); err != nil { + Logger.Error(err, "Write rtcp to peer err", "peer_id", r.id) + } + case <-r.stopCh: + return + } + } +} + +func (r *router) updateStats(stream *stats.Stream) { + calculateLatestMinMaxSenderNtpTime := func(cname string) (minPacketNtpTimeInMillisSinceSenderEpoch uint64, maxPacketNtpTimeInMillisSinceSenderEpoch uint64) { + if len(cname) < 1 { + return + } + r.RLock() + defer r.RUnlock() + + for _, s := range r.stats { + if s.GetCName() != cname { + continue + } + + clockRate := s.Buffer.GetClockRate() + srrtp, srntp, _ := s.Buffer.GetSenderReportData() + latestTimestamp, _ := s.Buffer.GetLatestTimestamp() + + fastForwardTimestampInClockRate := fastForwardTimestampAmount(latestTimestamp, srrtp) + fastForwardTimestampInMillis := (fastForwardTimestampInClockRate * 1000) / clockRate + latestPacketNtpTimeInMillisSinceSenderEpoch := ntpToMillisSinceEpoch(srntp) + uint64(fastForwardTimestampInMillis) + + if 0 == minPacketNtpTimeInMillisSinceSenderEpoch || latestPacketNtpTimeInMillisSinceSenderEpoch < minPacketNtpTimeInMillisSinceSenderEpoch { + minPacketNtpTimeInMillisSinceSenderEpoch = latestPacketNtpTimeInMillisSinceSenderEpoch + } + if 0 == maxPacketNtpTimeInMillisSinceSenderEpoch || latestPacketNtpTimeInMillisSinceSenderEpoch > maxPacketNtpTimeInMillisSinceSenderEpoch { + maxPacketNtpTimeInMillisSinceSenderEpoch = latestPacketNtpTimeInMillisSinceSenderEpoch + } + } + return minPacketNtpTimeInMillisSinceSenderEpoch, maxPacketNtpTimeInMillisSinceSenderEpoch + } + + setDrift := func(cname string, driftInMillis uint64) { + if len(cname) < 1 { + return + } + r.RLock() + defer r.RUnlock() + + for _, s := range r.stats { + if s.GetCName() != cname { + continue + } + s.SetDriftInMillis(driftInMillis) + } + } + + cname := stream.GetCName() + + minPacketNtpTimeInMillisSinceSenderEpoch, maxPacketNtpTimeInMillisSinceSenderEpoch := calculateLatestMinMaxSenderNtpTime(cname) + + driftInMillis := maxPacketNtpTimeInMillisSinceSenderEpoch - minPacketNtpTimeInMillisSinceSenderEpoch + + setDrift(cname, driftInMillis) + + stream.CalcStats() +} diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go new file mode 100644 index 000000000..98eafe1c1 --- /dev/null +++ b/pkg/sfu/sequencer.go @@ -0,0 +1,161 @@ +package sfu + +import ( + "sync" + "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +const ( + ignoreRetransmission = 100 // Ignore packet retransmission after ignoreRetransmission milliseconds +) + +type packetMeta struct { + // Original sequence number from stream. + // The original sequence number is used to find the original + // packet from publisher + sourceSeqNo uint16 + // Modified sequence number after offset. + // This sequence number is used for the associated + // down track, is modified according the offsets, and + // must not be shared + targetSeqNo uint16 + // Modified timestamp for current associated + // down track. + timestamp uint32 + // The last time this packet was nack requested. + // Sometimes clients request the same packet more than once, so keep + // track of the requested packets helps to avoid writing multiple times + // the same packet. + // The resolution is 1 ms counting after the sequencer start time. + lastNack uint32 + // Spatial layer of packet + layer uint8 + // Information that differs depending the codec + misc uint64 +} + +func (p *packetMeta) setVP8PayloadMeta(tlz0Idx uint8, picID uint16) { + p.misc = uint64(tlz0Idx)<<16 | uint64(picID) +} + +func (p *packetMeta) getVP8PayloadMeta() (uint8, uint16) { + return uint8(p.misc >> 16), uint16(p.misc) +} + +func (p *packetMeta) packVP8(vp8 *buffer.VP8) { + p.misc = uint64(vp8.FirstByte)<<56 | + uint64(vp8.PictureIDPresent)<<55 | + uint64(vp8.TL0PICIDXPresent)<<54 | + uint64(vp8.TIDPresent)<<53 | + uint64(vp8.KEYIDXPresent)<<52 | + uint64(vp8.PictureID)<<32 | + uint64(vp8.TL0PICIDX)<<24 | + uint64(vp8.TID)<<22 | + uint64(vp8.Y)<<21 | + uint64(vp8.KEYIDX)<<16 | + uint64(vp8.HeaderSize)<<8 +} + +func (p *packetMeta) unpackVP8() *buffer.VP8 { + return &buffer.VP8{ + FirstByte: byte(p.misc >> 56), + PictureIDPresent: int((p.misc >> 55) & 0x1), + PictureID: uint16((p.misc >> 32) & 0xFFFF), + TL0PICIDXPresent: int((p.misc >> 54) & 0x1), + TL0PICIDX: uint8((p.misc >> 24) & 0xFF), + TIDPresent: int((p.misc >> 53) & 0x1), + TID: uint8((p.misc >> 22) & 0x3), + Y: uint8((p.misc >> 21) & 0x1), + KEYIDXPresent: int((p.misc >> 52) & 0x1), + KEYIDX: uint8((p.misc >> 16) & 0x1F), + HeaderSize: int((p.misc >> 8) & 0xFF), + } +} + +// Sequencer stores the packet sequence received by the down track +type sequencer struct { + sync.Mutex + init bool + max int + seq []packetMeta + step int + headSN uint16 + startTime int64 +} + +func newSequencer(maxTrack int) *sequencer { + return &sequencer{ + startTime: time.Now().UnixNano() / 1e6, + max: maxTrack, + seq: make([]packetMeta, maxTrack), + } +} + +func (n *sequencer) push(sn, offSn uint16, timeStamp uint32, layer uint8, head bool) *packetMeta { + n.Lock() + defer n.Unlock() + if !n.init { + n.headSN = offSn + n.init = true + } + + step := 0 + if head { + inc := offSn - n.headSN + for i := uint16(1); i < inc; i++ { + n.step++ + if n.step >= n.max { + n.step = 0 + } + } + step = n.step + n.headSN = offSn + } else { + step = n.step - int(n.headSN-offSn) + if step < 0 { + if step*-1 >= n.max { + Logger.V(0).Info("Old packet received, can not be sequenced", "head", sn, "received", offSn) + return nil + } + step = n.max + step + } + } + n.seq[n.step] = packetMeta{ + sourceSeqNo: sn, + targetSeqNo: offSn, + timestamp: timeStamp, + layer: layer, + } + pm := &n.seq[n.step] + n.step++ + if n.step >= n.max { + n.step = 0 + } + return pm +} + +func (n *sequencer) getSeqNoPairs(seqNo []uint16) []packetMeta { + n.Lock() + meta := make([]packetMeta, 0, 17) + refTime := uint32(time.Now().UnixNano()/1e6 - n.startTime) + for _, sn := range seqNo { + step := n.step - int(n.headSN-sn) - 1 + if step < 0 { + if step*-1 >= n.max { + continue + } + step = n.max + step + } + seq := &n.seq[step] + if seq.targetSeqNo == sn { + if seq.lastNack == 0 || refTime-seq.lastNack > ignoreRetransmission { + seq.lastNack = refTime + meta = append(meta, *seq) + } + } + } + n.Unlock() + return meta +} diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go new file mode 100644 index 000000000..ab97d9d3b --- /dev/null +++ b/pkg/sfu/sequencer_test.go @@ -0,0 +1,99 @@ +package sfu + +import ( + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_sequencer(t *testing.T) { + seq := newSequencer(500) + off := uint16(15) + + for i := uint16(1); i < 520; i++ { + seq.push(i, i+off, 123, 2, true) + } + + time.Sleep(60 * time.Millisecond) + req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} + res := seq.getSeqNoPairs(req) + assert.Equal(t, len(req), len(res)) + for i, val := range res { + assert.Equal(t, val.targetSeqNo, req[i]) + assert.Equal(t, val.sourceSeqNo, req[i]-off) + assert.Equal(t, val.layer, uint8(2)) + } + res = seq.getSeqNoPairs(req) + assert.Equal(t, 0, len(res)) + time.Sleep(150 * time.Millisecond) + res = seq.getSeqNoPairs(req) + assert.Equal(t, len(req), len(res)) + for i, val := range res { + assert.Equal(t, val.targetSeqNo, req[i]) + assert.Equal(t, val.sourceSeqNo, req[i]-off) + assert.Equal(t, val.layer, uint8(2)) + } + + s := seq.push(521, 521+off, 123, 1, true) + var ( + tlzIdx = uint8(15) + picID = uint16(16) + ) + s.setVP8PayloadMeta(tlzIdx, picID) + s.sourceSeqNo = 12 + m := seq.getSeqNoPairs([]uint16{521 + off}) + assert.Equal(t, 1, len(m)) + tlz0, pID := m[0].getVP8PayloadMeta() + assert.Equal(t, tlzIdx, tlz0) + assert.Equal(t, picID, pID) +} + +func Test_sequencer_getNACKSeqNo(t *testing.T) { + type args struct { + seqNo []uint16 + } + type fields struct { + input []uint16 + offset uint16 + } + + tests := []struct { + name string + fields fields + args args + want []uint16 + }{ + { + name: "Should get correct seq numbers", + fields: fields{ + input: []uint16{2, 3, 4, 7, 8}, + offset: 5, + }, + args: args{ + seqNo: []uint16{4 + 5, 5 + 5, 8 + 5}, + }, + want: []uint16{4, 8}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n := newSequencer(500) + + for _, i := range tt.fields.input { + n.push(i, i+tt.fields.offset, 123, 3, true) + } + + g := n.getSeqNoPairs(tt.args.seqNo) + var got []uint16 + for _, sn := range g { + got = append(got, sn.sourceSeqNo) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSeqNoPairs() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/sfu/session.go b/pkg/sfu/session.go new file mode 100644 index 000000000..f762c406c --- /dev/null +++ b/pkg/sfu/session.go @@ -0,0 +1,414 @@ +package sfu + +import ( + "encoding/json" + "sync" + "time" + + "github.com/pion/webrtc/v3" + "github.com/rs/zerolog/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/livekit/livekit-server/pkg/sfu/logger" + "github.com/livekit/livekit-server/pkg/sfu/relay" +) + +// Session represents a set of peers. Transports inside a SessionLocal +// are automatically subscribed to each other. +type Session interface { + ID() string + Publish(router Router, r Receiver) + Subscribe(peer Peer) + AddPeer(peer Peer) + GetPeer(peerID string) Peer + RemovePeer(peer Peer) + AddRelayPeer(peerID string, signalData []byte) ([]byte, error) + AudioObserver() *AudioObserver + AddDatachannel(owner string, dc *webrtc.DataChannel) + GetDCMiddlewares() []*Datachannel + GetFanOutDataChannelLabels() []string + GetDataChannels(peerID, label string) (dcs []*webrtc.DataChannel) + FanOutMessage(origin, label string, msg webrtc.DataChannelMessage) + Peers() []Peer + RelayPeers() []*RelayPeer +} + +type SessionLocal struct { + id string + mu sync.RWMutex + config WebRTCTransportConfig + peers map[string]Peer + relayPeers map[string]*RelayPeer + closed atomicBool + audioObs *AudioObserver + fanOutDCs []string + datachannels []*Datachannel + onCloseHandler func() +} + +const ( + AudioLevelsMethod = "audioLevels" +) + +// NewSession creates a new SessionLocal +func NewSession(id string, dcs []*Datachannel, cfg WebRTCTransportConfig) Session { + s := &SessionLocal{ + id: id, + peers: make(map[string]Peer), + relayPeers: make(map[string]*RelayPeer), + datachannels: dcs, + config: cfg, + audioObs: NewAudioObserver(cfg.Router.AudioLevelThreshold, cfg.Router.AudioLevelInterval, cfg.Router.AudioLevelFilter), + } + go s.audioLevelObserver(cfg.Router.AudioLevelInterval) + return s +} + +// ID return SessionLocal id +func (s *SessionLocal) ID() string { + return s.id +} + +func (s *SessionLocal) AudioObserver() *AudioObserver { + return s.audioObs +} + +func (s *SessionLocal) GetDCMiddlewares() []*Datachannel { + return s.datachannels +} + +func (s *SessionLocal) GetFanOutDataChannelLabels() []string { + s.mu.RLock() + defer s.mu.RUnlock() + fanout := make([]string, len(s.fanOutDCs)) + copy(fanout, s.fanOutDCs) + return fanout +} + +func (s *SessionLocal) AddPeer(peer Peer) { + s.mu.Lock() + s.peers[peer.ID()] = peer + s.mu.Unlock() +} + +func (s *SessionLocal) GetPeer(peerID string) Peer { + s.mu.RLock() + defer s.mu.RUnlock() + return s.peers[peerID] +} + +func (s *SessionLocal) AddRelayPeer(peerID string, signalData []byte) ([]byte, error) { + p, err := relay.NewPeer(relay.PeerMeta{ + PeerID: peerID, + SessionID: s.id, + }, &relay.PeerConfig{ + SettingEngine: s.config.Setting, + ICEServers: s.config.Configuration.ICEServers, + Logger: logger.New(), + }) + if err != nil { + log.Err(err).Msg("Creating relay peer") + return nil, status.Error(codes.Internal, err.Error()) + } + + resp, err := p.Answer(signalData) + if err != nil { + log.Err(err).Msg("Creating answer for relay") + return nil, err + } + + p.OnReady(func() { + rp := NewRelayPeer(p, s, &s.config) + s.mu.Lock() + s.relayPeers[peerID] = rp + s.mu.Unlock() + }) + + p.OnClose(func() { + s.mu.Lock() + delete(s.relayPeers, peerID) + s.mu.Unlock() + }) + + return resp, nil +} + +func (s *SessionLocal) GetRelayPeer(peerID string) *RelayPeer { + s.mu.RLock() + defer s.mu.RUnlock() + return s.relayPeers[peerID] +} + +// RemovePeer removes Peer from the SessionLocal +func (s *SessionLocal) RemovePeer(p Peer) { + pid := p.ID() + Logger.V(0).Info("RemovePeer from SessionLocal", "peer_id", pid, "session_id", s.id) + s.mu.Lock() + if s.peers[pid] == p { + delete(s.peers, pid) + } + peerCount := len(s.peers) + s.mu.Unlock() + + // Close SessionLocal if no peers + if peerCount == 0 { + s.Close() + } +} + +func (s *SessionLocal) AddDatachannel(owner string, dc *webrtc.DataChannel) { + label := dc.Label() + + s.mu.Lock() + for _, lbl := range s.fanOutDCs { + if label == lbl { + s.mu.Unlock() + return + } + } + s.fanOutDCs = append(s.fanOutDCs, label) + peerOwner := s.peers[owner] + s.mu.Unlock() + peers := s.Peers() + peerOwner.Subscriber().RegisterDatachannel(label, dc) + + dc.OnMessage(func(msg webrtc.DataChannelMessage) { + s.FanOutMessage(owner, label, msg) + }) + + for _, p := range peers { + peer := p + if peer.ID() == owner || peer.Subscriber() == nil { + continue + } + ndc, err := peer.Subscriber().AddDataChannel(label) + + if err != nil { + Logger.Error(err, "error adding datachannel") + continue + } + + if peer.Publisher() != nil && peer.Publisher().Relayed() { + peer.Publisher().AddRelayFanOutDataChannel(label) + } + + pid := peer.ID() + ndc.OnMessage(func(msg webrtc.DataChannelMessage) { + s.FanOutMessage(pid, label, msg) + + if peer.Publisher().Relayed() { + for _, rdc := range peer.Publisher().GetRelayedDataChannels(label) { + if msg.IsString { + if err = rdc.SendText(string(msg.Data)); err != nil { + Logger.Error(err, "Sending dc message err") + } + } else { + if err = rdc.Send(msg.Data); err != nil { + Logger.Error(err, "Sending dc message err") + } + } + } + } + }) + + peer.Subscriber().negotiate() + } +} + +// Publish will add a Sender to all peers in current SessionLocal from given +// Receiver +func (s *SessionLocal) Publish(router Router, r Receiver) { + for _, p := range s.Peers() { + // Don't sub to self + if router.ID() == p.ID() || p.Subscriber() == nil { + continue + } + + Logger.V(0).Info("Publishing track to peer", "peer_id", p.ID()) + + if err := router.AddDownTracks(p.Subscriber(), r); err != nil { + Logger.Error(err, "Error subscribing transport to Router") + continue + } + } +} + +// Subscribe will create a Sender for every other Receiver in the SessionLocal +func (s *SessionLocal) Subscribe(peer Peer) { + s.mu.RLock() + fdc := make([]string, len(s.fanOutDCs)) + copy(fdc, s.fanOutDCs) + peers := make([]Peer, 0, len(s.peers)) + for _, p := range s.peers { + if p == peer || p.Publisher() == nil { + continue + } + peers = append(peers, p) + } + s.mu.RUnlock() + + // Subscribe to fan out data channels + for _, label := range fdc { + dc, err := peer.Subscriber().AddDataChannel(label) + if err != nil { + Logger.Error(err, "error adding datachannel") + continue + } + l := label + dc.OnMessage(func(msg webrtc.DataChannelMessage) { + s.FanOutMessage(peer.ID(), l, msg) + + if peer.Publisher().Relayed() { + for _, rdc := range peer.Publisher().GetRelayedDataChannels(l) { + if msg.IsString { + if err = rdc.SendText(string(msg.Data)); err != nil { + Logger.Error(err, "Sending dc message err") + } + } else { + if err = rdc.Send(msg.Data); err != nil { + Logger.Error(err, "Sending dc message err") + } + } + + } + } + }) + } + + // Subscribe to publisher streams + for _, p := range peers { + err := p.Publisher().GetRouter().AddDownTracks(peer.Subscriber(), nil) + if err != nil { + Logger.Error(err, "Subscribing to Router err") + continue + } + } + + // Subscribe to relay streams + for _, p := range s.RelayPeers() { + err := p.GetRouter().AddDownTracks(peer.Subscriber(), nil) + if err != nil { + Logger.Error(err, "Subscribing to Router err") + continue + } + } + + peer.Subscriber().negotiate() +} + +// Peers returns peers in this SessionLocal +func (s *SessionLocal) Peers() []Peer { + s.mu.RLock() + defer s.mu.RUnlock() + p := make([]Peer, 0, len(s.peers)) + for _, peer := range s.peers { + p = append(p, peer) + } + return p +} + +// RelayPeers returns relay peers in this SessionLocal +func (s *SessionLocal) RelayPeers() []*RelayPeer { + s.mu.RLock() + defer s.mu.RUnlock() + p := make([]*RelayPeer, 0, len(s.peers)) + for _, peer := range s.relayPeers { + p = append(p, peer) + } + return p +} + +// OnClose is called when the SessionLocal is closed +func (s *SessionLocal) OnClose(f func()) { + s.onCloseHandler = f +} + +func (s *SessionLocal) Close() { + if !s.closed.set(true) { + return + } + if s.onCloseHandler != nil { + s.onCloseHandler() + } +} + +func (s *SessionLocal) FanOutMessage(origin, label string, msg webrtc.DataChannelMessage) { + dcs := s.GetDataChannels(origin, label) + for _, dc := range dcs { + if msg.IsString { + if err := dc.SendText(string(msg.Data)); err != nil { + Logger.Error(err, "Sending dc message err") + } + } else { + if err := dc.Send(msg.Data); err != nil { + Logger.Error(err, "Sending dc message err") + } + } + } +} + +func (s *SessionLocal) GetDataChannels(peerID, label string) []*webrtc.DataChannel { + s.mu.RLock() + defer s.mu.RUnlock() + dcs := make([]*webrtc.DataChannel, 0, len(s.peers)) + for pid, p := range s.peers { + if peerID == pid { + continue + } + + if p.Subscriber() != nil { + if dc := p.Subscriber().DataChannel(label); dc != nil && dc.ReadyState() == webrtc.DataChannelStateOpen { + dcs = append(dcs, dc) + } + } + + } + for _, rp := range s.relayPeers { + if dc := rp.DataChannel(label); dc != nil { + dcs = append(dcs, dc) + } + } + + return dcs +} + +func (s *SessionLocal) audioLevelObserver(audioLevelInterval int) { + if audioLevelInterval <= 50 { + Logger.V(0).Info("Values near/under 20ms may return unexpected values") + } + if audioLevelInterval == 0 { + audioLevelInterval = 1000 + } + for { + time.Sleep(time.Duration(audioLevelInterval) * time.Millisecond) + if s.closed.get() { + return + } + levels := s.audioObs.Calc() + + if levels == nil { + continue + } + + msg := ChannelAPIMessage{ + Method: AudioLevelsMethod, + Params: levels, + } + + l, err := json.Marshal(&msg) + if err != nil { + Logger.Error(err, "Marshaling audio levels err") + continue + } + + sl := string(l) + dcs := s.GetDataChannels("", APIChannelLabel) + + for _, ch := range dcs { + if err = ch.SendText(sl); err != nil { + Logger.Error(err, "Sending audio levels err") + } + } + } +} diff --git a/pkg/sfu/sfu.go b/pkg/sfu/sfu.go new file mode 100644 index 000000000..81171f3dc --- /dev/null +++ b/pkg/sfu/sfu.go @@ -0,0 +1,278 @@ +package sfu + +import ( + "math/rand" + "net" + "os" + "runtime" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/pion/ice/v2" + "github.com/pion/turn/v2" + "github.com/pion/webrtc/v3" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/stats" +) + +// Logger is an implementation of logr.Logger. If is not provided - will be turned off. +var Logger logr.Logger = logr.Discard() + +// ICEServerConfig defines parameters for ice servers +type ICEServerConfig struct { + URLs []string `mapstructure:"urls"` + Username string `mapstructure:"username"` + Credential string `mapstructure:"credential"` +} + +type Candidates struct { + IceLite bool `mapstructure:"icelite"` + NAT1To1IPs []string `mapstructure:"nat1to1"` +} + +// WebRTCTransportConfig represents Configuration options +type WebRTCTransportConfig struct { + Configuration webrtc.Configuration + Setting webrtc.SettingEngine + Router RouterConfig + BufferFactory *buffer.Factory +} + +type WebRTCTimeoutsConfig struct { + ICEDisconnectedTimeout int `mapstructure:"disconnected"` + ICEFailedTimeout int `mapstructure:"failed"` + ICEKeepaliveInterval int `mapstructure:"keepalive"` +} + +// WebRTCConfig defines parameters for ice +type WebRTCConfig struct { + ICESinglePort int `mapstructure:"singleport"` + ICEPortRange []uint16 `mapstructure:"portrange"` + ICEServers []ICEServerConfig `mapstructure:"iceserver"` + Candidates Candidates `mapstructure:"candidates"` + SDPSemantics string `mapstructure:"sdpsemantics"` + MDNS bool `mapstructure:"mdns"` + Timeouts WebRTCTimeoutsConfig `mapstructure:"timeouts"` +} + +// Config for base SFU +type Config struct { + SFU struct { + Ballast int64 `mapstructure:"ballast"` + WithStats bool `mapstructure:"withstats"` + } `mapstructure:"sfu"` + WebRTC WebRTCConfig `mapstructure:"webrtc"` + Router RouterConfig `mapstructure:"Router"` + Turn TurnConfig `mapstructure:"turn"` + BufferFactory *buffer.Factory + TurnAuth func(username string, realm string, srcAddr net.Addr) ([]byte, bool) +} + +var ( + packetFactory *sync.Pool +) + +// SFU represents an sfu instance +type SFU struct { + sync.RWMutex + webrtc WebRTCTransportConfig + turn *turn.Server + sessions map[string]Session + datachannels []*Datachannel + withStats bool +} + +// NewWebRTCTransportConfig parses our settings and returns a usable WebRTCTransportConfig for creating PeerConnections +func NewWebRTCTransportConfig(c Config) WebRTCTransportConfig { + se := webrtc.SettingEngine{} + se.DisableMediaEngineCopy(true) + + if c.WebRTC.ICESinglePort != 0 { + Logger.Info("Listen on ", "single-port", c.WebRTC.ICESinglePort) + udpListener, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.IP{0, 0, 0, 0}, + Port: c.WebRTC.ICESinglePort, + }) + if err != nil { + panic(err) + } + se.SetICEUDPMux(webrtc.NewICEUDPMux(nil, udpListener)) + } else { + var icePortStart, icePortEnd uint16 + + if c.Turn.Enabled && len(c.Turn.PortRange) == 0 { + icePortStart = sfuMinPort + icePortEnd = sfuMaxPort + } else if len(c.WebRTC.ICEPortRange) == 2 { + icePortStart = c.WebRTC.ICEPortRange[0] + icePortEnd = c.WebRTC.ICEPortRange[1] + } + if icePortStart != 0 || icePortEnd != 0 { + if err := se.SetEphemeralUDPPortRange(icePortStart, icePortEnd); err != nil { + panic(err) + } + } + } + + var iceServers []webrtc.ICEServer + if c.WebRTC.Candidates.IceLite { + se.SetLite(c.WebRTC.Candidates.IceLite) + } else { + for _, iceServer := range c.WebRTC.ICEServers { + s := webrtc.ICEServer{ + URLs: iceServer.URLs, + Username: iceServer.Username, + Credential: iceServer.Credential, + } + iceServers = append(iceServers, s) + } + } + + se.BufferFactory = c.BufferFactory.GetOrNew + + sdpSemantics := webrtc.SDPSemanticsUnifiedPlan + switch c.WebRTC.SDPSemantics { + case "unified-plan-with-fallback": + sdpSemantics = webrtc.SDPSemanticsUnifiedPlanWithFallback + case "plan-b": + sdpSemantics = webrtc.SDPSemanticsPlanB + } + + if c.WebRTC.Timeouts.ICEDisconnectedTimeout == 0 && + c.WebRTC.Timeouts.ICEFailedTimeout == 0 && + c.WebRTC.Timeouts.ICEKeepaliveInterval == 0 { + Logger.Info("No webrtc timeouts found in config, using default ones") + } else { + se.SetICETimeouts( + time.Duration(c.WebRTC.Timeouts.ICEDisconnectedTimeout)*time.Second, + time.Duration(c.WebRTC.Timeouts.ICEFailedTimeout)*time.Second, + time.Duration(c.WebRTC.Timeouts.ICEKeepaliveInterval)*time.Second, + ) + } + + w := WebRTCTransportConfig{ + Configuration: webrtc.Configuration{ + ICEServers: iceServers, + SDPSemantics: sdpSemantics, + }, + Setting: se, + Router: c.Router, + BufferFactory: c.BufferFactory, + } + + if len(c.WebRTC.Candidates.NAT1To1IPs) > 0 { + w.Setting.SetNAT1To1IPs(c.WebRTC.Candidates.NAT1To1IPs, webrtc.ICECandidateTypeHost) + } + + if !c.WebRTC.MDNS { + w.Setting.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + } + + if c.SFU.WithStats { + w.Router.WithStats = true + stats.InitStats() + } + + return w +} + +func init() { + // Init packet factory + packetFactory = &sync.Pool{ + New: func() interface{} { + b := make([]byte, 1460) + return &b + }, + } +} + +// NewSFU creates a new sfu instance +func NewSFU(c Config) *SFU { + // Init random seed + rand.Seed(time.Now().UnixNano()) + // Init ballast + ballast := make([]byte, c.SFU.Ballast*1024*1024) + + if c.BufferFactory == nil { + c.BufferFactory = buffer.NewBufferFactory(c.Router.MaxPacketTrack, Logger) + } + + w := NewWebRTCTransportConfig(c) + + sfu := &SFU{ + webrtc: w, + sessions: make(map[string]Session), + withStats: c.Router.WithStats, + } + + if c.Turn.Enabled { + ts, err := InitTurnServer(c.Turn, c.TurnAuth) + if err != nil { + Logger.Error(err, "Could not init turn server err") + os.Exit(1) + } + sfu.turn = ts + } + + runtime.KeepAlive(ballast) + return sfu +} + +// NewSession creates a new SessionLocal instance +func (s *SFU) newSession(id string) Session { + session := NewSession(id, s.datachannels, s.webrtc).(*SessionLocal) + + session.OnClose(func() { + s.Lock() + delete(s.sessions, id) + s.Unlock() + + if s.withStats { + stats.Sessions.Dec() + } + }) + + s.Lock() + s.sessions[id] = session + s.Unlock() + + if s.withStats { + stats.Sessions.Inc() + } + + return session +} + +// GetSession by id +func (s *SFU) getSession(id string) Session { + s.RLock() + defer s.RUnlock() + return s.sessions[id] +} + +func (s *SFU) GetSession(sid string) (Session, WebRTCTransportConfig) { + session := s.getSession(sid) + if session == nil { + session = s.newSession(sid) + } + return session, s.webrtc +} + +func (s *SFU) NewDatachannel(label string) *Datachannel { + dc := &Datachannel{Label: label} + s.datachannels = append(s.datachannels, dc) + return dc +} + +// GetSessions return all sessions +func (s *SFU) GetSessions() []Session { + s.RLock() + defer s.RUnlock() + sessions := make([]Session, 0, len(s.sessions)) + for _, session := range s.sessions { + sessions = append(sessions, session) + } + return sessions +} diff --git a/pkg/sfu/sfu_test.go b/pkg/sfu/sfu_test.go new file mode 100644 index 000000000..f858247e5 --- /dev/null +++ b/pkg/sfu/sfu_test.go @@ -0,0 +1,360 @@ +package sfu + +import ( + "sync" + "testing" + "time" + + "github.com/lucsky/cuid" + + "github.com/pion/webrtc/v3" + med "github.com/pion/webrtc/v3/pkg/media" + "github.com/stretchr/testify/assert" + + "github.com/livekit/livekit-server/pkg/sfu/logger" +) + +// Init test helpers + +func signalPair(pcOffer *webrtc.PeerConnection, pcAnswer *webrtc.PeerConnection) error { + offer, err := pcOffer.CreateOffer(nil) + if err != nil { + return err + } + gatherComplete := webrtc.GatheringCompletePromise(pcOffer) + if err = pcOffer.SetLocalDescription(offer); err != nil { + return err + } + <-gatherComplete + if err = pcAnswer.SetRemoteDescription(*pcOffer.LocalDescription()); err != nil { + return err + } + + answer, err := pcAnswer.CreateAnswer(nil) + if err != nil { + return err + } + if err = pcAnswer.SetLocalDescription(answer); err != nil { + return err + } + return pcOffer.SetRemoteDescription(*pcAnswer.LocalDescription()) +} + +func sendRTPUntilDone(start, done <-chan struct{}, t *testing.T, track *webrtc.TrackLocalStaticSample) { + <-start + for { + select { + case <-time.After(20 * time.Millisecond): + assert.NoError(t, track.WriteSample(med.Sample{Data: []byte{0x0, 0xff, 0xff, 0xff, 0xff}, Duration: time.Second})) + case <-done: + return + } + } +} + +// newPair creates two new peer connections (an offerer and an answerer) using +// the api. +func newPair(cfg webrtc.Configuration, api *webrtc.API) (pcOffer *webrtc.PeerConnection, pcAnswer *webrtc.PeerConnection, err error) { + pca, err := api.NewPeerConnection(cfg) + if err != nil { + return nil, nil, err + } + + pcb, err := api.NewPeerConnection(cfg) + if err != nil { + return nil, nil, err + } + + return pca, pcb, nil +} + +type media struct { + kind string + id string + tid string +} + +type action struct { + id string + kind string + sleep time.Duration + media []media +} + +type peer struct { + id string + mu sync.Mutex + local *PeerLocal + remotePub *webrtc.PeerConnection + remoteSub *webrtc.PeerConnection + subs sync.WaitGroup + pubs []*sender +} + +type step struct { + actions []*action +} + +type sender struct { + transceiver *webrtc.RTPTransceiver + start chan struct{} +} + +func addMedia(done <-chan struct{}, t *testing.T, pc *webrtc.PeerConnection, media []media) []*sender { + var senders []*sender + for _, media := range media { + var track *webrtc.TrackLocalStaticSample + var err error + + start := make(chan struct{}) + + switch media.kind { + case "audio": + track, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: "audio/opus"}, media.tid, media.id) + assert.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + }) + assert.NoError(t, err) + senders = append(senders, &sender{transceiver: transceiver, start: start}) + case "video": + track, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, media.tid, media.id) + assert.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + }) + assert.NoError(t, err) + senders = append(senders, &sender{transceiver: transceiver, start: start}) + } + + go sendRTPUntilDone(start, done, t, track) + } + return senders +} + +func newTestConfig() Config { + return Config{ + Router: RouterConfig{MaxPacketTrack: 200}, + } +} + +func TestSFU_SessionScenarios(t *testing.T) { + logger.SetGlobalOptions(logger.GlobalConfig{V: 2}) // 2 - TRACE + Logger = logger.New() + config := newTestConfig() + sfu := NewSFU(config) + sfu.NewDatachannel(APIChannelLabel) + tests := []struct { + name string + steps []step + }{ + { + name: "Multiple stream publish", + steps: []step{ + { + actions: []*action{{ + id: "remote1", + kind: "join", + }, { + id: "remote2", + kind: "join", + }}, + }, + { + actions: []*action{{ + id: "remote1", + kind: "publish", + media: []media{ + {kind: "audio", id: "stream1", tid: "audio1"}, + {kind: "video", id: "stream1", tid: "video1"}, + }, + }}, + }, { + actions: []*action{{ + id: "remote2", + kind: "publish", + media: []media{ + {kind: "audio", id: "stream2", tid: "audio2"}, + {kind: "video", id: "stream2", tid: "video2"}, + }, + }}, + }, + { + actions: []*action{{ + id: "remote1", + kind: "unpublish", + media: []media{ + {kind: "audio", id: "stream3", tid: "audio3"}, + {kind: "video", id: "stream3", tid: "video3"}, + }, + }}, + }, + { + actions: []*action{{ + id: "remote2", + kind: "unpublish", + media: []media{ + {kind: "audio", id: "stream1", tid: "audio1"}, + {kind: "video", id: "stream1", tid: "video1"}, + }, + }}, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + testDone := atomicBool(0) + var mu sync.RWMutex + done := make(chan struct{}) + peers := make(map[string]*peer) + + for _, step := range tt.steps { + for _, action := range step.actions { + func() { + switch action.kind { + case "join": + me, _ := getPublisherMediaEngine() + se := webrtc.SettingEngine{} + se.DisableMediaEngineCopy(true) + err := me.RegisterDefaultCodecs() + assert.NoError(t, err) + api := webrtc.NewAPI(webrtc.WithMediaEngine(me), webrtc.WithSettingEngine(se)) + pub, err := api.NewPeerConnection(webrtc.Configuration{}) + assert.NoError(t, err) + sub, err := api.NewPeerConnection(webrtc.Configuration{}) + assert.NoError(t, err) + local := NewPeer(sfu) + _, err = pub.CreateDataChannel("ion-sfu", nil) + p := &peer{id: action.id, remotePub: pub, remoteSub: sub, local: local} + sub.OnTrack(func(track *webrtc.TrackRemote, recv *webrtc.RTPReceiver) { + mu.Lock() + p.subs.Done() + mu.Unlock() + }) + + mu.Lock() + for id, existing := range peers { + if id != action.id { + p.subs.Add(len(existing.pubs)) + } + } + peers[action.id] = p + mu.Unlock() + + p.mu.Lock() + p.remotePub.OnNegotiationNeeded(func() { + p.mu.Lock() + defer p.mu.Unlock() + o, err := p.remotePub.CreateOffer(nil) + assert.NoError(t, err) + err = p.remotePub.SetLocalDescription(o) + assert.NoError(t, err) + a, err := p.local.Answer(o) + assert.NoError(t, err) + err = p.remotePub.SetRemoteDescription(*a) + assert.NoError(t, err) + for _, pub := range p.pubs { + if pub.start != nil { + close(pub.start) + pub.start = nil + } + } + }) + + p.local.OnIceCandidate = func(init *webrtc.ICECandidateInit, i int) { + switch i { + case subscriber: + p.remoteSub.AddICECandidate(*init) + case publisher: + p.remotePub.AddICECandidate(*init) + } + } + + p.local.OnOffer = func(o *webrtc.SessionDescription) { + if testDone.get() { + return + } + p.mu.Lock() + defer p.mu.Unlock() + err := p.remoteSub.SetRemoteDescription(*o) + assert.NoError(t, err) + a, err := p.remoteSub.CreateAnswer(nil) + assert.NoError(t, err) + err = p.remoteSub.SetLocalDescription(a) + assert.NoError(t, err) + go func() { + if testDone.get() { + return + } + err = p.local.SetRemoteDescription(a) + assert.NoError(t, err) + }() + } + + offer, err := p.remotePub.CreateOffer(nil) + assert.NoError(t, err) + gatherComplete := webrtc.GatheringCompletePromise(p.remotePub) + err = p.remotePub.SetLocalDescription(offer) + assert.NoError(t, err) + <-gatherComplete + err = p.local.Join("test sid", cuid.New()) + assert.NoError(t, err) + answer, err := p.local.Answer(*p.remotePub.LocalDescription()) + err = p.remotePub.SetRemoteDescription(*answer) + assert.NoError(t, err) + p.mu.Unlock() + + case "publish": + mu.Lock() + peer := peers[action.id] + peer.mu.Lock() + // all other peers should get sub'd + for id, p := range peers { + if id != peer.id { + p.subs.Add(len(action.media)) + } + } + + peer.pubs = append(peer.pubs, addMedia(done, t, peer.remotePub, action.media)...) + peer.mu.Unlock() + mu.Unlock() + + case "unpublish": + mu.Lock() + peer := peers[action.id] + peer.mu.Lock() + for _, media := range action.media { + for _, pub := range peer.pubs { + if pub.transceiver != nil && pub.transceiver.Sender().Track().ID() == media.tid { + peer.remotePub.RemoveTrack(pub.transceiver.Sender()) + pub.transceiver = nil + } + } + } + peer.mu.Unlock() + mu.Unlock() + } + }() + time.Sleep(1 * time.Second) + } + } + + for _, p := range peers { + p.subs.Wait() + } + testDone.set(true) + close(done) + + for _, p := range peers { + p.mu.Lock() + p.remotePub.Close() + p.remoteSub.Close() + p.local.Close() + p.mu.Unlock() + } + }) + } +} diff --git a/pkg/sfu/simulcast.go b/pkg/sfu/simulcast.go new file mode 100644 index 000000000..58cf521b0 --- /dev/null +++ b/pkg/sfu/simulcast.go @@ -0,0 +1,29 @@ +package sfu + +import "time" + +const ( + quarterResolution = "q" + halfResolution = "h" + fullResolution = "f" +) + +type SimulcastConfig struct { + BestQualityFirst bool `mapstructure:"bestqualityfirst"` + EnableTemporalLayer bool `mapstructure:"enabletemporallayer"` +} + +type simulcastTrackHelpers struct { + switchDelay time.Time + temporalSupported bool + temporalEnabled bool + lTSCalc atomicInt64 + + // VP8Helper temporal helpers + pRefPicID atomicUint16 + refPicID atomicUint16 + lPicID atomicUint16 + pRefTlZIdx atomicUint8 + refTlZIdx atomicUint8 + lTlZIdx atomicUint8 +} diff --git a/pkg/sfu/stats/stream.go b/pkg/sfu/stats/stream.go new file mode 100644 index 000000000..61655c97f --- /dev/null +++ b/pkg/sfu/stats/stream.go @@ -0,0 +1,174 @@ +package stats + +import ( + "math" + "sync" + "sync/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + driftBuckets = []float64{5, 10, 20, 40, 80, 160, math.Inf(+1)} + + drift = prometheus.NewHistogram(prometheus.HistogramOpts{ + Subsystem: "rtp", + Name: "drift_millis", + Buckets: driftBuckets, + }) + + expectedCount = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: "rtp", + Name: "expected", + }) + + receivedCount = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: "rtp", + Name: "received", + }) + + packetCount = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: "rtp", + Name: "packets", + }) + + totalBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Subsystem: "rtp", + Name: "bytes", + }) + + expectedMinusReceived = prometheus.NewSummary(prometheus.SummaryOpts{ + Subsystem: "rtp", + Name: "expected_minus_received", + }) + + lostRate = prometheus.NewSummary(prometheus.SummaryOpts{ + Subsystem: "rtp", + Name: "lost_rate", + }) + + jitter = prometheus.NewSummary(prometheus.SummaryOpts{ + Subsystem: "rtp", + Name: "jitter", + }) + + Sessions = prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: "sfu", + Name: "sessions", + Help: "Current number of sessions", + }) + + Peers = prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: "sfu", + Name: "peers", + Help: "Current number of peers connected", + }) + + AudioTracks = prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: "sfu", + Name: "audio_tracks", + Help: "Current number of audio tracks", + }) + + VideoTracks = prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: "sfu", + Name: "video_tracks", + Help: "Current number of video tracks", + }) +) + +func InitStats() { + prometheus.MustRegister(drift) + prometheus.MustRegister(expectedCount) + prometheus.MustRegister(receivedCount) + prometheus.MustRegister(packetCount) + prometheus.MustRegister(totalBytes) + prometheus.MustRegister(expectedMinusReceived) + prometheus.MustRegister(lostRate) + prometheus.MustRegister(jitter) + prometheus.MustRegister(Sessions) + prometheus.MustRegister(AudioTracks) + prometheus.MustRegister(VideoTracks) +} + +// Stream contains buffer statistics +type Stream struct { + sync.RWMutex + Buffer *buffer.Buffer + cname string + driftInMillis uint64 + hasStats bool + lastStats buffer.Stats + diffStats buffer.Stats +} + +// NewStream constructs a new Stream +func NewStream(buffer *buffer.Buffer) *Stream { + s := &Stream{ + Buffer: buffer, + } + return s +} + +// GetCName returns the cname for a given stream +func (s *Stream) GetCName() string { + s.RLock() + defer s.RUnlock() + + return s.cname +} + +func (s *Stream) SetCName(cname string) { + s.Lock() + defer s.Unlock() + + s.cname = cname +} + +func (s *Stream) SetDriftInMillis(driftInMillis uint64) { + atomic.StoreUint64(&s.driftInMillis, driftInMillis) +} + +func (s *Stream) GetDriftInMillis() uint64 { + return atomic.LoadUint64(&s.driftInMillis) +} + +func (s *Stream) UpdateStats(stats buffer.Stats) (hasDiff bool, diffStats buffer.Stats) { + s.Lock() + defer s.Unlock() + + hadStats := false + + if s.hasStats { + s.diffStats.LastExpected = stats.LastExpected - s.lastStats.LastExpected + s.diffStats.LastReceived = stats.LastReceived - s.lastStats.LastReceived + s.diffStats.PacketCount = stats.PacketCount - s.lastStats.PacketCount + s.diffStats.TotalByte = stats.TotalByte - s.lastStats.TotalByte + hadStats = true + } + + s.lastStats = stats + s.hasStats = true + + return hadStats, s.diffStats +} + +func (s *Stream) CalcStats() { + bufferStats := s.Buffer.GetStats() + driftInMillis := s.GetDriftInMillis() + + hadStats, diffStats := s.UpdateStats(bufferStats) + + drift.Observe(float64(driftInMillis)) + if hadStats { + expectedCount.Add(float64(diffStats.LastExpected)) + receivedCount.Add(float64(diffStats.LastReceived)) + packetCount.Add(float64(diffStats.PacketCount)) + totalBytes.Add(float64(diffStats.TotalByte)) + } + + expectedMinusReceived.Observe(float64(bufferStats.LastExpected - bufferStats.LastReceived)) + lostRate.Observe(float64(bufferStats.LostRate)) + jitter.Observe(bufferStats.Jitter) +} diff --git a/pkg/sfu/streamallocator.go b/pkg/sfu/streamallocator.go new file mode 100644 index 000000000..4498de2bf --- /dev/null +++ b/pkg/sfu/streamallocator.go @@ -0,0 +1,1051 @@ +// +// Design of StreamAllocator +// +// Each participant uses one peer connection for all downstream +// traffic. It is possible that the downstream peer connection +// gets congested. In such an event, the SFU (sender on that +// peer connection) should take measures to mitigate the +// media loss and latency that would result from such a congestion. +// +// This module is supposed to aggregate down stream tracks and +// drive bandwidth allocation with the goals of +// - Try and send highest quality media +// - React as quickly as possible to mitigate congestion +// +// Setup: +// ------ +// The following should be done to set up a stream allocator +// - There will be one of these per subscriber peer connection. +// Created in livekit-sever/transport.go for subscriber type +// peer connections. +// - In `AddSubscribedTrack` of livekit-server/participant.go, the created +// downTrack is added to the stream allocator. +// - In `RemoveSubscribedTrack` of livekit-server/participant.go, +// the downTrack is removed from the stream allocator. +// - Both video and audio tracks are added to this module. Although the +// stream allocator does not act on audio track forwarding, audio track +// information like loss rate may be used to adjust available bandwidth. +// +// Callbacks: +// ---------- +// StreamAllocator registers the following callbacks on all registered down tracks +// - OnREMB: called when down track receives RTCP REMB. Note that REMB is a +// peer connection level aggregate metric. But, it contains all the SSRCs +// used in the calculation of that REMB. So, there could be multiple +// callbacks per RTCP REMB received (one each from down track pertaining +// to the contained SSRCs) with the same estimated channel capacity. +// - AddReceiverReportListener: called when down track received RTCP RR (Receiver Report). +// - OnAvailableLayersChanged: called when the feeding track changes its layers. +// This could happen due to publisher throttling layers due to upstream congestion +// in its path. +// - OnSubscriptionChanged: called when a down track settings are changed resulting +// from client side requests (muting/pausing a video or limiting maximum layer). +// - OnPacketSent: called when a media packet is forwarded by the down track. As +// this happens once per forwarded packet, processing in this callback should be +// kept to a minimum. +// +// The following may be needed depending on the StreamAllocator algorithm +// - OnBitrateUpdate: called periodically to update the bit rate at which a down track +// is forwarding. This can be used to measure any overshoot and adjust allocations +// accordingly. This may have granular information like primary bitrate, retransmitted +// bitrate and padding bitrate. +// +// State machine: +// -------------- +// The most critical component. It should monitor current state of channel and +// take actions to provide the best user experience by striving to achieve the +// goals outlined earlier +// +// States: +// ------ +// - State_PRE_COMMIT: Before the first estimate is committed. +// Estimated channel capacity is initialized to some +// arbitrarily high value to start streaming immediately. +// Serves two purposes +// 1. Gives the bandwidth estimation algorithms data +// 2. Start streaming as soon as a user joins. Imagine +// a user joining a room with 10 participants already +// in it. That user should start receiving streams +// from everybody as soon as possible. +// - State_STABLE: When all streams are forwarded at their optimal requested layers. +// - State_DEFICIENT: When at least one stream is not able to forward optimal requested layers. +// - State_GRATUITOUS_PROBING: When all streams are forwarded at their optimal requested layers, +// but probing for extra capacity to be prepared for cases like +// new participant joining and streaming OR an existing participant +// starting a new stream like enabling camera or screen share. +// +// Signals: +// ------- +// Each state should take action based on these signals and advance the state machine based +// on the result of the action. +// - Signal_ADD_TRACK: A new track has been added. +// - Signal_REMOVE_TRACK: An existing track has been removed. +// - Signal_ESTIMATE_INCREASE: Estimated channel capacity is increasing. +// - Signal_ESTIMATE_DECREASE: Estimated channel capacity is decreasing. Note that when +// channel gets congested, it is possible to get several of these +// in a very short time window. +// - Signal_RECEIVER_REPORT: An RTCP Receiver Report received from some down track. +// - Signal_AVAILABLE_LAYERS_ADD: Available layers of publisher changed, new layer(s) available. +// - Signal_AVAILABLE_LAYERS_REMOVE: Available layers of publisher changed, some previously +// available layer(s) not available anymore. +// - Signal_SUBSCRIPTION_CHANGE: Subscription changed (mute/requested layers changed). +// - Signal_PERIODIC_PING: Periodic ping +// +// There are several interesting challenges which are documented in relevant code below. +// +package sfu + +import ( + "math" + "sync" + "time" + + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" +) + +const ( + InitialChannelCapacity = 100 * 1000 * 1000 // 100 Mbps + + EstimateEpsilon = 2000 // 2 kbps + + BoostPct = 8 + BoostMinBps = 20 * 1000 // 20 kbps + BoostMaxBps = 60 * 1000 // 60 kbps + + GratuitousProbeHeadroomBps = 1 * 1000 * 1000 // if headroom is more than 1 Mbps, don't probe + GratuitousProbePct = 10 + GratuitousProbeMaxBps = 300 * 1000 // 300 kbps + GratuitousProbeMinDurationMs = 500 + GratuitousProbeMaxDurationMs = 600 + + AudioLossWeight = 0.75 + VideoLossWeight = 0.25 +) + +type State int + +const ( + State_PRE_COMMIT State = iota + State_STABLE + State_DEFICIENT + State_GRATUITOUS_PROBING +) + +type Signal int + +const ( + Signal_NONE Signal = iota + Signal_ADD_TRACK + Signal_REMOVE_TRACK + Signal_ESTIMATE_INCREASE + Signal_ESTIMATE_DECREASE + Signal_RECEIVER_REPORT + Signal_AVAILABLE_LAYERS_ADD + Signal_AVAILABLE_LAYERS_REMOVE + Signal_SUBSCRIPTION_CHANGE + Signal_PERIODIC_PING +) + +type BoostMode int + +const ( + BoostMode_LAYER BoostMode = iota + BoostMode_BANDWIDTH +) + +var ( + // LK-TODO-START + // These constants will definitely require more tweaking. + // In fact, simple time tresholded rules most proably will not be enough. + // LK-TODO-END + EstimateCommitMs = 2 * 1000 * time.Millisecond // 2 seconds + ProbeWaitMs = 5 * 1000 * time.Millisecond // 5 seconds + GratuitousProbeWaitMs = 8 * 1000 * time.Millisecond // 8 seconds + BoostWaitMs = 3 * 1000 * time.Millisecond // 3 seconds +) + +// LK-TODO add logger and log interesting events +type StreamAllocator struct { + estimateMu sync.RWMutex + trackingSSRC uint32 + committedChannelCapacity uint64 + lastCommitTime time.Time + prevReceivedEstimate uint64 + receivedEstimate uint64 + lastEstimateDecreaseTime time.Time + + boostMode BoostMode + boostedChannelCapacity uint64 + lastBoostTime time.Time + + tracksMu sync.RWMutex + tracks map[string]*Track + + prober *Prober + + state State + + chMu sync.RWMutex + eventCh chan []Event + runningCh chan struct{} +} + +type Event struct { + Signal Signal + DownTrack *DownTrack +} + +func NewStreamAllocator() *StreamAllocator { + s := &StreamAllocator{ + committedChannelCapacity: InitialChannelCapacity, + lastCommitTime: time.Now(), + receivedEstimate: InitialChannelCapacity, + lastEstimateDecreaseTime: time.Now(), + boostMode: BoostMode_LAYER, + tracks: make(map[string]*Track), + prober: NewProber(), + state: State_PRE_COMMIT, + eventCh: make(chan []Event, 10), + runningCh: make(chan struct{}), + } + s.prober.OnSendProbe(s.onSendProbe) + + return s +} + +func (s *StreamAllocator) Start() { + go s.processEvents() + go s.ping() +} + +func (s *StreamAllocator) Stop() { + s.chMu.Lock() + defer s.chMu.Unlock() + + close(s.runningCh) + close(s.eventCh) +} + +func (s *StreamAllocator) AddTrack(downTrack *DownTrack) { + downTrack.OnREMB(s.onREMB) + downTrack.AddReceiverReportListener(s.onReceiverReport) + downTrack.OnAvailableLayersChanged(s.onAvailableLayersChanged) + downTrack.OnSubscriptionChanged(s.onSubscriptionChanged) + downTrack.OnPacketSent(s.onPacketSent) + + s.tracksMu.Lock() + s.tracks[downTrack.ID()] = NewTrack(downTrack) + s.tracksMu.Unlock() + + s.postEvent(Signal_ADD_TRACK, downTrack) +} + +func (s *StreamAllocator) RemoveTrack(downTrack *DownTrack) { + s.tracksMu.Lock() + + if _, ok := s.tracks[downTrack.ID()]; !ok { + s.tracksMu.Unlock() + return + } + + delete(s.tracks, downTrack.ID()) + s.tracksMu.Unlock() + + s.postEvent(Signal_REMOVE_TRACK, downTrack) +} + +func (s *StreamAllocator) onREMB(downTrack *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) { + // the channel capacity is estimated at a peer connection level. All down tracks + // in the peer connection will end up calling this for a REMB report with + // the same estimated channel capacity. Use a tracking SSRC to lock onto to + // one report. As SSRCs can be dropped over time, update tracking SSRC as needed + // + // A couple of things to keep in mind + // - REMB reports could be sent gratuitously as a way of providing + // periodic feedback, i. e. even if the estimated capacity does not + // change, there could be REMB packets on the wire. Those gratuitous + // REMBs should not trigger anything bad. + // - As each down track will issue this callback for the same REMB packet + // from the wire, theoretically it is possible that one down track's + // callback from previous REMB comes after another down track's callback + // from the new REMB. REMBs could fire very quickly especially when + // the network is entering congestion. + // LK-TODO-START + // Need to check if the same SSRC reports can somehow race, i.e. does pion send + // RTCP dispatch for same SSRC on different threads? If not, the tracking SSRC + // should prevent racing + // LK-TODO-END + + s.estimateMu.Lock() + + found := false + for _, ssrc := range remb.SSRCs { + if ssrc == s.trackingSSRC { + found = true + break + } + } + if !found { + if len(remb.SSRCs) == 0 { + // LK-TODO - log about REMB wihtout SSRCs + s.estimateMu.Unlock() + return + } + + // try to lock to track which is sending this update + for _, ssrc := range remb.SSRCs { + if ssrc == downTrack.SSRC() { + s.trackingSSRC = downTrack.SSRC() + found = true + break + } + } + + if !found { + s.trackingSSRC = remb.SSRCs[0] + } + } + + if s.trackingSSRC != downTrack.SSRC() { + s.estimateMu.Unlock() + return + } + + s.prevReceivedEstimate = s.receivedEstimate + s.receivedEstimate = uint64(remb.Bitrate) + signal := s.maybeCommitEstimate() + s.estimateMu.Unlock() + + if signal != Signal_NONE { + s.postEvent(signal, nil) + } +} + +// LK-TODO-START +// Receiver report stats are not used in the current implementation. +// +// The idea is to use a loss/rtt based estimator and compare against REMB like outlined here +// https://datatracker.ietf.org/doc/html/draft-ietf-rmcat-gcc-02#section-6 +// +// But the implementation could get quite tricky. So, a separate PR dedicated effort for that +// is required. Something like from Chrome, but hopefully much less complicated :-) +// https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/congestion_controller/goog_cc/loss_based_bandwidth_estimation.cc;bpv=0;bpt=1 +// LK-TODO-END +func (s *StreamAllocator) onReceiverReport(downTrack *DownTrack, rr *rtcp.ReceiverReport) { + s.tracksMu.RLock() + defer s.tracksMu.RUnlock() + + if track, ok := s.tracks[downTrack.ID()]; ok { + track.UpdatePacketStats(rr) + } +} + +// called when feeding track's simulcast layer availability changes +func (s *StreamAllocator) onAvailableLayersChanged(downTrack *DownTrack, layerAdded bool) { + // LK-TODO: Look at processing specific downtrack + if layerAdded { + s.postEvent(Signal_AVAILABLE_LAYERS_ADD, downTrack) + } else { + s.postEvent(Signal_AVAILABLE_LAYERS_REMOVE, downTrack) + } +} + +// called when subscription settings changes +func (s *StreamAllocator) onSubscriptionChanged(downTrack *DownTrack) { + // LK-TODO: Look at processing specific downtrack + s.postEvent(Signal_SUBSCRIPTION_CHANGE, downTrack) +} + +// called when DownTrack sends a packet +func (s *StreamAllocator) onPacketSent(downTrack *DownTrack, size int) { + if downTrack.Kind() == webrtc.RTPCodecTypeAudio { + return + } + + s.prober.PacketSent(size) +} + +// called when prober wants to send packets +func (s *StreamAllocator) onSendProbe(bytesToSend int) int { + if bytesToSend <= 0 { + return 0 + } + + s.tracksMu.RLock() + defer s.tracksMu.RUnlock() + + bytesSent := 0 + for _, track := range s.tracks { + sent := track.WritePaddingRTP(bytesToSend) + bytesSent += sent + bytesToSend -= sent + if bytesToSend <= 0 { + break + } + } + + return bytesSent +} + +func (s *StreamAllocator) postEvent(signal Signal, downTrack *DownTrack) { + s.chMu.RLock() + defer s.chMu.RUnlock() + + if !s.isRunning() { + return + } + + s.eventCh <- []Event{Event{ + Signal: signal, + DownTrack: downTrack, + }} +} + +func (s *StreamAllocator) processEvents() { + for events := range s.eventCh { + if events == nil { + return + } + + for _, event := range events { + s.runStateMachine(event) + } + } +} + +func (s *StreamAllocator) isRunning() bool { + select { + case <-s.runningCh: + return false + default: + return true + } +} + +func (s *StreamAllocator) ping() { + ticker := time.NewTicker(time.Second) + + for s.isRunning() { + <-ticker.C + if !s.isRunning() { + return + } + + s.estimateMu.Lock() + signal := s.maybeCommitEstimate() + s.estimateMu.Unlock() + + if signal != Signal_NONE { + s.postEvent(signal, nil) + } + + s.postEvent(Signal_PERIODIC_PING, nil) + } +} + +// LK-TODO-START +// Typically, in a system like this, there are track priorities. +// It is either implemented as policy +// Examples: +// 1. active speaker gets hi-res, all else lo-res +// 2. screen share streams get hi-res, all else lo-res +// OR it is left up to the clients to subscribe explicitly to the quality they want. +// +// When such a policy is implemented, some of the state machine behaviour needs +// to be changed. For example, in State_DEFICIENT when Signal_ADD_TRACK, it does not +// allocate immediately (it allocates only if enough time has passed since last +// estimate decrease or since the last artificial estimate boost). But, if there is +// a policy and the added track falls in the high priority bucket, an allocation +// would be required even in State_DEFICIENT to ensure higher priority streams are +// forwarded without delay. +// LK-TODO-END + +// LK-TODO-START +// A better implementation would be to handle each signal as for a lot of signals, +// all the states do the same thing. But, it is written the following way for +// better readability. Will do the refactor once this code is more stable and +// more people have a chance to get familiar with it. +// LK-TODO-END +func (s *StreamAllocator) runStateMachine(event Event) { + switch s.state { + case State_PRE_COMMIT: + s.runStatePreCommit(event) + case State_STABLE: + s.runStateStable(event) + case State_DEFICIENT: + s.runStateDeficient(event) + case State_GRATUITOUS_PROBING: + s.runStateGratuitousProbing(event) + } +} + +// LK-TODO-START +// Signal_ADD_TRACK is not useful. Probably can get rid of it. +// AVAILABLE_LAYERS_ADD/REMOVE should be how track start should +// be getting an allocation. +// LK-TODO-END +func (s *StreamAllocator) runStatePreCommit(event Event) { + switch event.Signal { + case Signal_ADD_TRACK: + s.allocate() + case Signal_REMOVE_TRACK: + s.allocate() + case Signal_ESTIMATE_INCREASE: + s.allocate() + case Signal_ESTIMATE_DECREASE: + s.allocate() + case Signal_RECEIVER_REPORT: + case Signal_AVAILABLE_LAYERS_ADD: + s.allocate() + case Signal_AVAILABLE_LAYERS_REMOVE: + s.allocate() + case Signal_SUBSCRIPTION_CHANGE: + s.allocate() + case Signal_PERIODIC_PING: + } +} + +func (s *StreamAllocator) runStateStable(event Event) { + switch event.Signal { + case Signal_ADD_TRACK: + s.allocate() + case Signal_REMOVE_TRACK: + // LK-TODO - may want to re-calculate channel usage? + case Signal_ESTIMATE_INCREASE: + // streaming optimally, no need to do anything + case Signal_ESTIMATE_DECREASE: + s.allocate() + case Signal_RECEIVER_REPORT: + case Signal_AVAILABLE_LAYERS_ADD: + s.allocate() + case Signal_AVAILABLE_LAYERS_REMOVE: + s.allocate() + case Signal_SUBSCRIPTION_CHANGE: + s.allocate() + case Signal_PERIODIC_PING: + // if bandwidth estimate has been stable for a while, maybe gratuitously probe + s.maybeGratuitousProbe() + } +} + +// LK-TODO-START +// The current implementation tries to probe using media if the allocation is not optimal. +// +// But, another option to try is using padding only to probe even when deficient. +// In the current impl, starting a new stream or moving stream to a new layer might end up +// affecting more streams, i.e. in the sense that we started a new stream and user sees +// that affecting all the streams. Using padding only means a couple of things +// 1. If padding packets get lost, no issue +// 2. From a user perspective, it will appear like a network glitch and not due to +// starting a new stream. This is not desirable (i. e. just because user does not see +// it as a direct effect of starting a stream does not mean it is preferred). +// But, something to keep in mind in terms of user perception. +// LK-TODO-END +func (s *StreamAllocator) runStateDeficient(event Event) { + switch event.Signal { + case Signal_ADD_TRACK: + s.maybeProbe() + case Signal_REMOVE_TRACK: + s.allocate() + case Signal_ESTIMATE_INCREASE: + // as long as estimate is increasing, keep going. + // Switch to STABLE state if estimate exceeds optimal bandwidth needed. + if s.getChannelCapacity() > s.getOptimalBandwidthUsage() { + s.resetBoost() + s.setState(State_STABLE) + } + case Signal_ESTIMATE_DECREASE: + // stop using the boosted estimate + s.resetBoost() + s.allocate() + case Signal_RECEIVER_REPORT: + case Signal_AVAILABLE_LAYERS_ADD: + s.maybeProbe() + case Signal_AVAILABLE_LAYERS_REMOVE: + s.allocate() + case Signal_SUBSCRIPTION_CHANGE: + s.allocate() + case Signal_PERIODIC_PING: + s.maybeProbe() + } +} + +func (s *StreamAllocator) runStateGratuitousProbing(event Event) { + // for anything that needs a run of allocation, stop the prober as the traffic + // shape will be altered after an allocation. Although prober will take into + // account regular traffic, conservatively stop the prober before an allocation + // to avoid any self-inflicted damaage + switch event.Signal { + case Signal_ADD_TRACK: + s.prober.Reset() + s.allocate() + case Signal_REMOVE_TRACK: + // LK-TODO - may want to re-calculate channel usage? + case Signal_ESTIMATE_INCREASE: + // good, got a better estimate. Prober may or may not have finished. + // Let it continue if it is still running. + case Signal_ESTIMATE_DECREASE: + // stop gratuitous probing immediately and allocate + s.prober.Reset() + s.allocate() + case Signal_RECEIVER_REPORT: + case Signal_AVAILABLE_LAYERS_ADD: + s.prober.Reset() + s.allocate() + case Signal_AVAILABLE_LAYERS_REMOVE: + s.prober.Reset() + s.allocate() + case Signal_SUBSCRIPTION_CHANGE: + s.prober.Reset() + s.allocate() + case Signal_PERIODIC_PING: + if !s.prober.IsRunning() { + // try for more + s.maybeGratuitousProbe() + // LK-TODO - log about more probing here + } + } +} + +func (s *StreamAllocator) setState(state State) { + if s.state != state { + // LK-TODO log state changes + } + + s.state = state +} + +func (s *StreamAllocator) maybeCommitEstimate() Signal { + // commit channel capacity estimate under following rules + // 1. Abs(receivedEstimate - prevReceivedEstimate) < EstimateEpsilon => estimate stable + // 2. time.Since(lastCommitTime) > EstimateCommitMs => to catch long oscillating estimate + if math.Abs(float64(s.receivedEstimate)-float64(s.prevReceivedEstimate)) > EstimateEpsilon { + // too large a change, wait for estimate to settle + return Signal_NONE + } + + if time.Since(s.lastCommitTime) < EstimateCommitMs { + // don't commit too often + return Signal_NONE + } + + if s.receivedEstimate == s.committedChannelCapacity { + // no change in estimate, no need to commit + return Signal_NONE + } + + signal := Signal_ESTIMATE_INCREASE + if s.receivedEstimate < s.committedChannelCapacity { + signal = Signal_ESTIMATE_DECREASE + s.lastEstimateDecreaseTime = time.Now() + } + + s.committedChannelCapacity = s.receivedEstimate + s.lastCommitTime = time.Now() + + return signal +} + +func (s *StreamAllocator) getChannelCapacity() uint64 { + s.estimateMu.RLock() + defer s.estimateMu.RUnlock() + + return s.committedChannelCapacity +} + +func (s *StreamAllocator) allocate() { + // LK-TODO-START + // Introduce some rules for allocate. Some thing like + // - When estimate decreases, immediately. + // Maybe have some threshold for decrease case also before triggering. + // o 5% decrease OR 200 kbps absolute decrease + // - When estimate increases + // o 10% increase - conservative in pushing more data + // o even if 10% increase, do it only once every 10/15/30 seconds + // When estimate goes up/down, there could be multiple updates. The challenge + // is to react quickly, but not too quickly. + // + // Some of the challenges here are + // - Audio packet loss in subscriber PC should be considered. + // If audio loss is too high, throttle video aggressively as + // audio quality trumps anything else in a conferencing application. + // Note that bandwidth estimation algorithms themselves + // might adjust for it and report estimated capacity. + // - Video packet loss should be taken into consideration too. + // - Especially tricky is video start/stop (either track start/stop + // or Simulcast spatial layer switching (temporal layer switching + // is fine)). That requires a key frame which is usually 10X + // the size of a P-frame. So when channel capacity goes down + // switching to a lower spatial layer could cause a temporary + // spike in bitrate exacerbating the already congested channel + // condition. This is a reason to use a Pacer in the path to + // smooth out spikes. But, a Pacer introduces significant + // overhead both in terms of memory (outgoing packets need to + // be queued) and CPU (a high frequency polling thread to drain + // the queued packets on to the wire at predictable rate) + // - Video retranmission rate should be taken into account while + // allocating to check which layer of publisher will + // fit in available bandwidth. + // - Increasing channel capacity is a tricky one. Some times, + // the bandwidth estimators will not report an increase unless + // the channel is probed with more traffic. So, may have to + // trigger an allocation if the channel is stable for a while + // and send some extra streams. Another option is to just + // send RTP padding only packets to probe the channel which + // can be done on an existing stream without re-enabling a + // stream. + // - There is also the issue of time synchronization. This makes + // debugging/simulating scenarios difficult. Basically, there + // are various kinds of delays in the system. So, when something + // really happened and when we are really responding is always + // going to be offset. So, need to be cognizant of that and + // apply necessary corrections whenever possible. For example + // o Bandwidth estimation itself takes time + // o RTCP packets could be lost + // o RTCP Receiver Report loss could have happened a while back. + // As RTCP RR usually reported once a second or so, if there + // is loss, there is no indication if that loss happened at + // the beginnning of the window or not. + // o When layer switching, there are more round trips needed. + // A PLI has to go to the publisher and publisher has to + // generate a key frame. Another very important event + // (generation of a key frame) happens potentially 100s of ms + // after we asked for it. + // In general, just need to be aware of these and tweak allocation + // to not cause oscillations. + // LK-TODO-END + + // + // LK-TODO-START + // Calculate the aggregate loss. This may or may not + // be necessary depending on the algorithm we choose. In this + // pass, we could also calculate audio & video track loss + // separately and use different rules. + // + // The loss calculation should be for the window between last + // allocation and now. The `lastPackets*` field in + // `Track` structure is used to cache the packet stats + // at the last allocation. Potentially need to think about + // giving higher weight to recent losses. So, might have + // to update the `lastPackets*` periodically even when + // there is no allocation for a long time to ensure loss calculation + // remains fresh. + // LK-TODO-END + // + + s.tracksMu.RLock() + // + // Ask down tracks adjust their forwarded layers. + // It is possible that tracks might all fit under boosted bandwidth scenarios. + // So, track total requested bandwidth and mark DEFICIENT state if the total is + // above the estimated channel capacity even if the optimal signal is true. + // + isOptimal := true + totalBandwidthRequested := uint64(0) + committedChannelCapacity := s.getChannelCapacity() + availableChannelCapacity := committedChannelCapacity + if availableChannelCapacity < s.boostedChannelCapacity { + availableChannelCapacity = s.boostedChannelCapacity + } + for _, track := range s.tracks { + // + // `audio` tracks will do nothing in this method. + // + // `video` tracks could do one of the following + // - no change, i. e. currently forwarding optimal available + // layer and there is enough bandwidth for that. + // - adjust layers up or down + // - mute if there is not enough capacity for any layer + // NOTE: When video switches layers, on layer switch up, + // the current layer can keep forwarding to ensure smooth + // video at the client. As layer up usually means there is + // enough bandwidth, the lower layer can keep streaming till + // the switch point for higher layer becomes available. + // But, in the other direction, higher layer forwarding should + // be stopped immediately to not further congest the channel. + // + // + bandwidthRequested, optimalBandwidthNeeded := track.AdjustAllocation(availableChannelCapacity) + totalBandwidthRequested += bandwidthRequested + if optimalBandwidthNeeded > 0 && bandwidthRequested < optimalBandwidthNeeded { + // + // Assuming this is a prioritized list of tracks + // and we are walking down in that priority order. + // Once one of those streams do not fit, set + // the availableChannelCapacity to 0 so that no + // other lower priority stream gets forwarded. + // Note that a lower priority stream may have + // a layer which might fit in the left over + // capacity. This is one type of policy + // implementation. There may be other policies + // which might allow lower priority to go through too. + // So, we need some sort of policy framework here + // to decide which streams get priority + // + availableChannelCapacity = 0 + isOptimal = false + } else { + availableChannelCapacity -= bandwidthRequested + } + } + s.tracksMu.RUnlock() + + if !isOptimal || totalBandwidthRequested > committedChannelCapacity { + s.setState(State_DEFICIENT) + } else { + if committedChannelCapacity != InitialChannelCapacity { + s.resetBoost() + s.setState(State_STABLE) + } + } + + // + // The above loop may become a concern. In a typical conference + // kind of scenario, there are probably not that many people, so + // the number of down tracks will be limited. + // + // But, can imagine a case of roomless having a single peer + // connection between RTC node and a relay where all the streams + // (even spanning multiple rooms) are on a single peer connection. + // In that case, I think this should mostly be disabled, i. e. + // that peer connection should be looked at as RTC node's publisher + // peer connection and any throttling mechanisms should be disabled. + // +} + +func (s *StreamAllocator) getExpectedBandwidthUsage() uint64 { + s.tracksMu.RLock() + defer s.tracksMu.RUnlock() + + expected := uint64(0) + for _, track := range s.tracks { + expected += track.BandwidthRequested() + } + + return expected +} + +func (s *StreamAllocator) getOptimalBandwidthUsage() uint64 { + s.tracksMu.RLock() + defer s.tracksMu.RUnlock() + + optimal := uint64(0) + for _, track := range s.tracks { + optimal += track.BandwidthOptimal() + } + + return optimal +} + +// LK-TODO: unused till loss based estimation is done, but just a sample impl of weighting audio higher +func (s *StreamAllocator) calculateLoss() float32 { + s.tracksMu.RLock() + defer s.tracksMu.RUnlock() + + packetsAudio := uint32(0) + packetsLostAudio := uint32(0) + packetsVideo := uint32(0) + packetsLostVideo := uint32(0) + for _, track := range s.tracks { + kind, packets, packetsLost := track.GetPacketStats() + + if kind == webrtc.RTPCodecTypeAudio { + packetsAudio += packets + packetsLostAudio += packetsLost + } + + if kind == webrtc.RTPCodecTypeVideo { + packetsVideo += packets + packetsLostVideo += packetsLost + } + } + + audioLossPct := float32(0.0) + if packetsAudio != 0 { + audioLossPct = (float32(packetsLostAudio) * 100.0) / float32(packetsAudio) + } + + videoLossPct := float32(0.0) + if packetsVideo != 0 { + videoLossPct = (float32(packetsLostVideo) * 100.0) / float32(packetsVideo) + } + + return AudioLossWeight*audioLossPct + VideoLossWeight*videoLossPct +} + +func (s *StreamAllocator) maybeProbe() { + if !s.isTimeToBoost() { + return + } + + switch s.boostMode { + case BoostMode_LAYER: + s.maybeBoostLayer() + case BoostMode_BANDWIDTH: + s.maybeBoostBandwidth() + } +} + +func (s *StreamAllocator) maybeBoostLayer() { + s.tracksMu.RLock() + for _, track := range s.tracks { + boosted, additionalBandwidth := track.IncreaseAllocation() + if boosted { + if s.boostedChannelCapacity > s.committedChannelCapacity { + s.boostedChannelCapacity += additionalBandwidth + } else { + s.boostedChannelCapacity = s.committedChannelCapacity + additionalBandwidth + } + s.lastBoostTime = time.Now() + break + } + } + s.tracksMu.RUnlock() +} + +func (s *StreamAllocator) maybeBoostBandwidth() { + // temporarily boost estimate for probing. + // Boost either the committed channel capacity or previous boost point if there is one + baseBps := s.getChannelCapacity() + if baseBps < s.boostedChannelCapacity { + baseBps = s.boostedChannelCapacity + } + + boostBps := (baseBps * BoostPct) / 100 + if boostBps < BoostMinBps { + boostBps = BoostMinBps + } + if boostBps > BoostMaxBps { + boostBps = BoostMaxBps + } + + s.boostedChannelCapacity = baseBps + boostBps + s.lastBoostTime = time.Now() + + s.allocate() +} + +func (s *StreamAllocator) isTimeToBoost() bool { + // if enough time has passed since last esitmate drop or last estimate boost, + // artificially boost estimate before allocating. + // Checking against last estimate boost prevents multiple artificial boosts + // in situations where multiple tracks become available in a short span. + if !s.lastBoostTime.IsZero() { + return time.Since(s.lastBoostTime) > BoostWaitMs + } else { + return time.Since(s.lastEstimateDecreaseTime) > ProbeWaitMs + } +} + +func (s *StreamAllocator) resetBoost() { + s.lastBoostTime = time.UnixMilli(0) + s.boostedChannelCapacity = 0 +} + +func (s *StreamAllocator) maybeGratuitousProbe() { + if time.Since(s.lastEstimateDecreaseTime) < GratuitousProbeWaitMs { + return + } + + committedChannelCapacity := s.getChannelCapacity() + expectedRateBps := s.getExpectedBandwidthUsage() + headroomBps := committedChannelCapacity - expectedRateBps + if headroomBps > GratuitousProbeHeadroomBps { + return + } + + probeRateBps := (committedChannelCapacity * GratuitousProbePct) / 100 + if probeRateBps > GratuitousProbeMaxBps { + probeRateBps = GratuitousProbeMaxBps + } + + s.prober.AddCluster(int(committedChannelCapacity+probeRateBps), int(expectedRateBps), GratuitousProbeMinDurationMs, GratuitousProbeMaxDurationMs) + + s.setState(State_GRATUITOUS_PROBING) +} + +type Track struct { + // LK-TODO-START + // Check if we can do without a lock? + // + // Packet stats are updated in a different thread. + // Maybe a specific lock for that? + // There may be more in the future though. + // LK-TODO-END + lock sync.RWMutex + + downTrack *DownTrack + + highestSN uint32 + packetsLost uint32 + lastHighestSN uint32 + lastPacketsLost uint32 + + bandwidthRequested uint64 + optimalBandwidthNeeded uint64 +} + +func NewTrack(downTrack *DownTrack) *Track { + return &Track{ + downTrack: downTrack, + } +} + +// LK-TODO this should probably be maintained in downTrack and this module can query what it needs +func (t *Track) UpdatePacketStats(rr *rtcp.ReceiverReport) { + t.lock.Lock() + defer t.lock.Unlock() + + t.lastHighestSN = t.highestSN + t.lastPacketsLost = t.packetsLost + + for _, report := range rr.Reports { + if report.LastSequenceNumber > t.highestSN { + t.highestSN = report.LastSequenceNumber + } + if report.TotalLost > t.packetsLost { + t.packetsLost = report.TotalLost + } + } +} + +func (t *Track) GetPacketStats() (webrtc.RTPCodecType, uint32, uint32) { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.downTrack.Kind(), t.highestSN - t.lastHighestSN, t.packetsLost - t.lastPacketsLost +} + +func (t *Track) WritePaddingRTP(bytesToSend int) int { + return t.downTrack.WritePaddingRTP(bytesToSend) +} + +func (t *Track) AdjustAllocation(availableChannelCapacity uint64) (uint64, uint64) { + t.bandwidthRequested, t.optimalBandwidthNeeded = t.downTrack.AdjustAllocation(availableChannelCapacity) + return t.bandwidthRequested, t.optimalBandwidthNeeded +} + +func (t *Track) IncreaseAllocation() (bool, uint64) { + increased, bandwidthRequested, optimalBandwidthNeeded := t.downTrack.IncreaseAllocation() + additionalBandwidth := 0 + if increased { + additionalBandwidth = int(bandwidthRequested) - int(t.bandwidthRequested) + if additionalBandwidth < 0 { + additionalBandwidth = 0 + } + + t.bandwidthRequested = bandwidthRequested + t.optimalBandwidthNeeded = optimalBandwidthNeeded + } + + return increased, uint64(additionalBandwidth) +} + +func (t *Track) BandwidthRequested() uint64 { + return t.bandwidthRequested +} + +func (t *Track) BandwidthOptimal() uint64 { + return t.optimalBandwidthNeeded +} diff --git a/pkg/sfu/streamtracker.go b/pkg/sfu/streamtracker.go new file mode 100644 index 000000000..5896ff35f --- /dev/null +++ b/pkg/sfu/streamtracker.go @@ -0,0 +1,147 @@ +package sfu + +import ( + "sync/atomic" + "time" +) + +type StreamStatus int32 + +func (s StreamStatus) String() string { + switch s { + case StreamStatusStopped: + return "stopped" + case StreamStatusActive: + return "active" + default: + return "unknown" + } +} + +const ( + StreamStatusStopped StreamStatus = 0 + StreamStatusActive StreamStatus = 1 +) + +// StreamTracker keeps track of packet flow and ensures a particular uptrack is consistently producing +// It runs its own goroutine for detection, and fires OnStatusChanged callback +type StreamTracker struct { + // number of samples needed per cycle + SamplesRequired uint32 + // number of cycles needed to be active + CyclesRequired uint64 + CycleDuration time.Duration + OnStatusChanged func(StreamStatus) + paused atomicBool + status atomicInt32 // stores StreamStatus + countSinceLast uint32 // number of packets received since last check + running chan struct{} + + // only access within detectWorker + cycleCount uint64 + + // only access by the same goroutine as Observe + lastSN uint16 +} + +func NewStreamTracker() *StreamTracker { + s := &StreamTracker{ + SamplesRequired: 5, + CyclesRequired: 60, // 30s of continuous stream + CycleDuration: 500 * time.Millisecond, + } + s.status.set(int32(StreamStatusActive)) + return s +} + +func (s *StreamTracker) Status() StreamStatus { + return StreamStatus(s.status.get()) +} + +func (s *StreamTracker) setStatus(status StreamStatus) { + if status != s.Status() { + s.status.set(int32(status)) + if s.OnStatusChanged != nil { + s.OnStatusChanged(status) + } + } +} + +func (s *StreamTracker) Start() { + if s.isRunning() { + return + } + s.running = make(chan struct{}) + go s.detectWorker() +} + +func (s *StreamTracker) Stop() { + if s.running != nil { + close(s.running) + s.running = nil + } +} + +func (s *StreamTracker) SetPaused(paused bool) { + s.paused.set(paused) +} + +func (s *StreamTracker) isRunning() bool { + if s.running == nil { + return false + } + select { + case <-s.running: + return false + default: + return true + } +} + +// Observe a packet that's received +func (s *StreamTracker) Observe(sn uint16) { + if s.paused.get() { + return + } + // ignore out-of-order SNs + if (sn - s.lastSN) > uint16(1<<15) { + return + } + s.lastSN = sn + atomic.AddUint32(&s.countSinceLast, 1) +} + +func (s *StreamTracker) detectWorker() { + ticker := time.NewTicker(s.CycleDuration) + + for s.isRunning() { + <-ticker.C + if !s.isRunning() { + return + } + + s.detectChanges() + } +} + +func (s *StreamTracker) detectChanges() { + if s.paused.get() { + return + } + + if atomic.LoadUint32(&s.countSinceLast) >= s.SamplesRequired { + s.cycleCount += 1 + } else { + s.cycleCount = 0 + } + + if s.cycleCount == 0 && s.Status() == StreamStatusActive { + // flip to stopped + s.setStatus(StreamStatusStopped) + } else if s.cycleCount >= s.CyclesRequired && s.Status() == StreamStatusStopped { + // flip to active + s.setStatus(StreamStatusActive) + } + + atomic.StoreUint32(&s.countSinceLast, 0) +} diff --git a/pkg/sfu/streamtracker_test.go b/pkg/sfu/streamtracker_test.go new file mode 100644 index 000000000..82de99d8c --- /dev/null +++ b/pkg/sfu/streamtracker_test.go @@ -0,0 +1,45 @@ +package sfu + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStreamTracker(t *testing.T) { + t.Run("flips to inactive immediately", func(t *testing.T) { + callbackCalled := atomicBool(0) + tracker := NewStreamTracker() + tracker.OnStatusChanged = func(status StreamStatus) { + callbackCalled.set(true) + } + require.Equal(t, StreamStatusActive, tracker.Status()) + + // run a single interation + tracker.detectChanges() + require.Equal(t, StreamStatusStopped, tracker.Status()) + require.True(t, callbackCalled.get()) + }) + + t.Run("flips back to active after iterations", func(t *testing.T) { + tracker := NewStreamTracker() + tracker.CyclesRequired = 2 + tracker.SamplesRequired = 1 + tracker.setStatus(StreamStatusStopped) + + tracker.Observe(1) + tracker.detectChanges() + require.Equal(t, StreamStatusStopped, tracker.Status()) + + tracker.Observe(2) + tracker.detectChanges() + require.Equal(t, StreamStatusActive, tracker.Status()) + }) + + t.Run("does not change to inactive when paused", func(t *testing.T) { + tracker := NewStreamTracker() + tracker.SetPaused(true) + tracker.detectChanges() + require.Equal(t, StreamStatusActive, tracker.Status()) + }) +} diff --git a/pkg/sfu/subscriber.go b/pkg/sfu/subscriber.go new file mode 100644 index 000000000..63ea9e0c2 --- /dev/null +++ b/pkg/sfu/subscriber.go @@ -0,0 +1,322 @@ +package sfu + +import ( + "context" + "io" + "sync" + "time" + + "github.com/bep/debounce" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" +) + +const APIChannelLabel = "ion-sfu" + +type Subscriber struct { + sync.RWMutex + + id string + pc *webrtc.PeerConnection + me *webrtc.MediaEngine + + tracks map[string][]*DownTrack + channels map[string]*webrtc.DataChannel + candidates []webrtc.ICECandidateInit + + negotiate func() + closeOnce sync.Once + + noAutoSubscribe bool +} + +// NewSubscriber creates a new Subscriber +func NewSubscriber(id string, cfg WebRTCTransportConfig) (*Subscriber, error) { + me, err := getSubscriberMediaEngine() + if err != nil { + Logger.Error(err, "NewPeer error") + return nil, errPeerConnectionInitFailed + } + api := webrtc.NewAPI(webrtc.WithMediaEngine(me), webrtc.WithSettingEngine(cfg.Setting)) + pc, err := api.NewPeerConnection(cfg.Configuration) + + if err != nil { + Logger.Error(err, "NewPeer error") + return nil, errPeerConnectionInitFailed + } + + s := &Subscriber{ + id: id, + me: me, + pc: pc, + tracks: make(map[string][]*DownTrack), + channels: make(map[string]*webrtc.DataChannel), + noAutoSubscribe: false, + } + + pc.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + Logger.V(1).Info("ice connection status", "state", connectionState) + switch connectionState { + case webrtc.ICEConnectionStateFailed: + fallthrough + case webrtc.ICEConnectionStateClosed: + s.closeOnce.Do(func() { + Logger.V(1).Info("webrtc ice closed", "peer_id", s.id) + if err := s.Close(); err != nil { + Logger.Error(err, "webrtc transport close err") + } + }) + } + }) + + go s.downTracksReports() + + return s, nil +} + +func (s *Subscriber) AddDatachannel(peer Peer, dc *Datachannel) error { + ndc, err := s.pc.CreateDataChannel(dc.Label, &webrtc.DataChannelInit{}) + if err != nil { + return err + } + + mws := newDCChain(dc.middlewares) + p := mws.Process(ProcessFunc(func(ctx context.Context, args ProcessArgs) { + if dc.onMessage != nil { + dc.onMessage(ctx, args) + } + })) + ndc.OnMessage(func(msg webrtc.DataChannelMessage) { + p.Process(context.Background(), ProcessArgs{ + Peer: peer, + Message: msg, + DataChannel: ndc, + }) + }) + + s.channels[dc.Label] = ndc + + return nil +} + +// DataChannel returns the channel for a label +func (s *Subscriber) DataChannel(label string) *webrtc.DataChannel { + s.RLock() + defer s.RUnlock() + return s.channels[label] +} + +func (s *Subscriber) OnNegotiationNeeded(f func()) { + debounced := debounce.New(250 * time.Millisecond) + s.negotiate = func() { + debounced(f) + } +} + +func (s *Subscriber) CreateOffer() (webrtc.SessionDescription, error) { + offer, err := s.pc.CreateOffer(nil) + if err != nil { + return webrtc.SessionDescription{}, err + } + + err = s.pc.SetLocalDescription(offer) + if err != nil { + return webrtc.SessionDescription{}, err + } + + return offer, nil +} + +// OnICECandidate handler +func (s *Subscriber) OnICECandidate(f func(c *webrtc.ICECandidate)) { + s.pc.OnICECandidate(f) +} + +// AddICECandidate to peer connection +func (s *Subscriber) AddICECandidate(candidate webrtc.ICECandidateInit) error { + if s.pc.RemoteDescription() != nil { + return s.pc.AddICECandidate(candidate) + } + s.candidates = append(s.candidates, candidate) + return nil +} + +func (s *Subscriber) AddDownTrack(streamID string, downTrack *DownTrack) { + s.Lock() + defer s.Unlock() + if dt, ok := s.tracks[streamID]; ok { + dt = append(dt, downTrack) + s.tracks[streamID] = dt + } else { + s.tracks[streamID] = []*DownTrack{downTrack} + } +} + +func (s *Subscriber) RemoveDownTrack(streamID string, downTrack *DownTrack) { + s.Lock() + defer s.Unlock() + if dts, ok := s.tracks[streamID]; ok { + idx := -1 + for i, dt := range dts { + if dt == downTrack { + idx = i + break + } + } + if idx >= 0 { + dts[idx] = dts[len(dts)-1] + dts[len(dts)-1] = nil + dts = dts[:len(dts)-1] + s.tracks[streamID] = dts + } + } +} + +func (s *Subscriber) AddDataChannel(label string) (*webrtc.DataChannel, error) { + s.Lock() + defer s.Unlock() + + if s.channels[label] != nil { + return s.channels[label], nil + } + + dc, err := s.pc.CreateDataChannel(label, &webrtc.DataChannelInit{}) + if err != nil { + Logger.Error(err, "dc creation error") + return nil, errCreatingDataChannel + } + + s.channels[label] = dc + + return dc, nil +} + +// SetRemoteDescription sets the SessionDescription of the remote peer +func (s *Subscriber) SetRemoteDescription(desc webrtc.SessionDescription) error { + if err := s.pc.SetRemoteDescription(desc); err != nil { + Logger.Error(err, "SetRemoteDescription error") + return err + } + + for _, c := range s.candidates { + if err := s.pc.AddICECandidate(c); err != nil { + Logger.Error(err, "Add subscriber ice candidate to peer err", "peer_id", s.id) + } + } + s.candidates = nil + + return nil +} + +func (s *Subscriber) RegisterDatachannel(label string, dc *webrtc.DataChannel) { + s.Lock() + s.channels[label] = dc + s.Unlock() +} + +func (s *Subscriber) GetDatachannel(label string) *webrtc.DataChannel { + return s.DataChannel(label) +} + +func (s *Subscriber) DownTracks() []*DownTrack { + s.RLock() + defer s.RUnlock() + var downTracks []*DownTrack + for _, tracks := range s.tracks { + downTracks = append(downTracks, tracks...) + } + return downTracks +} + +func (s *Subscriber) GetDownTracks(streamID string) []*DownTrack { + s.RLock() + defer s.RUnlock() + return s.tracks[streamID] +} + +// Negotiate fires a debounced negotiation request +func (s *Subscriber) Negotiate() { + s.negotiate() +} + +// Close peer +func (s *Subscriber) Close() error { + return s.pc.Close() +} + +func (s *Subscriber) downTracksReports() { + for { + time.Sleep(5 * time.Second) + + if s.pc.ConnectionState() == webrtc.PeerConnectionStateClosed { + return + } + + var r []rtcp.Packet + var sd []rtcp.SourceDescriptionChunk + s.RLock() + for _, dts := range s.tracks { + for _, dt := range dts { + if !dt.bound.get() { + continue + } + if sr := dt.CreateSenderReport(); sr != nil { + r = append(r, sr) + } + sd = append(sd, dt.CreateSourceDescriptionChunks()...) + } + } + s.RUnlock() + i := 0 + j := 0 + for i < len(sd) { + i = (j + 1) * 15 + if i >= len(sd) { + i = len(sd) + } + nsd := sd[j*15 : i] + r = append(r, &rtcp.SourceDescription{Chunks: nsd}) + j++ + if err := s.pc.WriteRTCP(r); err != nil { + if err == io.EOF || err == io.ErrClosedPipe { + return + } + Logger.Error(err, "Sending downtrack reports err") + } + r = r[:0] + } + } +} + +func (s *Subscriber) sendStreamDownTracksReports(streamID string) { + var r []rtcp.Packet + var sd []rtcp.SourceDescriptionChunk + + s.RLock() + dts := s.tracks[streamID] + for _, dt := range dts { + if !dt.bound.get() { + continue + } + sd = append(sd, dt.CreateSourceDescriptionChunks()...) + } + s.RUnlock() + if len(sd) == 0 { + return + } + r = append(r, &rtcp.SourceDescription{Chunks: sd}) + go func() { + r := r + i := 0 + for { + if err := s.pc.WriteRTCP(r); err != nil { + Logger.Error(err, "Sending track binding reports err") + } + if i > 5 { + return + } + i++ + time.Sleep(20 * time.Millisecond) + } + }() +} diff --git a/pkg/sfu/turn.go b/pkg/sfu/turn.go new file mode 100644 index 000000000..17c47b668 --- /dev/null +++ b/pkg/sfu/turn.go @@ -0,0 +1,170 @@ +package sfu + +import ( + "context" + "crypto/rand" + "crypto/tls" + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/pion/dtls/v2" + "github.com/pion/logging" + "github.com/pion/turn/v2" +) + +const ( + turnMinPort = 32768 + turnMaxPort = 46883 + sfuMinPort = 46884 + sfuMaxPort = 60999 +) + +type TurnAuth struct { + Credentials string `mapstructure:"credentials"` + Secret string `mapstructure:"secret"` +} + +// WebRTCConfig defines parameters for ice +type TurnConfig struct { + Enabled bool `mapstructure:"enabled"` + Realm string `mapstructure:"realm"` + Address string `mapstructure:"address"` + Cert string `mapstructure:"cert"` + Key string `mapstructure:"key"` + Auth TurnAuth `mapstructure:"auth"` + PortRange []uint16 `mapstructure:"portrange"` +} + +func InitTurnServer(conf TurnConfig, auth func(username, realm string, srcAddr net.Addr) ([]byte, bool)) (*turn.Server, error) { + var listeners []turn.ListenerConfig + + // Create a UDP listener to pass into pion/turn + // pion/turn itself doesn't allocate any UDP sockets, but lets the user pass them in + // this allows us to add logging, storage or modify inbound/outbound traffic + udpListener, err := net.ListenPacket("udp4", conf.Address) + if err != nil { + return nil, err + } + // Create a TCP listener to pass into pion/turn + // pion/turn itself doesn't allocate any TCP listeners, but lets the user pass them in + // this allows us to add logging, storage or modify inbound/outbound traffic + tcpListener, err := net.Listen("tcp4", conf.Address) + if err != nil { + return nil, err + } + + addr := strings.Split(conf.Address, ":") + + var minPort uint16 = turnMinPort + var maxPort uint16 = turnMaxPort + + if len(conf.PortRange) == 2 { + minPort = conf.PortRange[0] + maxPort = conf.PortRange[1] + } + + if len(conf.Cert) > 0 && len(conf.Key) > 0 { + // Create a TLS listener to pass into pion/turn + cert, err := tls.LoadX509KeyPair(conf.Cert, conf.Key) + if err != nil { + return nil, err + } + config := tls.Config{Certificates: []tls.Certificate{cert}} + config.Rand = rand.Reader + tlsListener := tls.NewListener(tcpListener, &config) + listeners = append(listeners, turn.ListenerConfig{ + Listener: tlsListener, + RelayAddressGenerator: &turn.RelayAddressGeneratorPortRange{ + RelayAddress: net.ParseIP(addr[0]), + Address: "0.0.0.0", + MinPort: minPort, + MaxPort: maxPort, + }, + }) + // Create a DTLS listener to pass into pion/turn + ctx := context.Background() + dtlsConf := &dtls.Config{ + Certificates: []tls.Certificate{cert}, + ExtendedMasterSecret: dtls.RequireExtendedMasterSecret, + // Create timeout context for accepted connection. + ConnectContextMaker: func() (context.Context, func()) { + return context.WithTimeout(ctx, 30*time.Second) + }, + } + port, err := strconv.ParseInt(addr[1], 10, 64) + if err != nil { + return nil, err + } + a := &net.UDPAddr{IP: net.ParseIP(addr[0]), Port: int(port)} + dtlsListener, err := dtls.Listen("udp4", a, dtlsConf) + if err != nil { + return nil, err + } + listeners = append(listeners, turn.ListenerConfig{ + Listener: dtlsListener, + RelayAddressGenerator: &turn.RelayAddressGeneratorPortRange{ + RelayAddress: net.ParseIP(addr[0]), + Address: "0.0.0.0", + MinPort: minPort, + MaxPort: maxPort, + }, + }) + } + + if auth == nil { + if conf.Auth.Secret != "" { + logger := logging.NewDefaultLeveledLoggerForScope("lt-creds", logging.LogLevelTrace, os.Stdout) + auth = turn.NewLongTermAuthHandler(conf.Auth.Secret, logger) + } else { + usersMap := map[string][]byte{} + for _, kv := range regexp.MustCompile(`(\w+)=(\w+)`).FindAllStringSubmatch(conf.Auth.Credentials, -1) { + usersMap[kv[1]] = turn.GenerateAuthKey(kv[1], conf.Realm, kv[2]) + } + if len(usersMap) == 0 { + Logger.Error(fmt.Errorf("No turn auth provided"), "Got err") + } + auth = func(username string, realm string, srcAddr net.Addr) ([]byte, bool) { + if key, ok := usersMap[username]; ok { + return key, true + } + return nil, false + } + } + } + + return turn.NewServer(turn.ServerConfig{ + Realm: conf.Realm, + // Set AuthHandler callback + // This is called everytime a user tries to authenticate with the TURN server + // Return the key for that user, or false when no user is found + AuthHandler: auth, + // ListenerConfig is a list of Listeners and the Configuration around them + ListenerConfigs: append(listeners, turn.ListenerConfig{ + Listener: tcpListener, + RelayAddressGenerator: &turn.RelayAddressGeneratorPortRange{ + RelayAddress: net.ParseIP(addr[0]), + Address: "0.0.0.0", + MinPort: minPort, + MaxPort: maxPort, + }, + }, + ), + // PacketConnConfigs is a list of UDP Listeners and the Configuration around them + PacketConnConfigs: []turn.PacketConnConfig{ + { + PacketConn: udpListener, + RelayAddressGenerator: &turn.RelayAddressGeneratorPortRange{ + RelayAddress: net.ParseIP(addr[0]), + Address: "0.0.0.0", + MinPort: minPort, + MaxPort: maxPort, + }, + }, + }, + }) +} diff --git a/pkg/sfu/twcc/twcc.go b/pkg/sfu/twcc/twcc.go new file mode 100644 index 000000000..e91537d09 --- /dev/null +++ b/pkg/sfu/twcc/twcc.go @@ -0,0 +1,313 @@ +package twcc + +import ( + "encoding/binary" + "math" + "math/rand" + "sort" + "sync" + + "github.com/gammazero/deque" + "github.com/pion/rtcp" +) + +const ( + baseSequenceNumberOffset = 8 + packetStatusCountOffset = 10 + referenceTimeOffset = 12 + + tccReportDelta = 1e8 + tccReportDeltaAfterMark = 50e6 +) + +type rtpExtInfo struct { + ExtTSN uint32 + Timestamp int64 +} + +// Responder will get the transport wide sequence number from rtp +// extension header, and reply with the rtcp feedback message +// according to: +// https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 +type Responder struct { + sync.Mutex + + extInfo []rtpExtInfo + lastReport int64 + cycles uint32 + lastExtSN uint32 + pktCtn uint8 + lastSn uint16 + lastExtInfo uint16 + mSSRC uint32 + sSSRC uint32 + + len uint16 + deltaLen uint16 + payload [100]byte + deltas [200]byte + chunk uint16 + + onFeedback func(packet rtcp.RawPacket) +} + +func NewTransportWideCCResponder(ssrc uint32) *Responder { + return &Responder{ + extInfo: make([]rtpExtInfo, 0, 101), + sSSRC: rand.Uint32(), + mSSRC: ssrc, + } +} + +// Push a sequence number read from rtp packet ext packet +func (t *Responder) Push(sn uint16, timeNS int64, marker bool) { + t.Lock() + defer t.Unlock() + + if sn < 0x0fff && (t.lastSn&0xffff) > 0xf000 { + t.cycles += 1 << 16 + } + t.extInfo = append(t.extInfo, rtpExtInfo{ + ExtTSN: t.cycles | uint32(sn), + Timestamp: timeNS / 1e3, + }) + if t.lastReport == 0 { + t.lastReport = timeNS + } + t.lastSn = sn + delta := timeNS - t.lastReport + if len(t.extInfo) > 20 && t.mSSRC != 0 && + (delta >= tccReportDelta || len(t.extInfo) > 100 || (marker && delta >= tccReportDeltaAfterMark)) { + if pkt := t.buildTransportCCPacket(); pkt != nil { + t.onFeedback(pkt) + } + t.lastReport = timeNS + } +} + +// OnFeedback sets the callback for the formed twcc feedback rtcp packet +func (t *Responder) OnFeedback(f func(p rtcp.RawPacket)) { + t.onFeedback = f +} + +func (t *Responder) buildTransportCCPacket() rtcp.RawPacket { + if len(t.extInfo) == 0 { + return nil + } + sort.Slice(t.extInfo, func(i, j int) bool { + return t.extInfo[i].ExtTSN < t.extInfo[j].ExtTSN + }) + tccPkts := make([]rtpExtInfo, 0, int(float64(len(t.extInfo))*1.2)) + for _, tccExtInfo := range t.extInfo { + if tccExtInfo.ExtTSN < t.lastExtSN { + continue + } + if t.lastExtSN != 0 { + for j := t.lastExtSN + 1; j < tccExtInfo.ExtTSN; j++ { + tccPkts = append(tccPkts, rtpExtInfo{ExtTSN: j}) + } + } + t.lastExtSN = tccExtInfo.ExtTSN + tccPkts = append(tccPkts, tccExtInfo) + } + t.extInfo = t.extInfo[:0] + + firstRecv := false + same := true + timestamp := int64(0) + lastStatus := rtcp.TypeTCCPacketReceivedWithoutDelta + maxStatus := rtcp.TypeTCCPacketNotReceived + + var statusList deque.Deque + statusList.SetMinCapacity(3) + + for _, stat := range tccPkts { + status := rtcp.TypeTCCPacketNotReceived + if stat.Timestamp != 0 { + var delta int64 + if !firstRecv { + firstRecv = true + refTime := stat.Timestamp / 64e3 + timestamp = refTime * 64e3 + t.writeHeader( + uint16(tccPkts[0].ExtTSN), + uint16(len(tccPkts)), + uint32(refTime), + ) + t.pktCtn++ + } + + delta = (stat.Timestamp - timestamp) / 250 + if delta < 0 || delta > 255 { + status = rtcp.TypeTCCPacketReceivedLargeDelta + rDelta := int16(delta) + if int64(rDelta) != delta { + if rDelta > 0 { + rDelta = math.MaxInt16 + } else { + rDelta = math.MinInt16 + } + } + t.writeDelta(status, uint16(rDelta)) + } else { + status = rtcp.TypeTCCPacketReceivedSmallDelta + t.writeDelta(status, uint16(delta)) + } + timestamp = stat.Timestamp + } + + if same && status != lastStatus && lastStatus != rtcp.TypeTCCPacketReceivedWithoutDelta { + if statusList.Len() > 7 { + t.writeRunLengthChunk(lastStatus, uint16(statusList.Len())) + statusList.Clear() + lastStatus = rtcp.TypeTCCPacketReceivedWithoutDelta + maxStatus = rtcp.TypeTCCPacketNotReceived + same = true + } else { + same = false + } + } + statusList.PushBack(status) + if status > maxStatus { + maxStatus = status + } + lastStatus = status + + if !same && maxStatus == rtcp.TypeTCCPacketReceivedLargeDelta && statusList.Len() > 6 { + for i := 0; i < 7; i++ { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit, statusList.PopFront().(uint16), i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit) + lastStatus = rtcp.TypeTCCPacketReceivedWithoutDelta + maxStatus = rtcp.TypeTCCPacketNotReceived + same = true + + for i := 0; i < statusList.Len(); i++ { + status = statusList.At(i).(uint16) + if status > maxStatus { + maxStatus = status + } + if same && lastStatus != rtcp.TypeTCCPacketReceivedWithoutDelta && status != lastStatus { + same = false + } + lastStatus = status + } + } else if !same && statusList.Len() > 13 { + for i := 0; i < 14; i++ { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit, statusList.PopFront().(uint16), i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit) + lastStatus = rtcp.TypeTCCPacketReceivedWithoutDelta + maxStatus = rtcp.TypeTCCPacketNotReceived + same = true + } + } + + if statusList.Len() > 0 { + if same { + t.writeRunLengthChunk(lastStatus, uint16(statusList.Len())) + } else if maxStatus == rtcp.TypeTCCPacketReceivedLargeDelta { + for i := 0; i < statusList.Len(); i++ { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit, statusList.PopFront().(uint16), i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit) + } else { + for i := 0; i < statusList.Len(); i++ { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit, statusList.PopFront().(uint16), i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit) + } + } + + pLen := t.len + t.deltaLen + 4 + pad := pLen%4 != 0 + var padSize uint8 + for pLen%4 != 0 { + padSize++ + pLen++ + } + hdr := rtcp.Header{ + Padding: pad, + Length: (pLen / 4) - 1, + Count: rtcp.FormatTCC, + Type: rtcp.TypeTransportSpecificFeedback, + } + hb, _ := hdr.Marshal() + pkt := make(rtcp.RawPacket, pLen) + copy(pkt, hb) + copy(pkt[4:], t.payload[:t.len]) + copy(pkt[4+t.len:], t.deltas[:t.deltaLen]) + if pad { + pkt[len(pkt)-1] = padSize + } + t.deltaLen = 0 + return pkt +} + +func (t *Responder) writeHeader(bSN, packetCount uint16, refTime uint32) { + /* + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of packet sender | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of media source | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | base sequence number | packet status count | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | reference time | fb pkt. count | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + binary.BigEndian.PutUint32(t.payload[0:], t.sSSRC) + binary.BigEndian.PutUint32(t.payload[4:], t.mSSRC) + binary.BigEndian.PutUint16(t.payload[baseSequenceNumberOffset:], bSN) + binary.BigEndian.PutUint16(t.payload[packetStatusCountOffset:], packetCount) + binary.BigEndian.PutUint32(t.payload[referenceTimeOffset:], refTime<<8|uint32(t.pktCtn)) + t.len = 16 +} + +func (t *Responder) writeRunLengthChunk(symbol uint16, runLength uint16) { + /* + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |T| S | Run Length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + binary.BigEndian.PutUint16(t.payload[t.len:], symbol<<13|runLength) + t.len += 2 +} + +func (t *Responder) createStatusSymbolChunk(symbolSize, symbol uint16, i int) { + /* + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |T|S| symbol list | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + numOfBits := symbolSize + 1 + t.chunk = setNBitsOfUint16(t.chunk, numOfBits, numOfBits*uint16(i)+2, symbol) +} + +func (t *Responder) writeStatusSymbolChunk(symbolSize uint16) { + t.chunk = setNBitsOfUint16(t.chunk, 1, 0, 1) + t.chunk = setNBitsOfUint16(t.chunk, 1, 1, symbolSize) + binary.BigEndian.PutUint16(t.payload[t.len:], t.chunk) + t.chunk = 0 + t.len += 2 +} + +func (t *Responder) writeDelta(deltaType, delta uint16) { + if deltaType == rtcp.TypeTCCPacketReceivedSmallDelta { + t.deltas[t.deltaLen] = byte(delta) + t.deltaLen++ + return + } + binary.BigEndian.PutUint16(t.deltas[t.deltaLen:], delta) + t.deltaLen += 2 +} + +// setNBitsOfUint16 will truncate the value to size, left-shift to startIndex position and set +func setNBitsOfUint16(src, size, startIndex, val uint16) uint16 { + if startIndex+size > 16 { + return 0 + } + // truncate val to size bits + val &= (1 << size) - 1 + return src | (val << (16 - size - startIndex)) +} diff --git a/pkg/sfu/twcc/twcc_test.go b/pkg/sfu/twcc/twcc_test.go new file mode 100644 index 000000000..2a1a47f1c --- /dev/null +++ b/pkg/sfu/twcc/twcc_test.go @@ -0,0 +1,346 @@ +package twcc + +import ( + "math/rand" + "testing" + "time" + + "github.com/pion/rtcp" + "github.com/stretchr/testify/assert" +) + +func TestTransportWideCC_writeRunLengthChunk(t1 *testing.T) { + type fields struct { + len uint16 + } + type args struct { + symbol uint16 + runLength uint16 + } + tests := []struct { + name string + fields fields + args args + wantErr bool + wantBytes []byte + }{ + { + name: "Must not return error", + + args: args{ + symbol: rtcp.TypeTCCPacketNotReceived, + runLength: 221, + }, + wantErr: false, + wantBytes: []byte{0, 0xdd}, + }, { + name: "Must set run length after padding", + fields: fields{ + len: 1, + }, + args: args{ + symbol: rtcp.TypeTCCPacketReceivedWithoutDelta, + runLength: 24, + }, + wantBytes: []byte{0, 0x60, 0x18}, + }, + } + for _, tt := range tests { + tt := tt + t1.Run(tt.name, func(t1 *testing.T) { + t := &Responder{ + len: tt.fields.len, + } + t.writeRunLengthChunk(tt.args.symbol, tt.args.runLength) + assert.Equal(t1, tt.wantBytes, t.payload[:t.len]) + }) + } +} + +func TestTransportWideCC_writeStatusSymbolChunk(t1 *testing.T) { + type fields struct { + len uint16 + } + type args struct { + symbolSize uint16 + symbolList []uint16 + } + tests := []struct { + name string + fields fields + args args + wantBytes []byte + }{ + { + name: "Must not return error", + args: args{ + symbolSize: rtcp.TypeTCCSymbolSizeOneBit, + symbolList: []uint16{rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived}, + }, + wantBytes: []byte{0x9F, 0x1C}, + }, + { + name: "Must set symbol chunk after padding", + fields: fields{ + len: 1, + }, + args: args{ + symbolSize: rtcp.TypeTCCSymbolSizeTwoBit, + symbolList: []uint16{ + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedWithoutDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived}, + }, + wantBytes: []byte{0x0, 0xcd, 0x50}, + }, + } + for _, tt := range tests { + tt := tt + t1.Run(tt.name, func(t1 *testing.T) { + t := &Responder{ + len: tt.fields.len, + } + for i, v := range tt.args.symbolList { + t.createStatusSymbolChunk(tt.args.symbolSize, v, i) + } + t.writeStatusSymbolChunk(tt.args.symbolSize) + assert.Equal(t1, tt.wantBytes, t.payload[:t.len]) + }) + } +} + +func TestTransportWideCC_writeDelta(t1 *testing.T) { + a := -32768 + type fields struct { + deltaLen uint16 + } + type args struct { + deltaType uint16 + delta uint16 + } + tests := []struct { + name string + fields fields + args args + want []byte + }{ + { + name: "Must set correct small delta", + args: args{ + deltaType: rtcp.TypeTCCPacketReceivedSmallDelta, + delta: 255, + }, + want: []byte{0xff}, + }, + { + name: "Must set correct small delta with padding", + fields: fields{ + deltaLen: 1, + }, + args: args{ + deltaType: rtcp.TypeTCCPacketReceivedSmallDelta, + delta: 255, + }, + want: []byte{0, 0xff}, + }, + { + name: "Must set correct large delta", + args: args{ + deltaType: rtcp.TypeTCCPacketReceivedLargeDelta, + delta: 32767, + }, + want: []byte{0x7F, 0xFF}, + }, + { + name: "Must set correct large delta with padding", + fields: fields{ + deltaLen: 1, + }, + args: args{ + deltaType: rtcp.TypeTCCPacketReceivedLargeDelta, + delta: uint16(a), + }, + want: []byte{0, 0x80, 0x00}, + }, + } + for _, tt := range tests { + tt := tt + t1.Run(tt.name, func(t1 *testing.T) { + t := &Responder{ + deltaLen: tt.fields.deltaLen, + } + t.writeDelta(tt.args.deltaType, tt.args.delta) + assert.Equal(t1, tt.want, t.deltas[:t.deltaLen]) + assert.Equal(t1, tt.fields.deltaLen+tt.args.deltaType, t.deltaLen) + }) + } +} + +func TestTransportWideCC_writeHeader(t1 *testing.T) { + type fields struct { + tccPktCtn uint8 + sSSRC uint32 + mSSRC uint32 + } + type args struct { + bSN uint16 + packetCount uint16 + refTime uint32 + } + tests := []struct { + name string + fields fields + args args + want []byte + }{ + { + name: "Must construct correct header", + fields: fields{ + tccPktCtn: 23, + sSSRC: 4195875351, + mSSRC: 1124282272, + }, + args: args{ + bSN: 153, + packetCount: 1, + refTime: 4057090, + }, + want: []byte{ + 0xfa, 0x17, 0xfa, 0x17, + 0x43, 0x3, 0x2f, 0xa0, + 0x0, 0x99, 0x0, 0x1, + 0x3d, 0xe8, 0x2, 0x17}, + }, + } + for _, tt := range tests { + tt := tt + t1.Run(tt.name, func(t1 *testing.T) { + t := &Responder{ + pktCtn: tt.fields.tccPktCtn, + sSSRC: tt.fields.sSSRC, + mSSRC: tt.fields.mSSRC, + } + t.writeHeader(tt.args.bSN, tt.args.packetCount, tt.args.refTime) + assert.Equal(t1, tt.want, t.payload[0:16]) + }) + } +} + +func TestTccPacket(t1 *testing.T) { + want := []byte{ + 0xfa, 0x17, 0xfa, 0x17, + 0x43, 0x3, 0x2f, 0xa0, + 0x0, 0x99, 0x0, 0x1, + 0x3d, 0xe8, 0x2, 0x17, + 0x60, 0x18, 0x0, 0xdd, + 0x9F, 0x1C, 0xcd, 0x50, + } + + delta := []byte{ + 0xff, 0x80, 0xaa, + } + + symbol1 := []uint16{rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived} + symbol2 := []uint16{ + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketReceivedWithoutDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketReceivedSmallDelta, + rtcp.TypeTCCPacketNotReceived, + rtcp.TypeTCCPacketNotReceived} + + t := &Responder{ + pktCtn: 23, + sSSRC: 4195875351, + mSSRC: 1124282272, + } + t.writeHeader(153, 1, 4057090) + t.writeRunLengthChunk(rtcp.TypeTCCPacketReceivedWithoutDelta, 24) + t.writeRunLengthChunk(rtcp.TypeTCCPacketNotReceived, 221) + for i, v := range symbol1 { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit, v, i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeOneBit) + for i, v := range symbol2 { + t.createStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit, v, i) + } + t.writeStatusSymbolChunk(rtcp.TypeTCCSymbolSizeTwoBit) + t.deltaLen = uint16(len(delta)) + assert.Equal(t1, want, t.payload[:24]) + + pLen := t.len + t.deltaLen + 4 + pad := pLen%4 != 0 + for pLen%4 != 0 { + pLen++ + } + hdr := rtcp.Header{ + Padding: pad, + Length: (pLen / 4) - 1, + Count: rtcp.FormatTCC, + Type: rtcp.TypeTransportSpecificFeedback, + } + assert.Equal(t1, int(pLen), len(want)+3+4+1) + hb, _ := hdr.Marshal() + pkt := make([]byte, pLen) + copy(pkt, hb) + assert.Equal(t1, hb, pkt[:len(hb)]) + copy(pkt[4:], t.payload[:t.len]) + assert.Equal(t1, append(hb, t.payload[:t.len]...), pkt[:len(hb)+int(t.len)]) + copy(pkt[4+t.len:], delta[:t.deltaLen]) + assert.Equal(t1, delta, pkt[len(hb)+int(t.len):len(pkt)-1]) + var ss rtcp.TransportLayerCC + err := ss.Unmarshal(pkt) + assert.NoError(t1, err) + + assert.Equal(t1, hdr, ss.Header) + +} + +func BenchmarkBuildPacket(b *testing.B) { + rand.Seed(time.Now().UnixNano()) + b.ReportAllocs() + n := 1 + rand.Intn(4-1+1) + var twcc Responder + tm := time.Now() + for i := 1; i < 100; i++ { + tm := tm.Add(time.Duration(60*n) * time.Millisecond) + twcc.extInfo = append(twcc.extInfo, rtpExtInfo{ + ExtTSN: uint32(i), + Timestamp: tm.UnixNano(), + }) + } + for i := 0; i < b.N; i++ { + _ = twcc.buildTransportCCPacket() + } +}