From dd024571a1e4a9002e681bcb71855e8ca79986fc Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Thu, 30 Nov 2023 19:46:24 +0200 Subject: [PATCH 001/114] SIP: Move dispatch rule evaluation to protocol package. (#2279) --- go.mod | 2 +- go.sum | 4 +- pkg/service/ioservice_sip.go | 296 ++--------------------- pkg/service/ioservice_sip_test.go | 390 ------------------------------ 4 files changed, 21 insertions(+), 671 deletions(-) delete mode 100644 pkg/service/ioservice_sip_test.go diff --git a/go.mod b/go.mod index 558fd67d9..8a05272f4 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb - github.com/livekit/protocol v1.9.3-0.20231129173544-1c3f5fe919b0 + github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3 github.com/livekit/psrpc v0.5.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index a1b81d76c..5193e6f52 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb h1:KiGg4k+kYQD9NjKixaSDMMeYOO2//XBM4IROTI1Itjo= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.3-0.20231129173544-1c3f5fe919b0 h1:AhJlQejQ+Ma9Q+EPqCNt2S7h6ETJXDiO7qsQdTq9VvM= -github.com/livekit/protocol v1.9.3-0.20231129173544-1c3f5fe919b0/go.mod h1:8f342d5nvfNp9YAEfJokSR+zbNFpaivgU0h6vwaYhes= +github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3 h1:am72beYtXZM71MRr+12lkG3IyqKxzrCa6slsbKrwMe8= +github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3/go.mod h1:8f342d5nvfNp9YAEfJokSR+zbNFpaivgU0h6vwaYhes= github.com/livekit/psrpc v0.5.2 h1:+MvG8Otm/J6MTg2MP/uuMbrkxOWsrj2hDhu/I1VIU1U= github.com/livekit/psrpc v0.5.2/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index 3f0b34042..076bff512 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -1,252 +1,27 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( "context" - "fmt" - "math" - "regexp" - "sort" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/sip" ) -// sipRulePriority returns sorting priority for dispatch rules. Lower value means higher priority. -func sipRulePriority(info *livekit.SIPDispatchRuleInfo) int32 { - // In all these cases, prefer pin-protected rules. - // Thus, the order will be the following: - // - 0: Direct or Pin (both pin-protected) - // - 1: Individual (pin-protected) - // - 100: Direct (open) - // - 101: Individual (open) - const ( - last = math.MaxInt32 - ) - // TODO: Maybe allow setting specific priorities for dispatch rules? - switch rule := info.GetRule().GetRule().(type) { - default: - return last - case *livekit.SIPDispatchRule_DispatchRuleDirect: - if rule.DispatchRuleDirect.GetPin() != "" { - return 0 - } - return 100 - case *livekit.SIPDispatchRule_DispatchRuleIndividual: - if rule.DispatchRuleIndividual.GetPin() != "" { - return 1 - } - return 101 - } -} - -// sipSortRules predictably sorts dispatch rules by priority (first one is highest). -func sipSortRules(rules []*livekit.SIPDispatchRuleInfo) { - sort.Slice(rules, func(i, j int) bool { - p1, p2 := sipRulePriority(rules[i]), sipRulePriority(rules[j]) - if p1 < p2 { - return true - } else if p1 > p2 { - return false - } - // For predictable sorting order. - room1, _, _ := sipGetPinAndRoom(rules[i]) - room2, _, _ := sipGetPinAndRoom(rules[j]) - return room1 < room2 - }) -} - -// sipSelectDispatch takes a list of dispatch rules, and takes the decision which one should be selected. -// It returns an error if there are conflicting rules. Returns nil if no rules match. -func sipSelectDispatch(rules []*livekit.SIPDispatchRuleInfo, req *rpc.EvaluateSIPDispatchRulesRequest) (*livekit.SIPDispatchRuleInfo, error) { - if len(rules) == 0 { - return nil, nil - } - // Sorting will do the selection for us. We already filtered out irrelevant ones in matchSIPDispatchRule. - sipSortRules(rules) - byPin := make(map[string]*livekit.SIPDispatchRuleInfo) - var ( - pinRule *livekit.SIPDispatchRuleInfo - openRule *livekit.SIPDispatchRuleInfo - ) - openCnt := 0 - for _, r := range rules { - _, pin, err := sipGetPinAndRoom(r) - if err != nil { - return nil, err - } - if pin == "" { - openRule = r // last one - openCnt++ - } else if r2 := byPin[pin]; r2 != nil { - return nil, fmt.Errorf("Conflicting SIP Dispatch Rules: Same PIN for %q and %q", - r.SipDispatchRuleId, r2.SipDispatchRuleId) - } else { - byPin[pin] = r - // Pick the first one with a Pin. If Pin was provided in the request, we already filtered the right rules. - // If not, this rule will just be used to send RequestPin=true flag. - if pinRule == nil { - pinRule = r - } - } - } - if req.GetPin() != "" { - // If it's still nil that's fine. We will report "no rules matched" later. - return pinRule, nil - } - if pinRule != nil { - return pinRule, nil - } - if openCnt > 1 { - return nil, fmt.Errorf("Conflicting SIP Dispatch Rules: Matched %d open rules for %q", openCnt, req.CallingNumber) - } - return openRule, nil -} - -// sipGetPinAndRoom returns a room name/prefix and the pin for a dispatch rule. Just a convenience wrapper. -func sipGetPinAndRoom(info *livekit.SIPDispatchRuleInfo) (room, pin string, err error) { - // TODO: Could probably add methods on SIPDispatchRuleInfo struct instead. - switch rule := info.GetRule().GetRule().(type) { - default: - return "", "", fmt.Errorf("Unsupported SIP Dispatch Rule: %T", rule) - case *livekit.SIPDispatchRule_DispatchRuleDirect: - pin = rule.DispatchRuleDirect.GetPin() - room = rule.DispatchRuleDirect.GetRoomName() - case *livekit.SIPDispatchRule_DispatchRuleIndividual: - pin = rule.DispatchRuleIndividual.GetPin() - room = rule.DispatchRuleIndividual.GetRoomPrefix() - } - return room, pin, nil -} - -// sipMatchTrunk finds a SIP Trunk definition matching the request. -// Returns nil if no rules matched or an error if there are conflicting definitions. -func sipMatchTrunk(trunks []*livekit.SIPTrunkInfo, calling, called string) (*livekit.SIPTrunkInfo, error) { - var ( - selectedTrunk *livekit.SIPTrunkInfo - defaultTrunk *livekit.SIPTrunkInfo - defaultTrunkCnt int // to error in case there are multiple ones - ) - for _, tr := range trunks { - // Do not consider it if regexp doesn't match. - matches := len(tr.InboundNumbersRegex) == 0 - for _, reStr := range tr.InboundNumbersRegex { - // TODO: we should cache it - re, err := regexp.Compile(reStr) - if err != nil { - logger.Errorw("cannot parse SIP trunk regexp", err, "trunkID", tr.SipTrunkId) - continue - } - if re.MatchString(calling) { - matches = true - break - } - } - if !matches { - continue - } - if tr.OutboundNumber == "" { - // Default/wildcard trunk. - defaultTrunk = tr - defaultTrunkCnt++ - } else if tr.OutboundNumber == called { - // Trunk specific to the number. - if selectedTrunk != nil { - return nil, fmt.Errorf("Multiple SIP Trunks matched for %q", called) - } - selectedTrunk = tr - // Keep searching! We want to know if there are any conflicting Trunk definitions. - } - } - if selectedTrunk != nil { - return selectedTrunk, nil - } - if defaultTrunkCnt > 1 { - return nil, fmt.Errorf("Multiple default SIP Trunks matched for %q", called) - } - // Could still be nil here. - return defaultTrunk, nil -} - -// sipMatchDispatchRule finds the best dispatch rule matching the request parameters. Returns an error if no rule matched. -// Trunk parameter can be nil, in which case only wildcard dispatch rules will be effective (ones without Trunk IDs). -func sipMatchDispatchRule(trunk *livekit.SIPTrunkInfo, rules []*livekit.SIPDispatchRuleInfo, req *rpc.EvaluateSIPDispatchRulesRequest) (*livekit.SIPDispatchRuleInfo, error) { - // Trunk can still be nil here in case none matched or were defined. - // This is still fine, but only in case we'll match exactly one wildcard dispatch rule. - if len(rules) == 0 { - return nil, fmt.Errorf("No SIP Dispatch Rules defined") - } - // We split the matched dispatch rules into two sets: specific and default (aka wildcard). - // First, attempt to match any of the specific rules, where we did match the Trunk ID. - // If nothing matches there - fallback to default/wildcard rules, where no Trunk IDs were mentioned. - var ( - specificRules []*livekit.SIPDispatchRuleInfo - defaultRules []*livekit.SIPDispatchRuleInfo - ) - noPin := req.NoPin - sentPin := req.GetPin() - for _, info := range rules { - _, rulePin, err := sipGetPinAndRoom(info) - if err != nil { - logger.Errorw("Invalid SIP Dispatch Rule", err, "dispatchRuleID", info.SipDispatchRuleId) - continue - } - // Filter heavily on the Pin, so that only relevant rules remain. - if noPin { - if rulePin != "" { - // Skip pin-protected rules if no pin mode requested. - continue - } - } else if sentPin != "" { - if rulePin == "" { - // Pin already sent, skip non-pin-protected rules. - continue - } - if sentPin != rulePin { - // Pin doesn't match. Don't return an error here, just wait for other rule to match (or none at all). - // Note that we will NOT match non-pin-protected rules, thus it will not fallback to open rules. - continue - } - } - if len(info.TrunkIds) == 0 { - // Default/wildcard dispatch rule. - defaultRules = append(defaultRules, info) - continue - } - // Specific dispatch rules. Require a Trunk associated with the number. - if trunk == nil { - continue - } - matches := false - for _, id := range info.TrunkIds { - if id == trunk.SipTrunkId { - matches = true - break - } - } - if !matches { - continue - } - specificRules = append(specificRules, info) - } - best, err := sipSelectDispatch(specificRules, req) - if err != nil { - return nil, err - } else if best != nil { - return best, nil - } - best, err = sipSelectDispatch(defaultRules, req) - if err != nil { - return nil, err - } else if best != nil { - return best, nil - } - if trunk == nil { - return nil, fmt.Errorf("No SIP Trunk or Dispatch Rules matched for %q", req.CalledNumber) - } - return nil, fmt.Errorf("No SIP Dispatch Rules matched for %q", req.CalledNumber) -} - // matchSIPTrunk finds a SIP Trunk definition matching the request. // Returns nil if no rules matched or an error if there are conflicting definitions. func (s *IOInfoService) matchSIPTrunk(ctx context.Context, calling, called string) (*livekit.SIPTrunkInfo, error) { @@ -254,7 +29,7 @@ func (s *IOInfoService) matchSIPTrunk(ctx context.Context, calling, called strin if err != nil { return nil, err } - return sipMatchTrunk(trunks, calling, called) + return sip.MatchTrunk(trunks, calling, called) } // matchSIPDispatchRule finds the best dispatch rule matching the request parameters. Returns an error if no rule matched. @@ -266,7 +41,7 @@ func (s *IOInfoService) matchSIPDispatchRule(ctx context.Context, trunk *livekit if err != nil { return nil, err } - return sipMatchDispatchRule(trunk, rules, req) + return sip.MatchDispatchRule(trunk, rules, req) } func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.EvaluateSIPDispatchRulesRequest) (*rpc.EvaluateSIPDispatchRulesResponse, error) { @@ -278,42 +53,7 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E if err != nil { return nil, err } - sentPin := req.GetPin() - - from := req.CallingNumber - if best.HidePhoneNumber { - // TODO: Decide on the phone masking format. - // Maybe keep regional code, but mask all but 4 last digits? - from = from[len(from)-4:] - } - fromName := "Phone " + from - - room, rulePin, err := sipGetPinAndRoom(best) - if err != nil { - return nil, err - } - if rulePin != "" { - if sentPin == "" { - return &rpc.EvaluateSIPDispatchRulesResponse{ - RequestPin: true, - }, nil - } - if rulePin != sentPin { - // This should never happen in practice, because matchSIPDispatchRule should remove rules with the wrong pin. - return nil, fmt.Errorf("Incorrect PIN for SIP room") - } - } else { - // Pin was sent, but room doesn't require one. Assume user accidentally pressed phone button. - } - switch rule := best.GetRule().GetRule().(type) { - case *livekit.SIPDispatchRule_DispatchRuleIndividual: - // TODO: Decide on the suffix. Do we need to escape specific characters? - room = rule.DispatchRuleIndividual.GetRoomPrefix() + from - } - return &rpc.EvaluateSIPDispatchRulesResponse{ - RoomName: room, - ParticipantIdentity: fromName, - }, nil + return sip.EvaluateDispatchRule(best, req) } func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc.GetSIPTrunkAuthenticationRequest) (*rpc.GetSIPTrunkAuthenticationResponse, error) { diff --git a/pkg/service/ioservice_sip_test.go b/pkg/service/ioservice_sip_test.go deleted file mode 100644 index b8b57fd07..000000000 --- a/pkg/service/ioservice_sip_test.go +++ /dev/null @@ -1,390 +0,0 @@ -package service - -import ( - "fmt" - "testing" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/rpc" - "github.com/stretchr/testify/require" -) - -const ( - sipNumber1 = "1111 1111" - sipNumber2 = "2222 2222" - sipNumber3 = "3333 3333" - sipTrunkID1 = "aaa" - sipTrunkID2 = "bbb" -) - -func TestSIPMatchTrunk(t *testing.T) { - cases := []struct { - name string - trunks []*livekit.SIPTrunkInfo - exp int - expErr bool - }{ - { - name: "empty", - trunks: nil, - exp: -1, // no error; nil result - }, - { - name: "one wildcard", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa"}, - }, - exp: 0, - }, - { - name: "matching", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber2}, - }, - exp: 0, - }, - { - name: "matching regexp", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber2, InboundNumbersRegex: []string{`^\d+ \d+$`}}, - }, - exp: 0, - }, - { - name: "not matching", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - }, - exp: -1, - }, - { - name: "not matching regexp", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber2, InboundNumbersRegex: []string{`^\d+$`}}, - }, - exp: -1, - }, - { - name: "one match", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - {SipTrunkId: "bbb", OutboundNumber: sipNumber2}, - }, - exp: 1, - }, - { - name: "many matches", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - {SipTrunkId: "bbb", OutboundNumber: sipNumber2}, - {SipTrunkId: "ccc", OutboundNumber: sipNumber2}, - }, - expErr: true, - }, - { - name: "many matches default", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - {SipTrunkId: "bbb"}, - {SipTrunkId: "ccc", OutboundNumber: sipNumber2}, - {SipTrunkId: "ddd"}, - }, - exp: 2, - }, - { - name: "regexp", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - {SipTrunkId: "bbb", OutboundNumber: sipNumber2}, - {SipTrunkId: "ccc", OutboundNumber: sipNumber2, InboundNumbersRegex: []string{`^\d+$`}}, - }, - exp: 1, - }, - { - name: "multiple defaults", - trunks: []*livekit.SIPTrunkInfo{ - {SipTrunkId: "aaa", OutboundNumber: sipNumber3}, - {SipTrunkId: "bbb"}, - {SipTrunkId: "ccc"}, - }, - expErr: true, - }, - } - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - got, err := sipMatchTrunk(c.trunks, sipNumber1, sipNumber2) - if c.expErr { - require.Error(t, err) - require.Nil(t, got) - t.Log(err) - } else { - var exp *livekit.SIPTrunkInfo - if c.exp >= 0 { - exp = c.trunks[c.exp] - } - require.NoError(t, err) - require.Equal(t, exp, got) - } - }) - } -} - -func newSIPTrunkDispatch() *livekit.SIPTrunkInfo { - return &livekit.SIPTrunkInfo{ - SipTrunkId: sipTrunkID1, - OutboundNumber: sipNumber2, - } -} - -func newSIPReqDispatch(pin string, noPin bool) *rpc.EvaluateSIPDispatchRulesRequest { - return &rpc.EvaluateSIPDispatchRulesRequest{ - CallingNumber: sipNumber1, - CalledNumber: sipNumber2, - Pin: pin, - //NoPin: noPin, // TODO - } -} - -func newDirectDispatch(room, pin string) *livekit.SIPDispatchRule { - return &livekit.SIPDispatchRule{ - Rule: &livekit.SIPDispatchRule_DispatchRuleDirect{ - DispatchRuleDirect: &livekit.SIPDispatchRuleDirect{ - RoomName: room, Pin: pin, - }, - }, - } -} - -func newIndividualDispatch(roomPref, pin string) *livekit.SIPDispatchRule { - return &livekit.SIPDispatchRule{ - Rule: &livekit.SIPDispatchRule_DispatchRuleIndividual{ - DispatchRuleIndividual: &livekit.SIPDispatchRuleIndividual{ - RoomPrefix: roomPref, Pin: pin, - }, - }, - } -} - -func TestSIPMatchDispatchRule(t *testing.T) { - cases := []struct { - name string - trunk *livekit.SIPTrunkInfo - rules []*livekit.SIPDispatchRuleInfo - reqPin string - noPin bool - exp int - expErr bool - }{ - // These cases just validate that no rules produce an error. - { - name: "empty", - trunk: nil, - rules: nil, - expErr: true, - }, - { - name: "only trunk", - trunk: newSIPTrunkDispatch(), - rules: nil, - expErr: true, - }, - // Default rules should work even if no trunk is defined. - { - name: "one rule/no trunk", - trunk: nil, - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip", "")}, - }, - exp: 0, - }, - // Default rule should work with a trunk too. - { - name: "one rule/default trunk", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip", "")}, - }, - exp: 0, - }, - // Rule matching the trunk should be selected. - { - name: "one rule/specific trunk", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{sipTrunkID1, sipTrunkID2}, Rule: newDirectDispatch("sip", "")}, - }, - exp: 0, - }, - // Rule NOT matching the trunk should NOT be selected. - { - name: "one rule/wrong trunk", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{"zzz"}, Rule: newDirectDispatch("sip", "")}, - }, - expErr: true, - }, - // Direct rule with a pin should be selected, even if no pin is provided. - { - name: "direct pin/correct", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip", "123")}, - {TrunkIds: []string{sipTrunkID2}, Rule: newDirectDispatch("sip", "456")}, - }, - reqPin: "123", - exp: 0, - }, - // Direct rule with a pin should reject wrong pin. - { - name: "direct pin/wrong", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip", "123")}, - {TrunkIds: []string{sipTrunkID2}, Rule: newDirectDispatch("sip", "456")}, - }, - reqPin: "zzz", - expErr: true, - }, - // Multiple direct rules with the same pin should result in an error. - { - name: "direct pin/conflict", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip1", "123")}, - {TrunkIds: []string{sipTrunkID1, sipTrunkID2}, Rule: newDirectDispatch("sip2", "123")}, - }, - reqPin: "123", - expErr: true, - }, - // Multiple direct rules with the same pin on different trunks are ok. - { - name: "direct pin/no conflict on different trunk", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip1", "123")}, - {TrunkIds: []string{sipTrunkID2}, Rule: newDirectDispatch("sip2", "123")}, - }, - reqPin: "123", - exp: 0, - }, - // Specific direct rules should take priority over default direct rules. - { - name: "direct pin/default and specific", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip1", "123")}, - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip2", "123")}, - }, - reqPin: "123", - exp: 1, - }, - // Specific direct rules should take priority over default direct rules. No pin. - { - name: "direct/default and specific", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip1", "")}, - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip2", "")}, - }, - exp: 1, - }, - // Specific direct rules should take priority over default direct rules. One with pin, other without. - { - name: "direct/default and specific/mixed 1", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip1", "123")}, - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip2", "")}, - }, - exp: 1, - }, - { - name: "direct/default and specific/mixed 2", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip1", "")}, - {TrunkIds: []string{sipTrunkID1}, Rule: newDirectDispatch("sip2", "123")}, - }, - exp: 1, - }, - // Multiple default direct rules are not allowed. - { - name: "direct/multiple defaults", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newDirectDispatch("sip1", "")}, - {TrunkIds: nil, Rule: newDirectDispatch("sip2", "")}, - }, - expErr: true, - }, - // Cannot use both direct and individual rules with the same pin setup. - { - name: "direct vs individual/private", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newIndividualDispatch("pref_", "123")}, - {TrunkIds: nil, Rule: newDirectDispatch("sip", "123")}, - }, - expErr: true, - }, - { - name: "direct vs individual/open", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newIndividualDispatch("pref_", "")}, - {TrunkIds: nil, Rule: newDirectDispatch("sip", "")}, - }, - expErr: true, - }, - // Direct rules take priority over individual rules. - { - name: "direct vs individual/priority", - trunk: newSIPTrunkDispatch(), - rules: []*livekit.SIPDispatchRuleInfo{ - {TrunkIds: nil, Rule: newIndividualDispatch("pref_", "123")}, - {TrunkIds: nil, Rule: newDirectDispatch("sip", "456")}, - }, - reqPin: "456", - exp: 1, - }, - } - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - pins := []string{c.reqPin} - if !c.expErr && c.reqPin != "" { - // Should match the same rule, even if no pin is set (so that it can be requested). - pins = append(pins, "") - } - for i, r := range c.rules { - if r.SipDispatchRuleId == "" { - r.SipDispatchRuleId = fmt.Sprintf("rule_%d", i) - } - } - for _, pin := range pins { - pin := pin - name := pin - if name == "" { - name = "no pin" - } - t.Run(name, func(t *testing.T) { - got, err := sipMatchDispatchRule(c.trunk, c.rules, newSIPReqDispatch(pin, c.noPin)) - if c.expErr { - require.Error(t, err) - require.Nil(t, got) - t.Log(err) - } else { - var exp *livekit.SIPDispatchRuleInfo - if c.exp >= 0 { - exp = c.rules[c.exp] - } - require.NoError(t, err) - require.Equal(t, exp, got) - } - }) - } - }) - } -} From 7b778c50eb1124ecd163feda86a3031486da4760 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Dec 2023 11:37:14 +0530 Subject: [PATCH 002/114] Group SDES items for one SSRC in the same chunk. (#2280) --- pkg/rtc/participant.go | 6 +++++- pkg/sfu/downtrack.go | 23 ++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 6e45fcef9..cd2fe7f39 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1486,7 +1486,11 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { pkts = append(pkts, sr) sd = append(sd, chunks...) - batchSize = batchSize + 1 + len(chunks) + numItems := 0 + for _, chunk := range chunks { + numItems += len(chunk.Items) + } + batchSize = batchSize + 1 + numItems if batchSize >= sdBatchSize { if len(sd) != 0 { pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index c15e68dde..30bf71d20 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1266,22 +1266,23 @@ func (d *DownTrack) Resync() { } func (d *DownTrack) CreateSourceDescriptionChunks() []rtcp.SourceDescriptionChunk { - if !d.bound.Load() || d.transceiver.Load() == nil { + transceiver := d.transceiver.Load() + if !d.bound.Load() || transceiver == nil { return nil } return []rtcp.SourceDescriptionChunk{ { Source: d.ssrc, - Items: []rtcp.SourceDescriptionItem{{ - Type: rtcp.SDESCNAME, - Text: d.params.StreamID, - }}, - }, { - Source: d.ssrc, - Items: []rtcp.SourceDescriptionItem{{ - Type: rtcp.SDESType(15), - Text: d.transceiver.Load().Mid(), - }}, + Items: []rtcp.SourceDescriptionItem{ + { + Type: rtcp.SDESCNAME, + Text: d.params.StreamID, + }, + { + Type: rtcp.SDESType(15), + Text: transceiver.Mid(), + }, + }, }, } } From 2299a493de0b686b9b7f1b45fa7097984f0da0c9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Dec 2023 12:42:12 +0530 Subject: [PATCH 003/114] Throttle DD parse logs (#2281) --- pkg/sfu/buffer/dependencydescriptorparser.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 5b2744716..da79c301b 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -19,6 +19,7 @@ import ( "sort" "github.com/pion/rtp" + "go.uber.org/atomic" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/utils" @@ -44,6 +45,8 @@ type DependencyDescriptorParser struct { activeDecodeTargetsExtSeq uint64 activeDecodeTargetsMask uint32 frameChecker *FrameIntegrityChecker + + ddNotFoundCount atomic.Uint32 } func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { @@ -73,7 +76,10 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr var videoLayer VideoLayer ddBuf := pkt.GetExtension(r.ddExtID) if ddBuf == nil { - r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber) + ddNotFoundCount := r.ddNotFoundCount.Inc() + if ddNotFoundCount%100 == 0 { + r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount) + } return nil, videoLayer, nil } From dcff75a51684113daa3d58b909613b0c21a0593a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 1 Dec 2023 16:10:57 +0530 Subject: [PATCH 004/114] Record number of data messages in prometheus. (#2282) --- pkg/telemetry/signalanddatastats.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index 5ed99ebb0..5ab5855ab 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -49,6 +49,7 @@ type BytesTrackStats struct { trackID livekit.TrackID pID livekit.ParticipantID send, recv atomic.Uint64 + sendMessages, recvMessages atomic.Uint32 totalSendBytes, totalRecvBytes atomic.Uint64 totalSendMessages, totalRecvMessages atomic.Uint32 telemetry TelemetryService @@ -68,10 +69,12 @@ func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, tele func (s *BytesTrackStats) AddBytes(bytes uint64, isSend bool) { if isSend { s.send.Add(bytes) + s.sendMessages.Inc() s.totalSendBytes.Add(bytes) s.totalSendMessages.Inc() } else { s.recv.Add(bytes) + s.recvMessages.Inc() s.totalRecvBytes.Add(bytes) s.totalRecvMessages.Inc() } @@ -95,7 +98,10 @@ func (s *BytesTrackStats) report() { if recv := s.recv.Swap(0); recv > 0 { s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ - {PrimaryBytes: recv}, + { + PrimaryBytes: recv, + PrimaryPackets: s.recvMessages.Swap(0), + }, }, }) } @@ -103,7 +109,10 @@ func (s *BytesTrackStats) report() { if send := s.send.Swap(0); send > 0 { s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ - {PrimaryBytes: send}, + { + PrimaryBytes: send, + PrimaryPackets: s.sendMessages.Swap(0), + }, }, }) } From d866b5110fd0da42257abdd3d5d6981ba8dc848f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 2 Dec 2023 12:44:37 +0530 Subject: [PATCH 005/114] Restrict scope of negotiation time out error logs (#2283) * Restrict scope of negotiation time out error logs 1. Log "negotiation failed" only if signal channel was active within half window of negotiation timeout. Negotiation timeout currently is at 15 seconds. Signal pings are every 10 seconds. 2. In transport.go, do not report negotiation timed out and do not callback negotiation failure if the peer connection state is not connected. Goal of negotiation failure tracker is to take remedial action when an in-session negotiation fails. Seeing a bunch of cases of the case hitting even without ICE connection forming. Negotiation timer is not intended for those cases. * fix test --- pkg/rtc/participant.go | 2 +- pkg/rtc/transport.go | 14 ++++-- pkg/rtc/transport_test.go | 53 +++++++++++++++++----- pkg/sfu/streamallocator/streamallocator.go | 2 +- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index cd2fe7f39..46acee7e0 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2308,7 +2308,7 @@ func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal boo } func (p *ParticipantImpl) onAnyTransportNegotiationFailed() { - if p.TransportManager.SinceLastSignal() < negotiationFailedTimeout { + if p.TransportManager.SinceLastSignal() < negotiationFailedTimeout/2 { p.params.Logger.Infow("negotiation failed, starting full reconnect") } p.IssueFullReconnect(types.ParticipantCloseReasonNegotiateFailed) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 16e454e78..b72b1411e 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -390,7 +390,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params: params, debouncedNegotiate: debounce.New(negotiationFrequency), negotiationState: NegotiationStateNone, - eventCh: make(chan event, 50), + eventCh: make(chan event, 100), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, allowedLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), @@ -998,6 +998,12 @@ func (t *PCTransport) OnInitialConnected(f func()) { t.lock.Lock() t.onInitialConnected = f t.lock.Unlock() + + if f != nil { + if t.pc.ConnectionState() == webrtc.PeerConnectionStateConnected { + go f() + } + } } func (t *PCTransport) getOnInitialConnected() func() { @@ -1682,7 +1688,7 @@ func (t *PCTransport) setupSignalStateCheckTimer() { failed := t.negotiationState != NegotiationStateNone - if t.negotiateCounter.Load() == negotiateVersion && failed { + if t.negotiateCounter.Load() == negotiateVersion && failed && t.pc.ConnectionState() == webrtc.PeerConnectionStateConnected { t.params.Logger.Infow( "negotiation timed out", "localCurrent", t.pc.CurrentLocalDescription(), @@ -1944,6 +1950,8 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e } func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) error { + t.clearSignalStateCheckTimer() + if err := t.setRemoteDescription(*sd); err != nil { // Pion will call RTPSender.Send method for each new added Downtrack, and return error if the DownTrack.Bind // returns error. In case of Downtrack.Bind returns ErrUnsupportedCodec, the signal state will be stable as negotiation is aleady compelted @@ -1954,8 +1962,6 @@ func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) } } - t.clearSignalStateCheckTimer() - if t.negotiationState == NegotiationStateRetry { t.setNegotiationState(NegotiationStateNone) diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index eb59df779..e66670ea5 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -17,6 +17,7 @@ package rtc import ( "fmt" "strings" + "sync" "testing" "time" @@ -38,7 +39,7 @@ func TestMissingAnswerDuringICERestart(t *testing.T) { } transportA, err := NewPCTransport(params) require.NoError(t, err) - _, err = transportA.pc.CreateDataChannel("test", nil) + _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) paramsB := params @@ -88,7 +89,7 @@ func TestNegotiationTiming(t *testing.T) { } transportA, err := NewPCTransport(params) require.NoError(t, err) - _, err = transportA.pc.CreateDataChannel("test", nil) + _, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil) require.NoError(t, err) paramsB := params @@ -181,7 +182,7 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { } transportA, err := NewPCTransport(params) require.NoError(t, err) - _, err = transportA.pc.CreateDataChannel("test", nil) + _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) paramsB := params @@ -249,7 +250,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { } transportA, err := NewPCTransport(params) require.NoError(t, err) - _, err = transportA.pc.CreateDataChannel("test", nil) + _, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil) require.NoError(t, err) paramsB := params @@ -322,15 +323,21 @@ func TestNegotiationFailed(t *testing.T) { } transportA, err := NewPCTransport(params) require.NoError(t, err) + _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) + require.NoError(t, err) - transportA.OnICECandidate(func(candidate *webrtc.ICECandidate) error { - if candidate == nil { - return nil - } - t.Logf("got ICE candidate from A: %v", candidate) - return nil - }) + paramsB := params + paramsB.IsOfferer = false + transportB, err := NewPCTransport(paramsB) + require.NoError(t, err) + // exchange ICE + handleICEExchange(t, transportA, transportB) + + // wait for transport to be connected before maiming the signalling channel + connectTransports(t, transportA, transportB, false, 1, 1) + + // reset OnOffer to force a negotiation failure transportA.OnOffer(func(sd webrtc.SessionDescription) error { return nil }) var failed atomic.Int32 transportA.OnNegotiationFailed(func() { @@ -358,7 +365,7 @@ func TestFilteringCandidates(t *testing.T) { transport, err := NewPCTransport(params) require.NoError(t, err) - _, err = transport.pc.CreateDataChannel("test", nil) + _, err = transport.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) _, err = transport.pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio) @@ -519,6 +526,28 @@ func connectTransports(t *testing.T, offerer, answerer *PCTransport, isICERestar require.Eventually(t, func() bool { return answerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected }, 10*time.Second, time.Millisecond*10, "answerer did not become connected") + + transportsConnected := untilTransportsConnected(offerer, answerer) + transportsConnected.Wait() +} + +func untilTransportsConnected(transports ...*PCTransport) *sync.WaitGroup { + var triggered sync.WaitGroup + triggered.Add(len(transports)) + + for _, t := range transports { + var done atomic.Value + done.Store(false) + hdlr := func() { + if val, ok := done.Load().(bool); ok && !val { + done.Store(true) + triggered.Done() + } + } + + t.OnInitialConnected(hdlr) + } + return &triggered } func TestConfigureAudioTransceiver(t *testing.T) { diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 0c2d7d52b..9d65363a2 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -555,7 +555,7 @@ func (s *StreamAllocator) postEvent(event Event) { select { case s.eventCh <- event: default: - s.params.Logger.Warnw("stream allocator: event queue full", nil) + s.params.Logger.Warnw("stream allocator: event queue full", nil, "event", event.String()) } s.eventChMu.RUnlock() } From beecfe37107b7573be560774fa579f0422f52279 Mon Sep 17 00:00:00 2001 From: cfbraun Date: Sat, 2 Dec 2023 19:29:43 +0100 Subject: [PATCH 006/114] Send data (#2270) * Avoid dropping data packets on local router * Remove change not needed for PR --- pkg/routing/localrouter.go | 1 - pkg/routing/redisrouter.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index b0fcbbccb..01687ef3f 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -142,7 +142,6 @@ func (r *LocalRouter) WriteNodeRTC(_ context.Context, _ string, msg *livekit.RTC } func (r *LocalRouter) writeRTCMessage(sink MessageSink, msg *livekit.RTCNodeMessage) error { - defer sink.Close() msg.SenderTime = time.Now().Unix() return sink.WriteMessage(msg) } diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index a3e7e8365..ace768a11 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -215,6 +215,7 @@ func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.Ro rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), "ephemeral", pkey, pkeyB62) msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) + defer rtcSink.Close() return r.writeRTCMessage(rtcSink, msg) } @@ -230,6 +231,7 @@ func (r *RedisRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomNam func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *livekit.RTCNodeMessage) error { rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), "ephemeral", livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) + defer rtcSink.Close() return r.writeRTCMessage(rtcSink, msg) } From 3fe124c87f58748b42e119bf76c221983673d686 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 2 Dec 2023 15:07:31 -0800 Subject: [PATCH 007/114] Log cleanup pass (#2285) * Log cleanup pass Demoted a bunch of logs to DEBUG, consolidated logs. * use context logger and fix context var usage * moved common error types, fixed tests --- go.mod | 2 +- go.sum | 4 +- pkg/routing/errors.go | 9 +- pkg/routing/interfaces.go | 9 +- pkg/routing/localrouter.go | 13 +- pkg/routing/redisrouter.go | 20 +- pkg/routing/routingfakes/fake_router.go | 48 +- pkg/rtc/participant.go | 6 +- pkg/rtc/participant_signal.go | 4 +- pkg/rtc/subscriptionmanager.go | 2 +- pkg/rtc/transport.go | 53 +- pkg/rtc/uptrackmanager.go | 8 +- pkg/service/agentservice.go | 35 +- pkg/service/auth.go | 8 +- pkg/service/roomservice.go | 6 +- pkg/service/rtcservice.go | 35 +- pkg/service/server.go | 3 +- pkg/service/servicefakes/fake_sipstore.go | 975 +++++++++++++++++++++ pkg/service/twirp.go | 32 +- pkg/service/utils.go | 5 +- pkg/sfu/buffer/buffer.go | 5 +- pkg/sfu/downtrack.go | 12 +- pkg/sfu/streamallocator/streamallocator.go | 2 +- pkg/utils/context.go | 24 +- 24 files changed, 1171 insertions(+), 149 deletions(-) create mode 100644 pkg/service/servicefakes/fake_sipstore.go diff --git a/go.mod b/go.mod index 8a05272f4..4e163b610 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb - github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3 + github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f github.com/livekit/psrpc v0.5.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 5193e6f52..e60920f90 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb h1:KiGg4k+kYQD9NjKixaSDMMeYOO2//XBM4IROTI1Itjo= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3 h1:am72beYtXZM71MRr+12lkG3IyqKxzrCa6slsbKrwMe8= -github.com/livekit/protocol v1.9.3-0.20231130173607-ec88d89da1d3/go.mod h1:8f342d5nvfNp9YAEfJokSR+zbNFpaivgU0h6vwaYhes= +github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f h1:6XPC53t/XEcfIe8BUwKkeFmgTLKPfF77JmQ7nydqoOs= +github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f/go.mod h1:8f342d5nvfNp9YAEfJokSR+zbNFpaivgU0h6vwaYhes= github.com/livekit/psrpc v0.5.2 h1:+MvG8Otm/J6MTg2MP/uuMbrkxOWsrj2hDhu/I1VIU1U= github.com/livekit/psrpc v0.5.2/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/routing/errors.go b/pkg/routing/errors.go index 4b6af0686..28a0dd1ac 100644 --- a/pkg/routing/errors.go +++ b/pkg/routing/errors.go @@ -14,7 +14,9 @@ package routing -import "errors" +import ( + "errors" +) var ( ErrNotFound = errors.New("could not find object") @@ -26,4 +28,9 @@ var ( ErrInvalidRouterMessage = errors.New("invalid router message") ErrChannelClosed = errors.New("channel closed") ErrChannelFull = errors.New("channel is full") + + // errors when starting signal connection + ErrRequestChannelClosed = errors.New("request channel closed") + ErrCouldNotMigrateParticipant = errors.New("could not migrate participant") + ErrClientInfoNotSet = errors.New("client info not set") ) diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 7a450bd16..9411ed2c3 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -107,9 +107,16 @@ type Router interface { OnRTCMessage(callback RTCMessageCallback) } +type StartParticipantSignalResults struct { + ConnectionID livekit.ConnectionID + RequestSink MessageSink + ResponseSource MessageSource + NodeID livekit.NodeID +} + type MessageRouter interface { // StartParticipantSignal participant signal connection is ready to start - StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) + StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) // Write a message to a participant or room WriteParticipantRTC(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index 01687ef3f..d83f5ea60 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -97,18 +97,25 @@ func (r *LocalRouter) ListNodes() ([]*livekit.Node, error) { }, nil } -func (r *LocalRouter) StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) { +func (r *LocalRouter) StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) { return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(r.currentNode.Id)) } -func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) { - connectionID, reqSink, resSource, err = r.signalClient.StartParticipantSignal(ctx, roomName, pi, nodeID) +func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (res StartParticipantSignalResults, err error) { + connectionID, reqSink, resSource, err := r.signalClient.StartParticipantSignal(ctx, roomName, pi, nodeID) if err != nil { logger.Errorw("could not handle new participant", err, "room", roomName, "participant", pi.Identity, "connID", connectionID, ) + } else { + return StartParticipantSignalResults{ + ConnectionID: connectionID, + RequestSink: reqSink, + ResponseSource: resSource, + NodeID: nodeID, + }, nil } return } diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index ace768a11..5a920b9c5 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -156,7 +156,7 @@ func (r *RedisRouter) ListNodes() ([]*livekit.Node, error) { } // StartParticipantSignal signal connection sets up paths to the RTC node, and starts to route messages to that message queue -func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) { +func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) { // find the node where the room is hosted at rtcNode, err := r.GetNodeForRoom(ctx, roomName) if err != nil { @@ -164,33 +164,33 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek } if r.usePSRPCSignal { - connectionID, reqSink, resSource, err = r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) + res, err = r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) if err != nil { return } // map signal & rtc nodes - err = r.setParticipantSignalNode(connectionID, r.currentNode.Id) + err = r.setParticipantSignalNode(res.ConnectionID, r.currentNode.Id) return } - connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) + res.ConnectionID = livekit.ConnectionID(utils.NewGuid("CO_")) pKey := ParticipantKeyLegacy(roomName, pi.Identity) pKeyB62 := ParticipantKey(roomName, pi.Identity) // map signal & rtc nodes - if err = r.setParticipantSignalNode(connectionID, r.currentNode.Id); err != nil { + if err = r.setParticipantSignalNode(res.ConnectionID, r.currentNode.Id); err != nil { return } // index by connectionID, since there may be multiple connections for the participant // set up response channel before sending StartSession and be ready to receive responses. - resChan := r.getOrCreateMessageChannel(r.responseChannels, string(connectionID)) + resChan := r.getOrCreateMessageChannel(r.responseChannels, string(res.ConnectionID)) - sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), connectionID, pKey, pKeyB62) + sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), res.ConnectionID, pKey, pKeyB62) // serialize claims - ss, err := pi.ToStartSession(roomName, connectionID) + ss, err := pi.ToStartSession(roomName, res.ConnectionID) if err != nil { return } @@ -201,7 +201,9 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } - return connectionID, sink, resChan, nil + res.RequestSink = sink + res.ResponseSource = resChan + return res, nil } func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { diff --git a/pkg/routing/routingfakes/fake_router.go b/pkg/routing/routingfakes/fake_router.go index 190996099..b52f36bb0 100644 --- a/pkg/routing/routingfakes/fake_router.go +++ b/pkg/routing/routingfakes/fake_router.go @@ -115,7 +115,7 @@ type FakeRouter struct { startReturnsOnCall map[int]struct { result1 error } - StartParticipantSignalStub func(context.Context, livekit.RoomName, routing.ParticipantInit) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) + StartParticipantSignalStub func(context.Context, livekit.RoomName, routing.ParticipantInit) (routing.StartParticipantSignalResults, error) startParticipantSignalMutex sync.RWMutex startParticipantSignalArgsForCall []struct { arg1 context.Context @@ -123,16 +123,12 @@ type FakeRouter struct { arg3 routing.ParticipantInit } startParticipantSignalReturns struct { - result1 livekit.ConnectionID - result2 routing.MessageSink - result3 routing.MessageSource - result4 error + result1 routing.StartParticipantSignalResults + result2 error } startParticipantSignalReturnsOnCall map[int]struct { - result1 livekit.ConnectionID - result2 routing.MessageSink - result3 routing.MessageSource - result4 error + result1 routing.StartParticipantSignalResults + result2 error } StopStub func() stopMutex sync.RWMutex @@ -725,7 +721,7 @@ func (fake *FakeRouter) StartReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeRouter) StartParticipantSignal(arg1 context.Context, arg2 livekit.RoomName, arg3 routing.ParticipantInit) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) { +func (fake *FakeRouter) StartParticipantSignal(arg1 context.Context, arg2 livekit.RoomName, arg3 routing.ParticipantInit) (routing.StartParticipantSignalResults, error) { fake.startParticipantSignalMutex.Lock() ret, specificReturn := fake.startParticipantSignalReturnsOnCall[len(fake.startParticipantSignalArgsForCall)] fake.startParticipantSignalArgsForCall = append(fake.startParticipantSignalArgsForCall, struct { @@ -741,9 +737,9 @@ func (fake *FakeRouter) StartParticipantSignal(arg1 context.Context, arg2 liveki return stub(arg1, arg2, arg3) } if specificReturn { - return ret.result1, ret.result2, ret.result3, ret.result4 + return ret.result1, ret.result2 } - return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3, fakeReturns.result4 + return fakeReturns.result1, fakeReturns.result2 } func (fake *FakeRouter) StartParticipantSignalCallCount() int { @@ -752,7 +748,7 @@ func (fake *FakeRouter) StartParticipantSignalCallCount() int { return len(fake.startParticipantSignalArgsForCall) } -func (fake *FakeRouter) StartParticipantSignalCalls(stub func(context.Context, livekit.RoomName, routing.ParticipantInit) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error)) { +func (fake *FakeRouter) StartParticipantSignalCalls(stub func(context.Context, livekit.RoomName, routing.ParticipantInit) (routing.StartParticipantSignalResults, error)) { fake.startParticipantSignalMutex.Lock() defer fake.startParticipantSignalMutex.Unlock() fake.StartParticipantSignalStub = stub @@ -765,36 +761,30 @@ func (fake *FakeRouter) StartParticipantSignalArgsForCall(i int) (context.Contex return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeRouter) StartParticipantSignalReturns(result1 livekit.ConnectionID, result2 routing.MessageSink, result3 routing.MessageSource, result4 error) { +func (fake *FakeRouter) StartParticipantSignalReturns(result1 routing.StartParticipantSignalResults, result2 error) { fake.startParticipantSignalMutex.Lock() defer fake.startParticipantSignalMutex.Unlock() fake.StartParticipantSignalStub = nil fake.startParticipantSignalReturns = struct { - result1 livekit.ConnectionID - result2 routing.MessageSink - result3 routing.MessageSource - result4 error - }{result1, result2, result3, result4} + result1 routing.StartParticipantSignalResults + result2 error + }{result1, result2} } -func (fake *FakeRouter) StartParticipantSignalReturnsOnCall(i int, result1 livekit.ConnectionID, result2 routing.MessageSink, result3 routing.MessageSource, result4 error) { +func (fake *FakeRouter) StartParticipantSignalReturnsOnCall(i int, result1 routing.StartParticipantSignalResults, result2 error) { fake.startParticipantSignalMutex.Lock() defer fake.startParticipantSignalMutex.Unlock() fake.StartParticipantSignalStub = nil if fake.startParticipantSignalReturnsOnCall == nil { fake.startParticipantSignalReturnsOnCall = make(map[int]struct { - result1 livekit.ConnectionID - result2 routing.MessageSink - result3 routing.MessageSource - result4 error + result1 routing.StartParticipantSignalResults + result2 error }) } fake.startParticipantSignalReturnsOnCall[i] = struct { - result1 livekit.ConnectionID - result2 routing.MessageSink - result3 routing.MessageSource - result4 error - }{result1, result2, result3, result4} + result1 routing.StartParticipantSignalResults + result2 error + }{result1, result2} } func (fake *FakeRouter) Stop() { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 46acee7e0..3f82ec85b 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -584,7 +584,6 @@ func (p *ParticipantImpl) HandleSignalSourceClose() { if !p.TransportManager.HasPublisherEverConnected() && !p.TransportManager.HasSubscriberEverConnected() { reason := types.ParticipantCloseReasonJoinFailed - p.params.Logger.Infow("closing disconnected participant", "reason", reason) _ = p.Close(false, reason, false) } } @@ -858,7 +857,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { if p.IsClosed() || p.IsDisconnected() { return } - p.subLogger.Infow("closing subscriber peer connection to aid migration") + p.subLogger.Debugw("closing subscriber peer connection to aid migration") // // Close all down tracks before closing subscriber peer connection. @@ -1444,7 +1443,6 @@ func (p *ParticipantImpl) setupDisconnectTimer() { return } reason := types.ParticipantCloseReasonPeerConnectionDisconnected - p.params.Logger.Infow("closing disconnected participant", "reason", reason) _ = p.Close(true, reason, false) }) p.lock.Unlock() @@ -1693,7 +1691,7 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } p.pendingTracks[req.Cid] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}} - p.pubLogger.Infow("pending track added", "trackID", ti.Sid, "track", logger.Proto(ti), "request", logger.Proto(req)) + p.pubLogger.Debugw("pending track added", "trackID", ti.Sid, "track", logger.Proto(ti), "request", logger.Proto(req)) return ti } diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 5214df47e..863fd67d6 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -289,7 +289,9 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { sink := p.getResponseSink() if sink != nil { - p.params.Logger.Infow("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) + if reason != types.SignallingCloseReasonParticipantClose { + p.params.Logger.Infow("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) + } sink.Close() p.SetResponseSink(nil) } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 602e786fd..eec97e026 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -924,7 +924,7 @@ func (s *trackSubscription) handleSourceTrackRemoved() { } // source track removed, we would unsubscribe - s.logger.Infow("unsubscribing from track since source track was removed") + s.logger.Debugw("unsubscribing from track since source track was removed") s.desired = false s.setChangedNotifierLocked(nil) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index b72b1411e..4230b9dfd 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -466,10 +466,10 @@ func (t *PCTransport) setICEStartedAt(at time.Time) { } else if tcpICETimeout > maxTcpICEConnectTimeout { tcpICETimeout = maxTcpICEConnectTimeout } - t.params.Logger.Debugw("set tcp ice connect timer", "timeout", tcpICETimeout, "signalRTT", signalingRTT) + t.params.Logger.Debugw("set TCP ICE connect timer", "timeout", tcpICETimeout, "signalRTT", signalingRTT) t.tcpICETimer = time.AfterFunc(tcpICETimeout, func() { if t.pc.ICEConnectionState() == webrtc.ICEConnectionStateChecking { - t.params.Logger.Infow("tcp ice connect timeout", "timeout", tcpICETimeout, "signalRTT", signalingRTT) + t.params.Logger.Infow("TCP ICE connect timeout", "timeout", tcpICETimeout, "signalRTT", signalingRTT) t.handleConnectionFailed(true) } }) @@ -497,7 +497,7 @@ func (t *PCTransport) setICEConnectedAt(at time.Time) { if connTimeoutAfterICE > maxConnectTimeoutAfterICE { connTimeoutAfterICE = maxConnectTimeoutAfterICE } - t.params.Logger.Debugw("setting connection timer after ice connected", "timeout", connTimeoutAfterICE, "iceDuration", iceDuration) + t.params.Logger.Debugw("setting connection timer after ICE connected", "timeout", connTimeoutAfterICE, "iceDuration", iceDuration) t.connectAfterICETimer = time.AfterFunc(connTimeoutAfterICE, func() { state := t.pc.ConnectionState() // if pc is still checking or connected but not fully established after timeout, then fire connection fail @@ -546,12 +546,12 @@ func (t *PCTransport) IsShortConnection(at time.Time) (bool, time.Duration) { } func (t *PCTransport) getSelectedPair() (*webrtc.ICECandidatePair, error) { - sctp := t.pc.SCTP() - if sctp == nil { + s := t.pc.SCTP() + if s == nil { return nil, errors.New("no SCTP") } - dtlsTransport := sctp.Transport() + dtlsTransport := s.Transport() if dtlsTransport == nil { return nil, errors.New("no DTLS transport") } @@ -629,11 +629,6 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState switch state { case webrtc.ICEConnectionStateConnected: t.setICEConnectedAt(time.Now()) - if pair, err := t.getSelectedPair(); err != nil { - t.params.Logger.Errorw("error getting selected ICE candidate pair", err) - } else { - t.params.Logger.Infow("selected ICE candidate pair", "pair", pair) - } case webrtc.ICEConnectionStateChecking: t.setICEStartedAt(time.Now()) @@ -1268,7 +1263,7 @@ func (t *PCTransport) preparePC(previousAnswer webrtc.SessionDescription) error // trying to replicate previous setup, read from previous answer and use that role. // se := webrtc.SettingEngine{} - se.SetAnsweringDTLSRole(lksdp.ExtractDTLSRole(parsed)) + _ = se.SetAnsweringDTLSRole(lksdp.ExtractDTLSRole(parsed)) api := webrtc.NewAPI( webrtc.WithSettingEngine(se), webrtc.WithMediaEngine(t.me), @@ -1446,9 +1441,11 @@ func (t *PCTransport) processEvents() { err := t.handleEvent(&event) if err != nil { - t.params.Logger.Errorw("error handling event", err, "event", event.String()) - if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { - onNegotiationFailed() + if !t.isClosed.Load() { + t.params.Logger.Errorw("error handling event", err, "event", event.String()) + if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { + onNegotiationFailed() + } } break } @@ -1480,7 +1477,7 @@ func (t *PCTransport) handleEvent(e *event) error { return nil } -func (t *PCTransport) handleICEGatheringComplete(e *event) error { +func (t *PCTransport) handleICEGatheringComplete(_ *event) error { if t.params.IsOfferer { return t.handleICEGatheringCompleteOfferer() } else { @@ -1609,17 +1606,25 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { return nil } -func (t *PCTransport) handleLogICECandidates(e *event) error { +func (t *PCTransport) handleLogICECandidates(_ *event) error { lc := t.allowedLocalCandidates.Get() rc := t.allowedRemoteCandidates.Get() + var fields []interface{} if len(lc) != 0 || len(rc) != 0 { - t.params.Logger.Infow( - "ice candidates", + fields = append(fields, "lc", lc, "rc", rc, - "lc (filtered)", t.filteredLocalCandidates.Get(), - "rc (filtered)", t.filteredRemoteCandidates.Get(), + "lc_filtered", t.filteredLocalCandidates.Get(), + "rc_filtered", t.filteredRemoteCandidates.Get(), ) + + } + if pair, err := t.getSelectedPair(); err == nil { + fields = append(fields, "selected_pair", pair) + } + + if len(fields) > 0 { + t.params.Logger.Infow("ice candidates", fields...) } return nil @@ -1711,7 +1716,7 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { // when there's an ongoing negotiation, let it finish and not disrupt its state if t.negotiationState == NegotiationStateRemote { - t.params.Logger.Infow("skipping negotiation, trying again later") + t.params.Logger.Debugw("skipping negotiation, trying again later") t.setNegotiationState(NegotiationStateRetry) return nil } else if t.negotiationState == NegotiationStateRetry { @@ -1801,7 +1806,7 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { return ErrNoOfferHandler } -func (t *PCTransport) handleSendOffer(e *event) error { +func (t *PCTransport) handleSendOffer(_ *event) error { return t.createAndSendOffer(nil) } @@ -2030,7 +2035,7 @@ func (t *PCTransport) doICERestart() error { } } -func (t *PCTransport) handleICERestart(e *event) error { +func (t *PCTransport) handleICERestart(_ *event) error { return t.doICERestart() } diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 1c079dc34..ce5177661 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -111,7 +111,7 @@ func (u *UpTrackManager) SetPublishedTrackMuted(trackID livekit.TrackID, muted b track.SetMuted(muted) if currentMuted != track.IsMuted() { - u.params.Logger.Infow("publisher mute status changed", "trackID", trackID, "muted", track.IsMuted()) + u.params.Logger.Debugw("publisher mute status changed", "trackID", trackID, "muted", track.IsMuted()) if u.onTrackUpdated != nil { u.onTrackUpdated(track) } @@ -142,7 +142,7 @@ func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack { func (u *UpTrackManager) UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, timedVersion utils.TimedVersion, - resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, + _ func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, // TODO: separate PR to remove this argument resolverBySid func(participantID livekit.ParticipantID) types.LocalParticipant, ) error { u.lock.Lock() @@ -203,7 +203,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( } u.lock.Unlock() - u.maybeRevokeSubscriptions(resolverByIdentity) + u.maybeRevokeSubscriptions() return nil } @@ -381,7 +381,7 @@ func (u *UpTrackManager) getAllowedSubscribersLocked(trackID livekit.TrackID) [] return allowed } -func (u *UpTrackManager) maybeRevokeSubscriptions(resolver func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant) { +func (u *UpTrackManager) maybeRevokeSubscriptions() { u.lock.Lock() defer u.lock.Unlock() diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index f2ba672b2..9e7eef0e8 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -28,6 +28,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -65,6 +66,7 @@ type worker struct { jobType livekit.JobType status livekit.WorkerStatus activeJobs int + logger logger.Logger } type availability struct { @@ -102,18 +104,18 @@ func (s *AgentService) ServeHTTP(writer http.ResponseWriter, r *http.Request) { // require a claim claims := GetGrants(r.Context()) if claims == nil || claims.Video == nil || !claims.Video.Agent { - handleError(writer, http.StatusUnauthorized, rtc.ErrPermissionDenied) + handleError(writer, r, http.StatusUnauthorized, rtc.ErrPermissionDenied) return } // upgrade conn, err := s.upgrader.Upgrade(writer, r, nil) if err != nil { - handleError(writer, http.StatusInternalServerError, err) + handleError(writer, r, http.StatusInternalServerError, err) return } - s.HandleConnection(conn) + s.HandleConnection(r.Context(), conn) } func NewAgentHandler(agentServer rpc.AgentInternalServer, roomTopic, publisherTopic string) *AgentHandler { @@ -128,11 +130,12 @@ func NewAgentHandler(agentServer rpc.AgentInternalServer, roomTopic, publisherTo } } -func (s *AgentHandler) HandleConnection(conn *websocket.Conn) { +func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Conn) { sigConn := NewWSSignalConnection(conn) w := &worker{ conn: conn, sigConn: sigConn, + logger: utils.GetLogger(ctx), } s.mu.Lock() @@ -177,9 +180,9 @@ func (s *AgentHandler) HandleConnection(conn *websocket.Conn) { websocket.CloseNormalClosure, websocket.CloseNoStatusReceived, ) { - logger.Infow("exit ws read loop for closed connection", "wsError", err) + w.logger.Infow("Agent worker closed WS connection", "wsError", err) } else { - logger.Errorw("error reading from websocket", err) + w.logger.Errorw("error reading from websocket", err) } return } @@ -199,7 +202,7 @@ func (s *AgentHandler) HandleConnection(conn *websocket.Conn) { func (s *AgentHandler) handleRegister(worker *worker, msg *livekit.RegisterWorkerRequest) { if err := s.doHandleRegister(worker, msg); err != nil { - logger.Errorw("failed to register worker", err, "workerID", msg.WorkerId, "jobType", msg.Type) + worker.logger.Errorw("failed to register worker", err, "workerID", msg.WorkerId, "jobType", msg.Type) worker.conn.Close() } } @@ -225,7 +228,7 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor if !s.roomRegistered { err := s.agentServer.RegisterJobRequestTopic(s.roomTopic) if err != nil { - logger.Errorw("failed to register room agents", err) + worker.logger.Errorw("failed to register room agents", err) } else { s.roomRegistered = true } @@ -240,7 +243,7 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor if !s.publisherRegistered { err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic) if err != nil { - logger.Errorw("failed to register publisher agents", err) + worker.logger.Errorw("failed to register publisher agents", err) } else { s.publisherRegistered = true } @@ -260,7 +263,7 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor }, }) if err != nil { - logger.Errorw("failed to write server message", err) + worker.logger.Errorw("failed to write server message", err) } return nil @@ -282,9 +285,9 @@ func (s *AgentHandler) handleAvailability(w *worker, msg *livekit.AvailabilityRe func (s *AgentHandler) handleJobUpdate(w *worker, msg *livekit.JobStatusUpdate) { switch msg.Status { case livekit.JobStatus_JS_SUCCESS: - logger.Debugw("job complete", "jobID", msg.JobId) + w.logger.Debugw("job complete", "jobID", msg.JobId) case livekit.JobStatus_JS_FAILED: - logger.Warnw("job failed", errors.New(msg.Error), "jobID", msg.JobId) + w.logger.Warnw("job failed", errors.New(msg.Error), "jobID", msg.JobId) } w.mu.Lock() @@ -307,7 +310,7 @@ func (s *AgentHandler) handleStatus(w *worker, msg *livekit.UpdateWorkerStatus) s.agentServer.DeregisterJobRequestTopic(s.roomTopic) } else if !s.roomRegistered && s.roomAvailableLocked() { if err := s.agentServer.RegisterJobRequestTopic(s.roomTopic); err != nil { - logger.Errorw("failed to register room agents", err) + w.logger.Errorw("failed to register room agents", err) } else { s.roomRegistered = true } @@ -318,7 +321,7 @@ func (s *AgentHandler) handleStatus(w *worker, msg *livekit.UpdateWorkerStatus) s.agentServer.DeregisterJobRequestTopic(s.publisherTopic) } else if !s.publisherRegistered && s.publisherAvailableLocked() { if err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic); err != nil { - logger.Errorw("failed to register publisher agents", err) + w.logger.Errorw("failed to register publisher agents", err) } else { s.publisherRegistered = true } @@ -388,7 +391,7 @@ func (s *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*empty Availability: &livekit.AvailabilityRequest{Job: job}, }}) if err != nil { - logger.Errorw("failed to send availability request", err, "workerID", selected.id) + selected.logger.Errorw("failed to send availability request", err, "workerID", selected.id) } select { @@ -400,7 +403,7 @@ func (s *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*empty Assignment: &livekit.JobAssignment{Job: job}, }}) if err != nil { - logger.Errorw("failed to assign job", err, "workerID", selected.id) + selected.logger.Errorw("failed to assign job", err, "workerID", selected.id) } else { selected.mu.Lock() selected.activeJobs++ diff --git a/pkg/service/auth.go b/pkg/service/auth.go index 1b5829e30..5633b2889 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -62,7 +62,7 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, if authHeader != "" { if !strings.HasPrefix(authHeader, bearerPrefix) { - handleError(w, http.StatusUnauthorized, ErrMissingAuthorization) + handleError(w, r, http.StatusUnauthorized, ErrMissingAuthorization) return } @@ -75,19 +75,19 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, if authToken != "" { v, err := auth.ParseAPIToken(authToken) if err != nil { - handleError(w, http.StatusUnauthorized, ErrInvalidAuthorizationToken) + handleError(w, r, http.StatusUnauthorized, ErrInvalidAuthorizationToken) return } secret := m.provider.GetSecret(v.APIKey()) if secret == "" { - handleError(w, http.StatusUnauthorized, errors.New("invalid API key: "+v.APIKey())) + handleError(w, r, http.StatusUnauthorized, errors.New("invalid API key: "+v.APIKey())) return } grants, err := v.Verify(secret) if err != nil { - handleError(w, http.StatusUnauthorized, errors.New("invalid token: "+authToken+", error: "+err.Error())) + handleError(w, r, http.StatusUnauthorized, errors.New("invalid token: "+authToken+", error: "+err.Error())) return } diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 174ea3391..2076b5ca4 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -93,15 +93,15 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq } // actually start the room on an RTC node, to ensure metadata & empty timeout functionality - _, sink, source, err := s.router.StartParticipantSignal(ctx, + res, err := s.router.StartParticipantSignal(ctx, livekit.RoomName(req.Name), routing.ParticipantInit{}, ) if err != nil { return nil, err } - defer sink.Close() - defer source.Close() + defer res.RequestSink.Close() + defer res.ResponseSource.Close() // ensure it's created correctly err = s.confirmExecution(func() error { diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 86ff5f630..a273edecf 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -29,6 +29,7 @@ import ( "github.com/gorilla/websocket" "github.com/ua-parser/uap-go/uaparser" + "go.uber.org/atomic" "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/config" @@ -39,7 +40,6 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" putil "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" ) @@ -97,7 +97,7 @@ func NewRTCService( func (s *RTCService) Validate(w http.ResponseWriter, r *http.Request) { _, _, code, err := s.validate(r) if err != nil { - handleError(w, code, err) + handleError(w, r, code, err) return } _, _ = w.Write([]byte("success")) @@ -202,7 +202,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { roomName, pi, code, err := s.validate(r) if err != nil { - handleError(w, code, err) + handleError(w, r, code, err) return } @@ -213,6 +213,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { "remote", false, } + l := utils.GetLogger(r.Context()) + // give it a few attempts to start session var cr connectionResult var initialResponse *livekit.SignalResponse @@ -229,13 +231,13 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if i < 2 { fieldsWithAttempt := append(loggerFields, "attempt", i) - logger.Warnw("failed to start connection, retrying", err, fieldsWithAttempt...) + l.Warnw("failed to start connection, retrying", err, fieldsWithAttempt...) } } if err != nil { prometheus.IncrementParticipantJoinFail(1) - handleError(w, http.StatusInternalServerError, err, loggerFields...) + handleError(w, r, http.StatusInternalServerError, err, loggerFields...) return } @@ -254,16 +256,20 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { } pLogger := rtc.LoggerWithParticipant( - rtc.LoggerWithRoom(logger.GetLogger(), roomName, livekit.RoomID(cr.Room.Sid)), + rtc.LoggerWithRoom(l, roomName, livekit.RoomID(cr.Room.Sid)), pi.Identity, pi.ID, false, ) + closedByClient := atomic.NewBool(false) done := make(chan struct{}) // function exits when websocket terminates, it'll close the event reading off of request sink and response source as well defer func() { - pLogger.Infow("finishing WS connection", "connID", cr.ConnectionID) + pLogger.Infow("finishing WS connection", + "connID", cr.ConnectionID, + "closedByClient", closedByClient.Load(), + ) cr.ResponseSource.Close() cr.RequestSink.Close() close(done) @@ -276,7 +282,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // upgrade only once the basics are good to go conn, err := s.upgrader.Upgrade(w, r, nil) if err != nil { - handleError(w, http.StatusInternalServerError, err, loggerFields...) + handleError(w, r, http.StatusInternalServerError, err, loggerFields...) return } @@ -305,6 +311,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { "reconnect", pi.Reconnect, "reconnectReason", pi.ReconnectReason, "adaptiveStream", pi.AdaptiveStream, + "selectedNodeID", cr.NodeID, ) // handle responses @@ -366,7 +373,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { req, count, err := sigConn.ReadRequest() if err != nil { // normal/expected closure - if err == io.EOF || + if errors.Is(err, io.EOF) || strings.HasSuffix(err.Error(), "use of closed network connection") || strings.HasSuffix(err.Error(), "connection reset by peer") || websocket.IsCloseError( @@ -376,7 +383,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { websocket.CloseNormalClosure, websocket.CloseNoStatusReceived, ) { - pLogger.Infow("exit ws read loop for closed connection", "connID", cr.ConnectionID, "wsError", err) + closedByClient.Store(true) } else { pLogger.Errorw("error reading from websocket", err, "connID", cr.ConnectionID) } @@ -512,10 +519,8 @@ func (s *RTCService) DrainConnections(interval time.Duration) { } type connectionResult struct { - Room *livekit.Room - ConnectionID livekit.ConnectionID - RequestSink routing.MessageSink - ResponseSource routing.MessageSource + routing.StartParticipantSignalResults + Room *livekit.Room } func (s *RTCService) startConnection( @@ -533,7 +538,7 @@ func (s *RTCService) startConnection( } // this needs to be started first *before* using router functions on this node - cr.ConnectionID, cr.RequestSink, cr.ResponseSource, err = s.router.StartParticipantSignal(ctx, roomName, pi) + cr.StartParticipantSignalResults, err = s.router.StartParticipantSignal(ctx, roomName, pi) if err != nil { return cr, nil, err } diff --git a/pkg/service/server.go b/pkg/service/server.go index f66d848b5..fcb09a648 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -37,7 +37,6 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/livekit-server/version" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -107,7 +106,7 @@ func NewLivekitServer(conf *config.Config, middlewares = append(middlewares, NewAPIKeyAuthMiddleware(keyProvider)) } - twirpLoggingHook := TwirpLogger(logger.GetLogger().WithComponent(sutils.ComponentAPI)) + twirpLoggingHook := TwirpLogger() twirpRequestStatusHook := TwirpRequestStatusReporter() roomServer := livekit.NewRoomServiceServer(roomService, twirpLoggingHook) egressServer := livekit.NewEgressServer(egressService, twirp.WithServerHooks( diff --git a/pkg/service/servicefakes/fake_sipstore.go b/pkg/service/servicefakes/fake_sipstore.go new file mode 100644 index 000000000..069eff951 --- /dev/null +++ b/pkg/service/servicefakes/fake_sipstore.go @@ -0,0 +1,975 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package servicefakes + +import ( + "context" + "sync" + + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/protocol/livekit" +) + +type FakeSIPStore struct { + DeleteSIPDispatchRuleStub func(context.Context, *livekit.SIPDispatchRuleInfo) error + deleteSIPDispatchRuleMutex sync.RWMutex + deleteSIPDispatchRuleArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPDispatchRuleInfo + } + deleteSIPDispatchRuleReturns struct { + result1 error + } + deleteSIPDispatchRuleReturnsOnCall map[int]struct { + result1 error + } + DeleteSIPParticipantStub func(context.Context, *livekit.SIPParticipantInfo) error + deleteSIPParticipantMutex sync.RWMutex + deleteSIPParticipantArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPParticipantInfo + } + deleteSIPParticipantReturns struct { + result1 error + } + deleteSIPParticipantReturnsOnCall map[int]struct { + result1 error + } + DeleteSIPTrunkStub func(context.Context, *livekit.SIPTrunkInfo) error + deleteSIPTrunkMutex sync.RWMutex + deleteSIPTrunkArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPTrunkInfo + } + deleteSIPTrunkReturns struct { + result1 error + } + deleteSIPTrunkReturnsOnCall map[int]struct { + result1 error + } + ListSIPDispatchRuleStub func(context.Context) ([]*livekit.SIPDispatchRuleInfo, error) + listSIPDispatchRuleMutex sync.RWMutex + listSIPDispatchRuleArgsForCall []struct { + arg1 context.Context + } + listSIPDispatchRuleReturns struct { + result1 []*livekit.SIPDispatchRuleInfo + result2 error + } + listSIPDispatchRuleReturnsOnCall map[int]struct { + result1 []*livekit.SIPDispatchRuleInfo + result2 error + } + ListSIPParticipantStub func(context.Context) ([]*livekit.SIPParticipantInfo, error) + listSIPParticipantMutex sync.RWMutex + listSIPParticipantArgsForCall []struct { + arg1 context.Context + } + listSIPParticipantReturns struct { + result1 []*livekit.SIPParticipantInfo + result2 error + } + listSIPParticipantReturnsOnCall map[int]struct { + result1 []*livekit.SIPParticipantInfo + result2 error + } + ListSIPTrunkStub func(context.Context) ([]*livekit.SIPTrunkInfo, error) + listSIPTrunkMutex sync.RWMutex + listSIPTrunkArgsForCall []struct { + arg1 context.Context + } + listSIPTrunkReturns struct { + result1 []*livekit.SIPTrunkInfo + result2 error + } + listSIPTrunkReturnsOnCall map[int]struct { + result1 []*livekit.SIPTrunkInfo + result2 error + } + LoadSIPDispatchRuleStub func(context.Context, string) (*livekit.SIPDispatchRuleInfo, error) + loadSIPDispatchRuleMutex sync.RWMutex + loadSIPDispatchRuleArgsForCall []struct { + arg1 context.Context + arg2 string + } + loadSIPDispatchRuleReturns struct { + result1 *livekit.SIPDispatchRuleInfo + result2 error + } + loadSIPDispatchRuleReturnsOnCall map[int]struct { + result1 *livekit.SIPDispatchRuleInfo + result2 error + } + LoadSIPParticipantStub func(context.Context, string) (*livekit.SIPParticipantInfo, error) + loadSIPParticipantMutex sync.RWMutex + loadSIPParticipantArgsForCall []struct { + arg1 context.Context + arg2 string + } + loadSIPParticipantReturns struct { + result1 *livekit.SIPParticipantInfo + result2 error + } + loadSIPParticipantReturnsOnCall map[int]struct { + result1 *livekit.SIPParticipantInfo + result2 error + } + LoadSIPTrunkStub func(context.Context, string) (*livekit.SIPTrunkInfo, error) + loadSIPTrunkMutex sync.RWMutex + loadSIPTrunkArgsForCall []struct { + arg1 context.Context + arg2 string + } + loadSIPTrunkReturns struct { + result1 *livekit.SIPTrunkInfo + result2 error + } + loadSIPTrunkReturnsOnCall map[int]struct { + result1 *livekit.SIPTrunkInfo + result2 error + } + StoreSIPDispatchRuleStub func(context.Context, *livekit.SIPDispatchRuleInfo) error + storeSIPDispatchRuleMutex sync.RWMutex + storeSIPDispatchRuleArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPDispatchRuleInfo + } + storeSIPDispatchRuleReturns struct { + result1 error + } + storeSIPDispatchRuleReturnsOnCall map[int]struct { + result1 error + } + StoreSIPParticipantStub func(context.Context, *livekit.SIPParticipantInfo) error + storeSIPParticipantMutex sync.RWMutex + storeSIPParticipantArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPParticipantInfo + } + storeSIPParticipantReturns struct { + result1 error + } + storeSIPParticipantReturnsOnCall map[int]struct { + result1 error + } + StoreSIPTrunkStub func(context.Context, *livekit.SIPTrunkInfo) error + storeSIPTrunkMutex sync.RWMutex + storeSIPTrunkArgsForCall []struct { + arg1 context.Context + arg2 *livekit.SIPTrunkInfo + } + storeSIPTrunkReturns struct { + result1 error + } + storeSIPTrunkReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRule(arg1 context.Context, arg2 *livekit.SIPDispatchRuleInfo) error { + fake.deleteSIPDispatchRuleMutex.Lock() + ret, specificReturn := fake.deleteSIPDispatchRuleReturnsOnCall[len(fake.deleteSIPDispatchRuleArgsForCall)] + fake.deleteSIPDispatchRuleArgsForCall = append(fake.deleteSIPDispatchRuleArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPDispatchRuleInfo + }{arg1, arg2}) + stub := fake.DeleteSIPDispatchRuleStub + fakeReturns := fake.deleteSIPDispatchRuleReturns + fake.recordInvocation("DeleteSIPDispatchRule", []interface{}{arg1, arg2}) + fake.deleteSIPDispatchRuleMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRuleCallCount() int { + fake.deleteSIPDispatchRuleMutex.RLock() + defer fake.deleteSIPDispatchRuleMutex.RUnlock() + return len(fake.deleteSIPDispatchRuleArgsForCall) +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRuleCalls(stub func(context.Context, *livekit.SIPDispatchRuleInfo) error) { + fake.deleteSIPDispatchRuleMutex.Lock() + defer fake.deleteSIPDispatchRuleMutex.Unlock() + fake.DeleteSIPDispatchRuleStub = stub +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRuleArgsForCall(i int) (context.Context, *livekit.SIPDispatchRuleInfo) { + fake.deleteSIPDispatchRuleMutex.RLock() + defer fake.deleteSIPDispatchRuleMutex.RUnlock() + argsForCall := fake.deleteSIPDispatchRuleArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRuleReturns(result1 error) { + fake.deleteSIPDispatchRuleMutex.Lock() + defer fake.deleteSIPDispatchRuleMutex.Unlock() + fake.DeleteSIPDispatchRuleStub = nil + fake.deleteSIPDispatchRuleReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) DeleteSIPDispatchRuleReturnsOnCall(i int, result1 error) { + fake.deleteSIPDispatchRuleMutex.Lock() + defer fake.deleteSIPDispatchRuleMutex.Unlock() + fake.DeleteSIPDispatchRuleStub = nil + if fake.deleteSIPDispatchRuleReturnsOnCall == nil { + fake.deleteSIPDispatchRuleReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteSIPDispatchRuleReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) DeleteSIPParticipant(arg1 context.Context, arg2 *livekit.SIPParticipantInfo) error { + fake.deleteSIPParticipantMutex.Lock() + ret, specificReturn := fake.deleteSIPParticipantReturnsOnCall[len(fake.deleteSIPParticipantArgsForCall)] + fake.deleteSIPParticipantArgsForCall = append(fake.deleteSIPParticipantArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPParticipantInfo + }{arg1, arg2}) + stub := fake.DeleteSIPParticipantStub + fakeReturns := fake.deleteSIPParticipantReturns + fake.recordInvocation("DeleteSIPParticipant", []interface{}{arg1, arg2}) + fake.deleteSIPParticipantMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) DeleteSIPParticipantCallCount() int { + fake.deleteSIPParticipantMutex.RLock() + defer fake.deleteSIPParticipantMutex.RUnlock() + return len(fake.deleteSIPParticipantArgsForCall) +} + +func (fake *FakeSIPStore) DeleteSIPParticipantCalls(stub func(context.Context, *livekit.SIPParticipantInfo) error) { + fake.deleteSIPParticipantMutex.Lock() + defer fake.deleteSIPParticipantMutex.Unlock() + fake.DeleteSIPParticipantStub = stub +} + +func (fake *FakeSIPStore) DeleteSIPParticipantArgsForCall(i int) (context.Context, *livekit.SIPParticipantInfo) { + fake.deleteSIPParticipantMutex.RLock() + defer fake.deleteSIPParticipantMutex.RUnlock() + argsForCall := fake.deleteSIPParticipantArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) DeleteSIPParticipantReturns(result1 error) { + fake.deleteSIPParticipantMutex.Lock() + defer fake.deleteSIPParticipantMutex.Unlock() + fake.DeleteSIPParticipantStub = nil + fake.deleteSIPParticipantReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) DeleteSIPParticipantReturnsOnCall(i int, result1 error) { + fake.deleteSIPParticipantMutex.Lock() + defer fake.deleteSIPParticipantMutex.Unlock() + fake.DeleteSIPParticipantStub = nil + if fake.deleteSIPParticipantReturnsOnCall == nil { + fake.deleteSIPParticipantReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteSIPParticipantReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) DeleteSIPTrunk(arg1 context.Context, arg2 *livekit.SIPTrunkInfo) error { + fake.deleteSIPTrunkMutex.Lock() + ret, specificReturn := fake.deleteSIPTrunkReturnsOnCall[len(fake.deleteSIPTrunkArgsForCall)] + fake.deleteSIPTrunkArgsForCall = append(fake.deleteSIPTrunkArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPTrunkInfo + }{arg1, arg2}) + stub := fake.DeleteSIPTrunkStub + fakeReturns := fake.deleteSIPTrunkReturns + fake.recordInvocation("DeleteSIPTrunk", []interface{}{arg1, arg2}) + fake.deleteSIPTrunkMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) DeleteSIPTrunkCallCount() int { + fake.deleteSIPTrunkMutex.RLock() + defer fake.deleteSIPTrunkMutex.RUnlock() + return len(fake.deleteSIPTrunkArgsForCall) +} + +func (fake *FakeSIPStore) DeleteSIPTrunkCalls(stub func(context.Context, *livekit.SIPTrunkInfo) error) { + fake.deleteSIPTrunkMutex.Lock() + defer fake.deleteSIPTrunkMutex.Unlock() + fake.DeleteSIPTrunkStub = stub +} + +func (fake *FakeSIPStore) DeleteSIPTrunkArgsForCall(i int) (context.Context, *livekit.SIPTrunkInfo) { + fake.deleteSIPTrunkMutex.RLock() + defer fake.deleteSIPTrunkMutex.RUnlock() + argsForCall := fake.deleteSIPTrunkArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) DeleteSIPTrunkReturns(result1 error) { + fake.deleteSIPTrunkMutex.Lock() + defer fake.deleteSIPTrunkMutex.Unlock() + fake.DeleteSIPTrunkStub = nil + fake.deleteSIPTrunkReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) DeleteSIPTrunkReturnsOnCall(i int, result1 error) { + fake.deleteSIPTrunkMutex.Lock() + defer fake.deleteSIPTrunkMutex.Unlock() + fake.DeleteSIPTrunkStub = nil + if fake.deleteSIPTrunkReturnsOnCall == nil { + fake.deleteSIPTrunkReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteSIPTrunkReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) ListSIPDispatchRule(arg1 context.Context) ([]*livekit.SIPDispatchRuleInfo, error) { + fake.listSIPDispatchRuleMutex.Lock() + ret, specificReturn := fake.listSIPDispatchRuleReturnsOnCall[len(fake.listSIPDispatchRuleArgsForCall)] + fake.listSIPDispatchRuleArgsForCall = append(fake.listSIPDispatchRuleArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.ListSIPDispatchRuleStub + fakeReturns := fake.listSIPDispatchRuleReturns + fake.recordInvocation("ListSIPDispatchRule", []interface{}{arg1}) + fake.listSIPDispatchRuleMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) ListSIPDispatchRuleCallCount() int { + fake.listSIPDispatchRuleMutex.RLock() + defer fake.listSIPDispatchRuleMutex.RUnlock() + return len(fake.listSIPDispatchRuleArgsForCall) +} + +func (fake *FakeSIPStore) ListSIPDispatchRuleCalls(stub func(context.Context) ([]*livekit.SIPDispatchRuleInfo, error)) { + fake.listSIPDispatchRuleMutex.Lock() + defer fake.listSIPDispatchRuleMutex.Unlock() + fake.ListSIPDispatchRuleStub = stub +} + +func (fake *FakeSIPStore) ListSIPDispatchRuleArgsForCall(i int) context.Context { + fake.listSIPDispatchRuleMutex.RLock() + defer fake.listSIPDispatchRuleMutex.RUnlock() + argsForCall := fake.listSIPDispatchRuleArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSIPStore) ListSIPDispatchRuleReturns(result1 []*livekit.SIPDispatchRuleInfo, result2 error) { + fake.listSIPDispatchRuleMutex.Lock() + defer fake.listSIPDispatchRuleMutex.Unlock() + fake.ListSIPDispatchRuleStub = nil + fake.listSIPDispatchRuleReturns = struct { + result1 []*livekit.SIPDispatchRuleInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) ListSIPDispatchRuleReturnsOnCall(i int, result1 []*livekit.SIPDispatchRuleInfo, result2 error) { + fake.listSIPDispatchRuleMutex.Lock() + defer fake.listSIPDispatchRuleMutex.Unlock() + fake.ListSIPDispatchRuleStub = nil + if fake.listSIPDispatchRuleReturnsOnCall == nil { + fake.listSIPDispatchRuleReturnsOnCall = make(map[int]struct { + result1 []*livekit.SIPDispatchRuleInfo + result2 error + }) + } + fake.listSIPDispatchRuleReturnsOnCall[i] = struct { + result1 []*livekit.SIPDispatchRuleInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) ListSIPParticipant(arg1 context.Context) ([]*livekit.SIPParticipantInfo, error) { + fake.listSIPParticipantMutex.Lock() + ret, specificReturn := fake.listSIPParticipantReturnsOnCall[len(fake.listSIPParticipantArgsForCall)] + fake.listSIPParticipantArgsForCall = append(fake.listSIPParticipantArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.ListSIPParticipantStub + fakeReturns := fake.listSIPParticipantReturns + fake.recordInvocation("ListSIPParticipant", []interface{}{arg1}) + fake.listSIPParticipantMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) ListSIPParticipantCallCount() int { + fake.listSIPParticipantMutex.RLock() + defer fake.listSIPParticipantMutex.RUnlock() + return len(fake.listSIPParticipantArgsForCall) +} + +func (fake *FakeSIPStore) ListSIPParticipantCalls(stub func(context.Context) ([]*livekit.SIPParticipantInfo, error)) { + fake.listSIPParticipantMutex.Lock() + defer fake.listSIPParticipantMutex.Unlock() + fake.ListSIPParticipantStub = stub +} + +func (fake *FakeSIPStore) ListSIPParticipantArgsForCall(i int) context.Context { + fake.listSIPParticipantMutex.RLock() + defer fake.listSIPParticipantMutex.RUnlock() + argsForCall := fake.listSIPParticipantArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSIPStore) ListSIPParticipantReturns(result1 []*livekit.SIPParticipantInfo, result2 error) { + fake.listSIPParticipantMutex.Lock() + defer fake.listSIPParticipantMutex.Unlock() + fake.ListSIPParticipantStub = nil + fake.listSIPParticipantReturns = struct { + result1 []*livekit.SIPParticipantInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) ListSIPParticipantReturnsOnCall(i int, result1 []*livekit.SIPParticipantInfo, result2 error) { + fake.listSIPParticipantMutex.Lock() + defer fake.listSIPParticipantMutex.Unlock() + fake.ListSIPParticipantStub = nil + if fake.listSIPParticipantReturnsOnCall == nil { + fake.listSIPParticipantReturnsOnCall = make(map[int]struct { + result1 []*livekit.SIPParticipantInfo + result2 error + }) + } + fake.listSIPParticipantReturnsOnCall[i] = struct { + result1 []*livekit.SIPParticipantInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) ListSIPTrunk(arg1 context.Context) ([]*livekit.SIPTrunkInfo, error) { + fake.listSIPTrunkMutex.Lock() + ret, specificReturn := fake.listSIPTrunkReturnsOnCall[len(fake.listSIPTrunkArgsForCall)] + fake.listSIPTrunkArgsForCall = append(fake.listSIPTrunkArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.ListSIPTrunkStub + fakeReturns := fake.listSIPTrunkReturns + fake.recordInvocation("ListSIPTrunk", []interface{}{arg1}) + fake.listSIPTrunkMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) ListSIPTrunkCallCount() int { + fake.listSIPTrunkMutex.RLock() + defer fake.listSIPTrunkMutex.RUnlock() + return len(fake.listSIPTrunkArgsForCall) +} + +func (fake *FakeSIPStore) ListSIPTrunkCalls(stub func(context.Context) ([]*livekit.SIPTrunkInfo, error)) { + fake.listSIPTrunkMutex.Lock() + defer fake.listSIPTrunkMutex.Unlock() + fake.ListSIPTrunkStub = stub +} + +func (fake *FakeSIPStore) ListSIPTrunkArgsForCall(i int) context.Context { + fake.listSIPTrunkMutex.RLock() + defer fake.listSIPTrunkMutex.RUnlock() + argsForCall := fake.listSIPTrunkArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSIPStore) ListSIPTrunkReturns(result1 []*livekit.SIPTrunkInfo, result2 error) { + fake.listSIPTrunkMutex.Lock() + defer fake.listSIPTrunkMutex.Unlock() + fake.ListSIPTrunkStub = nil + fake.listSIPTrunkReturns = struct { + result1 []*livekit.SIPTrunkInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) ListSIPTrunkReturnsOnCall(i int, result1 []*livekit.SIPTrunkInfo, result2 error) { + fake.listSIPTrunkMutex.Lock() + defer fake.listSIPTrunkMutex.Unlock() + fake.ListSIPTrunkStub = nil + if fake.listSIPTrunkReturnsOnCall == nil { + fake.listSIPTrunkReturnsOnCall = make(map[int]struct { + result1 []*livekit.SIPTrunkInfo + result2 error + }) + } + fake.listSIPTrunkReturnsOnCall[i] = struct { + result1 []*livekit.SIPTrunkInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPDispatchRule(arg1 context.Context, arg2 string) (*livekit.SIPDispatchRuleInfo, error) { + fake.loadSIPDispatchRuleMutex.Lock() + ret, specificReturn := fake.loadSIPDispatchRuleReturnsOnCall[len(fake.loadSIPDispatchRuleArgsForCall)] + fake.loadSIPDispatchRuleArgsForCall = append(fake.loadSIPDispatchRuleArgsForCall, struct { + arg1 context.Context + arg2 string + }{arg1, arg2}) + stub := fake.LoadSIPDispatchRuleStub + fakeReturns := fake.loadSIPDispatchRuleReturns + fake.recordInvocation("LoadSIPDispatchRule", []interface{}{arg1, arg2}) + fake.loadSIPDispatchRuleMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) LoadSIPDispatchRuleCallCount() int { + fake.loadSIPDispatchRuleMutex.RLock() + defer fake.loadSIPDispatchRuleMutex.RUnlock() + return len(fake.loadSIPDispatchRuleArgsForCall) +} + +func (fake *FakeSIPStore) LoadSIPDispatchRuleCalls(stub func(context.Context, string) (*livekit.SIPDispatchRuleInfo, error)) { + fake.loadSIPDispatchRuleMutex.Lock() + defer fake.loadSIPDispatchRuleMutex.Unlock() + fake.LoadSIPDispatchRuleStub = stub +} + +func (fake *FakeSIPStore) LoadSIPDispatchRuleArgsForCall(i int) (context.Context, string) { + fake.loadSIPDispatchRuleMutex.RLock() + defer fake.loadSIPDispatchRuleMutex.RUnlock() + argsForCall := fake.loadSIPDispatchRuleArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) LoadSIPDispatchRuleReturns(result1 *livekit.SIPDispatchRuleInfo, result2 error) { + fake.loadSIPDispatchRuleMutex.Lock() + defer fake.loadSIPDispatchRuleMutex.Unlock() + fake.LoadSIPDispatchRuleStub = nil + fake.loadSIPDispatchRuleReturns = struct { + result1 *livekit.SIPDispatchRuleInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPDispatchRuleReturnsOnCall(i int, result1 *livekit.SIPDispatchRuleInfo, result2 error) { + fake.loadSIPDispatchRuleMutex.Lock() + defer fake.loadSIPDispatchRuleMutex.Unlock() + fake.LoadSIPDispatchRuleStub = nil + if fake.loadSIPDispatchRuleReturnsOnCall == nil { + fake.loadSIPDispatchRuleReturnsOnCall = make(map[int]struct { + result1 *livekit.SIPDispatchRuleInfo + result2 error + }) + } + fake.loadSIPDispatchRuleReturnsOnCall[i] = struct { + result1 *livekit.SIPDispatchRuleInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPParticipant(arg1 context.Context, arg2 string) (*livekit.SIPParticipantInfo, error) { + fake.loadSIPParticipantMutex.Lock() + ret, specificReturn := fake.loadSIPParticipantReturnsOnCall[len(fake.loadSIPParticipantArgsForCall)] + fake.loadSIPParticipantArgsForCall = append(fake.loadSIPParticipantArgsForCall, struct { + arg1 context.Context + arg2 string + }{arg1, arg2}) + stub := fake.LoadSIPParticipantStub + fakeReturns := fake.loadSIPParticipantReturns + fake.recordInvocation("LoadSIPParticipant", []interface{}{arg1, arg2}) + fake.loadSIPParticipantMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) LoadSIPParticipantCallCount() int { + fake.loadSIPParticipantMutex.RLock() + defer fake.loadSIPParticipantMutex.RUnlock() + return len(fake.loadSIPParticipantArgsForCall) +} + +func (fake *FakeSIPStore) LoadSIPParticipantCalls(stub func(context.Context, string) (*livekit.SIPParticipantInfo, error)) { + fake.loadSIPParticipantMutex.Lock() + defer fake.loadSIPParticipantMutex.Unlock() + fake.LoadSIPParticipantStub = stub +} + +func (fake *FakeSIPStore) LoadSIPParticipantArgsForCall(i int) (context.Context, string) { + fake.loadSIPParticipantMutex.RLock() + defer fake.loadSIPParticipantMutex.RUnlock() + argsForCall := fake.loadSIPParticipantArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) LoadSIPParticipantReturns(result1 *livekit.SIPParticipantInfo, result2 error) { + fake.loadSIPParticipantMutex.Lock() + defer fake.loadSIPParticipantMutex.Unlock() + fake.LoadSIPParticipantStub = nil + fake.loadSIPParticipantReturns = struct { + result1 *livekit.SIPParticipantInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPParticipantReturnsOnCall(i int, result1 *livekit.SIPParticipantInfo, result2 error) { + fake.loadSIPParticipantMutex.Lock() + defer fake.loadSIPParticipantMutex.Unlock() + fake.LoadSIPParticipantStub = nil + if fake.loadSIPParticipantReturnsOnCall == nil { + fake.loadSIPParticipantReturnsOnCall = make(map[int]struct { + result1 *livekit.SIPParticipantInfo + result2 error + }) + } + fake.loadSIPParticipantReturnsOnCall[i] = struct { + result1 *livekit.SIPParticipantInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPTrunk(arg1 context.Context, arg2 string) (*livekit.SIPTrunkInfo, error) { + fake.loadSIPTrunkMutex.Lock() + ret, specificReturn := fake.loadSIPTrunkReturnsOnCall[len(fake.loadSIPTrunkArgsForCall)] + fake.loadSIPTrunkArgsForCall = append(fake.loadSIPTrunkArgsForCall, struct { + arg1 context.Context + arg2 string + }{arg1, arg2}) + stub := fake.LoadSIPTrunkStub + fakeReturns := fake.loadSIPTrunkReturns + fake.recordInvocation("LoadSIPTrunk", []interface{}{arg1, arg2}) + fake.loadSIPTrunkMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeSIPStore) LoadSIPTrunkCallCount() int { + fake.loadSIPTrunkMutex.RLock() + defer fake.loadSIPTrunkMutex.RUnlock() + return len(fake.loadSIPTrunkArgsForCall) +} + +func (fake *FakeSIPStore) LoadSIPTrunkCalls(stub func(context.Context, string) (*livekit.SIPTrunkInfo, error)) { + fake.loadSIPTrunkMutex.Lock() + defer fake.loadSIPTrunkMutex.Unlock() + fake.LoadSIPTrunkStub = stub +} + +func (fake *FakeSIPStore) LoadSIPTrunkArgsForCall(i int) (context.Context, string) { + fake.loadSIPTrunkMutex.RLock() + defer fake.loadSIPTrunkMutex.RUnlock() + argsForCall := fake.loadSIPTrunkArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) LoadSIPTrunkReturns(result1 *livekit.SIPTrunkInfo, result2 error) { + fake.loadSIPTrunkMutex.Lock() + defer fake.loadSIPTrunkMutex.Unlock() + fake.LoadSIPTrunkStub = nil + fake.loadSIPTrunkReturns = struct { + result1 *livekit.SIPTrunkInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) LoadSIPTrunkReturnsOnCall(i int, result1 *livekit.SIPTrunkInfo, result2 error) { + fake.loadSIPTrunkMutex.Lock() + defer fake.loadSIPTrunkMutex.Unlock() + fake.LoadSIPTrunkStub = nil + if fake.loadSIPTrunkReturnsOnCall == nil { + fake.loadSIPTrunkReturnsOnCall = make(map[int]struct { + result1 *livekit.SIPTrunkInfo + result2 error + }) + } + fake.loadSIPTrunkReturnsOnCall[i] = struct { + result1 *livekit.SIPTrunkInfo + result2 error + }{result1, result2} +} + +func (fake *FakeSIPStore) StoreSIPDispatchRule(arg1 context.Context, arg2 *livekit.SIPDispatchRuleInfo) error { + fake.storeSIPDispatchRuleMutex.Lock() + ret, specificReturn := fake.storeSIPDispatchRuleReturnsOnCall[len(fake.storeSIPDispatchRuleArgsForCall)] + fake.storeSIPDispatchRuleArgsForCall = append(fake.storeSIPDispatchRuleArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPDispatchRuleInfo + }{arg1, arg2}) + stub := fake.StoreSIPDispatchRuleStub + fakeReturns := fake.storeSIPDispatchRuleReturns + fake.recordInvocation("StoreSIPDispatchRule", []interface{}{arg1, arg2}) + fake.storeSIPDispatchRuleMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) StoreSIPDispatchRuleCallCount() int { + fake.storeSIPDispatchRuleMutex.RLock() + defer fake.storeSIPDispatchRuleMutex.RUnlock() + return len(fake.storeSIPDispatchRuleArgsForCall) +} + +func (fake *FakeSIPStore) StoreSIPDispatchRuleCalls(stub func(context.Context, *livekit.SIPDispatchRuleInfo) error) { + fake.storeSIPDispatchRuleMutex.Lock() + defer fake.storeSIPDispatchRuleMutex.Unlock() + fake.StoreSIPDispatchRuleStub = stub +} + +func (fake *FakeSIPStore) StoreSIPDispatchRuleArgsForCall(i int) (context.Context, *livekit.SIPDispatchRuleInfo) { + fake.storeSIPDispatchRuleMutex.RLock() + defer fake.storeSIPDispatchRuleMutex.RUnlock() + argsForCall := fake.storeSIPDispatchRuleArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) StoreSIPDispatchRuleReturns(result1 error) { + fake.storeSIPDispatchRuleMutex.Lock() + defer fake.storeSIPDispatchRuleMutex.Unlock() + fake.StoreSIPDispatchRuleStub = nil + fake.storeSIPDispatchRuleReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) StoreSIPDispatchRuleReturnsOnCall(i int, result1 error) { + fake.storeSIPDispatchRuleMutex.Lock() + defer fake.storeSIPDispatchRuleMutex.Unlock() + fake.StoreSIPDispatchRuleStub = nil + if fake.storeSIPDispatchRuleReturnsOnCall == nil { + fake.storeSIPDispatchRuleReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.storeSIPDispatchRuleReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) StoreSIPParticipant(arg1 context.Context, arg2 *livekit.SIPParticipantInfo) error { + fake.storeSIPParticipantMutex.Lock() + ret, specificReturn := fake.storeSIPParticipantReturnsOnCall[len(fake.storeSIPParticipantArgsForCall)] + fake.storeSIPParticipantArgsForCall = append(fake.storeSIPParticipantArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPParticipantInfo + }{arg1, arg2}) + stub := fake.StoreSIPParticipantStub + fakeReturns := fake.storeSIPParticipantReturns + fake.recordInvocation("StoreSIPParticipant", []interface{}{arg1, arg2}) + fake.storeSIPParticipantMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) StoreSIPParticipantCallCount() int { + fake.storeSIPParticipantMutex.RLock() + defer fake.storeSIPParticipantMutex.RUnlock() + return len(fake.storeSIPParticipantArgsForCall) +} + +func (fake *FakeSIPStore) StoreSIPParticipantCalls(stub func(context.Context, *livekit.SIPParticipantInfo) error) { + fake.storeSIPParticipantMutex.Lock() + defer fake.storeSIPParticipantMutex.Unlock() + fake.StoreSIPParticipantStub = stub +} + +func (fake *FakeSIPStore) StoreSIPParticipantArgsForCall(i int) (context.Context, *livekit.SIPParticipantInfo) { + fake.storeSIPParticipantMutex.RLock() + defer fake.storeSIPParticipantMutex.RUnlock() + argsForCall := fake.storeSIPParticipantArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) StoreSIPParticipantReturns(result1 error) { + fake.storeSIPParticipantMutex.Lock() + defer fake.storeSIPParticipantMutex.Unlock() + fake.StoreSIPParticipantStub = nil + fake.storeSIPParticipantReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) StoreSIPParticipantReturnsOnCall(i int, result1 error) { + fake.storeSIPParticipantMutex.Lock() + defer fake.storeSIPParticipantMutex.Unlock() + fake.StoreSIPParticipantStub = nil + if fake.storeSIPParticipantReturnsOnCall == nil { + fake.storeSIPParticipantReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.storeSIPParticipantReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) StoreSIPTrunk(arg1 context.Context, arg2 *livekit.SIPTrunkInfo) error { + fake.storeSIPTrunkMutex.Lock() + ret, specificReturn := fake.storeSIPTrunkReturnsOnCall[len(fake.storeSIPTrunkArgsForCall)] + fake.storeSIPTrunkArgsForCall = append(fake.storeSIPTrunkArgsForCall, struct { + arg1 context.Context + arg2 *livekit.SIPTrunkInfo + }{arg1, arg2}) + stub := fake.StoreSIPTrunkStub + fakeReturns := fake.storeSIPTrunkReturns + fake.recordInvocation("StoreSIPTrunk", []interface{}{arg1, arg2}) + fake.storeSIPTrunkMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSIPStore) StoreSIPTrunkCallCount() int { + fake.storeSIPTrunkMutex.RLock() + defer fake.storeSIPTrunkMutex.RUnlock() + return len(fake.storeSIPTrunkArgsForCall) +} + +func (fake *FakeSIPStore) StoreSIPTrunkCalls(stub func(context.Context, *livekit.SIPTrunkInfo) error) { + fake.storeSIPTrunkMutex.Lock() + defer fake.storeSIPTrunkMutex.Unlock() + fake.StoreSIPTrunkStub = stub +} + +func (fake *FakeSIPStore) StoreSIPTrunkArgsForCall(i int) (context.Context, *livekit.SIPTrunkInfo) { + fake.storeSIPTrunkMutex.RLock() + defer fake.storeSIPTrunkMutex.RUnlock() + argsForCall := fake.storeSIPTrunkArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeSIPStore) StoreSIPTrunkReturns(result1 error) { + fake.storeSIPTrunkMutex.Lock() + defer fake.storeSIPTrunkMutex.Unlock() + fake.StoreSIPTrunkStub = nil + fake.storeSIPTrunkReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) StoreSIPTrunkReturnsOnCall(i int, result1 error) { + fake.storeSIPTrunkMutex.Lock() + defer fake.storeSIPTrunkMutex.Unlock() + fake.StoreSIPTrunkStub = nil + if fake.storeSIPTrunkReturnsOnCall == nil { + fake.storeSIPTrunkReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.storeSIPTrunkReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSIPStore) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.deleteSIPDispatchRuleMutex.RLock() + defer fake.deleteSIPDispatchRuleMutex.RUnlock() + fake.deleteSIPParticipantMutex.RLock() + defer fake.deleteSIPParticipantMutex.RUnlock() + fake.deleteSIPTrunkMutex.RLock() + defer fake.deleteSIPTrunkMutex.RUnlock() + fake.listSIPDispatchRuleMutex.RLock() + defer fake.listSIPDispatchRuleMutex.RUnlock() + fake.listSIPParticipantMutex.RLock() + defer fake.listSIPParticipantMutex.RUnlock() + fake.listSIPTrunkMutex.RLock() + defer fake.listSIPTrunkMutex.RUnlock() + fake.loadSIPDispatchRuleMutex.RLock() + defer fake.loadSIPDispatchRuleMutex.RUnlock() + fake.loadSIPParticipantMutex.RLock() + defer fake.loadSIPParticipantMutex.RUnlock() + fake.loadSIPTrunkMutex.RLock() + defer fake.loadSIPTrunkMutex.RUnlock() + fake.storeSIPDispatchRuleMutex.RLock() + defer fake.storeSIPDispatchRuleMutex.RUnlock() + fake.storeSIPParticipantMutex.RLock() + defer fake.storeSIPParticipantMutex.RUnlock() + fake.storeSIPTrunkMutex.RLock() + defer fake.storeSIPTrunkMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeSIPStore) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ service.SIPStore = new(FakeSIPStore) diff --git a/pkg/service/twirp.go b/pkg/service/twirp.go index 50f3f2dc0..b7b31d764 100644 --- a/pkg/service/twirp.go +++ b/pkg/service/twirp.go @@ -25,13 +25,11 @@ import ( "github.com/twitchtv/twirp" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" - "github.com/livekit/protocol/logger" + "github.com/livekit/livekit-server/pkg/utils" ) -var ( - loggerKey = struct{}{} - statusReporterKey = struct{ a int }{42} -) +type twirpLoggerContext struct{} +type statusReporterKey struct{} type twirpRequestFields struct { service string @@ -41,11 +39,10 @@ type twirpRequestFields struct { // logging handling inspired by https://github.com/bakins/twirpzap // License: Apache-2.0 -func TwirpLogger(logger logger.Logger) *twirp.ServerHooks { +func TwirpLogger() *twirp.ServerHooks { loggerPool := &sync.Pool{ New: func() interface{} { return &requestLogger{ - logger: logger, fieldsOrig: make([]interface{}, 0, 30), } }, @@ -65,14 +62,13 @@ func TwirpLogger(logger logger.Logger) *twirp.ServerHooks { type requestLogger struct { twirpRequestFields - logger logger.Logger fieldsOrig []interface{} fields []interface{} startedAt time.Time } func AppendLogFields(ctx context.Context, fields ...interface{}) { - r, ok := ctx.Value(loggerKey).(*requestLogger) + r, ok := ctx.Value(twirpLoggerContext{}).(*requestLogger) if !ok || r == nil { return } @@ -91,13 +87,13 @@ func requestReceived(ctx context.Context, requestLoggerPool *sync.Pool) (context r.fields = append(r.fields, "service", svc) } - ctx = context.WithValue(ctx, loggerKey, r) + ctx = context.WithValue(ctx, twirpLoggerContext{}, r) return ctx, nil } func responseRouted(ctx context.Context) (context.Context, error) { if meth, ok := twirp.MethodName(ctx); ok { - l, ok := ctx.Value(loggerKey).(*requestLogger) + l, ok := ctx.Value(twirpLoggerContext{}).(*requestLogger) if !ok || l == nil { return ctx, nil } @@ -109,7 +105,7 @@ func responseRouted(ctx context.Context) (context.Context, error) { } func responseSent(ctx context.Context, requestLoggerPool *sync.Pool) { - r, ok := ctx.Value(loggerKey).(*requestLogger) + r, ok := ctx.Value(twirpLoggerContext{}).(*requestLogger) if !ok || r == nil { return } @@ -125,7 +121,7 @@ func responseSent(ctx context.Context, requestLoggerPool *sync.Pool) { } serviceMethod := "API " + r.service + "." + r.method - r.logger.Infow(serviceMethod, r.fields...) + utils.GetLogger(ctx).WithComponent(utils.ComponentAPI).Infow(serviceMethod, r.fields...) r.fields = r.fieldsOrig r.error = nil @@ -134,7 +130,7 @@ func responseSent(ctx context.Context, requestLoggerPool *sync.Pool) { } func errorReceived(ctx context.Context, e twirp.Error) context.Context { - r, ok := ctx.Value(loggerKey).(*requestLogger) + r, ok := ctx.Value(twirpLoggerContext{}).(*requestLogger) if !ok || r == nil { return ctx } @@ -160,13 +156,13 @@ func statusReporterRequestReceived(ctx context.Context) (context.Context, error) r.service = svc } - ctx = context.WithValue(ctx, statusReporterKey, r) + ctx = context.WithValue(ctx, statusReporterKey{}, r) return ctx, nil } func statusReporterResponseRouted(ctx context.Context) (context.Context, error) { if meth, ok := twirp.MethodName(ctx); ok { - l, ok := ctx.Value(statusReporterKey).(*twirpRequestFields) + l, ok := ctx.Value(statusReporterKey{}).(*twirpRequestFields) if !ok || l == nil { return ctx, nil } @@ -177,7 +173,7 @@ func statusReporterResponseRouted(ctx context.Context) (context.Context, error) } func statusReporterResponseSent(ctx context.Context) { - r, ok := ctx.Value(statusReporterKey).(*twirpRequestFields) + r, ok := ctx.Value(statusReporterKey{}).(*twirpRequestFields) if !ok || r == nil { return } @@ -205,7 +201,7 @@ func statusReporterResponseSent(ctx context.Context) { } func statusReporterErrorReceived(ctx context.Context, e twirp.Error) context.Context { - r, ok := ctx.Value(statusReporterKey).(*twirpRequestFields) + r, ok := ctx.Value(statusReporterKey{}).(*twirpRequestFields) if !ok || r == nil { return ctx } diff --git a/pkg/service/utils.go b/pkg/service/utils.go index 32076455f..2d1bafd4e 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -22,8 +22,11 @@ import ( "github.com/livekit/protocol/logger" ) -func handleError(w http.ResponseWriter, status int, err error, keysAndValues ...interface{}) { +func handleError(w http.ResponseWriter, r *http.Request, status int, err error, keysAndValues ...interface{}) { keysAndValues = append(keysAndValues, "status", status) + if r != nil && r.URL != nil { + keysAndValues = append(keysAndValues, "method", r.Method, "path", r.URL.Path) + } logger.GetLogger().WithCallDepth(1).Warnw("error handling request", err, keysAndValues...) w.WriteHeader(status) _, _ = w.Write([]byte(err.Error())) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 573b85649..3cf102a08 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -350,7 +350,10 @@ func (b *Buffer) Close() error { if b.rtpStats != nil { b.rtpStats.Stop() - b.logger.Infow("rtp stats", "direction", "upstream", "stats", b.rtpStats.ToString()) + b.logger.Debugw("rtp stats", + "direction", "upstream", + "stats", func() interface{} { return b.rtpStats.ToString() }, + ) if b.onFinalRtpStats != nil { b.onFinalRtpStats(b.rtpStats.ToProto()) } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 30bf71d20..70a8e8fed 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -984,10 +984,14 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.bindLock.Unlock() d.connectionStats.Close() d.rtpStats.Stop() - rtpStats := d.rtpStats.ToString() - if rtpStats != "" { - d.params.Logger.Infow("rtp stats", "direction", "downstream", "mime", d.mime, "ssrc", d.ssrc, "stats", rtpStats) - } + d.params.Logger.Debugw("rtp stats", + "direction", "downstream", + "mime", d.mime, + "ssrc", d.ssrc, + // evaluate only if log level matches + "stats", func() interface{} { + return d.rtpStats.ToString() + }) d.maxLayerNotifierChMu.Lock() d.maxLayerNotifierChClosed = true diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 9d65363a2..8fde6e3c7 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -1297,7 +1297,7 @@ func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { s.channelObserver = s.newChannelObserverProbe() s.channelObserver.SeedEstimate(s.lastReceivedEstimate) - s.params.Logger.Infow( + s.params.Logger.Debugw( "stream allocator: starting probe", "probeClusterId", probeClusterId, "current usage", expectedBandwidthUsage, diff --git a/pkg/utils/context.go b/pkg/utils/context.go index f5262e178..4ad9086d2 100644 --- a/pkg/utils/context.go +++ b/pkg/utils/context.go @@ -16,17 +16,33 @@ package utils -import "context" +import ( + "context" -var attemptKey = struct{}{} + "github.com/livekit/protocol/logger" +) + +type attemptKey struct{} +type loggerKey = struct{} func ContextWithAttempt(ctx context.Context, attempt int) context.Context { - return context.WithValue(ctx, attemptKey, attempt) + return context.WithValue(ctx, attemptKey{}, attempt) } func GetAttempt(ctx context.Context) int { - if attempt, ok := ctx.Value(attemptKey).(int); ok { + if attempt, ok := ctx.Value(attemptKey{}).(int); ok { return attempt } return 0 } + +func ContextWithLogger(ctx context.Context, logger logger.Logger) context.Context { + return context.WithValue(ctx, loggerKey{}, logger) +} + +func GetLogger(ctx context.Context) logger.Logger { + if l, ok := ctx.Value(loggerKey{}).(logger.Logger); ok { + return l + } + return logger.GetLogger() +} From 4db930bd296222c34fa7502cfed1187085313139 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 18:22:01 -0800 Subject: [PATCH 008/114] Update module github.com/urfave/cli/v2 to v2.26.0 (#2286) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4e163b610..d78d3dd96 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc - github.com/urfave/cli/v2 v2.25.7 + github.com/urfave/cli/v2 v2.26.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20231127185646-65229373498e diff --git a/go.sum b/go.sum index e60920f90..d0a6d00c1 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc h1:iT5lwxf894PiMq7cnMMQg/7VOD1pxmu//gQuHWAFy4s= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= +github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 37e1864df82254439c360e23a44fbba1ad8eb055 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 3 Dec 2023 10:03:41 -0800 Subject: [PATCH 009/114] Expose detailed connection info with ICEConnectionDetails (#2287) * Expose detailed connection info with ICEConnectionDetails * clone to avoid data race * lower transport * simplify * address feedback --- pkg/rtc/participant.go | 10 +- pkg/rtc/room.go | 100 +++++-- pkg/rtc/transport.go | 142 +++------- pkg/rtc/transportmanager.go | 13 +- pkg/rtc/types/ice.go | 254 ++++++++++++++++++ pkg/rtc/types/interfaces.go | 12 +- .../typesfakes/fake_local_participant.go | 145 +++++++--- 7 files changed, 481 insertions(+), 195 deletions(-) create mode 100644 pkg/rtc/types/ice.go diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3f82ec85b..fd2d251ca 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -358,10 +358,6 @@ func (p *ParticipantImpl) GetClientConfiguration() *livekit.ClientConfiguration return p.params.ClientConf } -func (p *ParticipantImpl) GetICEConnectionType() types.ICEConnectionType { - return p.TransportManager.GetICEConnectionType() -} - func (p *ParticipantImpl) GetBufferFactory() *buffer.Factory { return p.params.Config.BufferFactory } @@ -582,7 +578,7 @@ func (p *ParticipantImpl) OnClaimsChanged(callback func(types.LocalParticipant)) func (p *ParticipantImpl) HandleSignalSourceClose() { p.TransportManager.SetSignalSourceValid(false) - if !p.TransportManager.HasPublisherEverConnected() && !p.TransportManager.HasSubscriberEverConnected() { + if !p.HasConnected() { reason := types.ParticipantCloseReasonJoinFailed _ = p.Close(false, reason, false) } @@ -1708,6 +1704,10 @@ func (p *ParticipantImpl) GetPendingTrack(trackID livekit.TrackID) *livekit.Trac return nil } +func (p *ParticipantImpl) HasConnected() bool { + return p.TransportManager.HasSubscriberEverConnected() || p.TransportManager.HasPublisherEverConnected() +} + func (p *ParticipantImpl) sendTrackPublished(cid string, ti *livekit.TrackInfo) { p.pubLogger.Debugw("sending track published", "cid", cid, "trackInfo", logger.Proto(ti)) _ = p.writeMessage(&livekit.SignalResponse{ diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 09904506f..d832a61a3 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -17,9 +17,11 @@ package rtc import ( "context" "errors" + "fmt" "io" "math" "sort" + "strings" "sync" "time" @@ -325,11 +327,6 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me // it's important to set this before connection, we don't want to miss out on any published tracks participant.OnTrackPublished(r.onTrackPublished) participant.OnStateChange(func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) { - r.Logger.Infow("participant state changed", - "state", p.State(), - "participant", p.Identity(), - "pID", p.ID(), - "oldState", oldState) if r.onParticipantChanged != nil { r.onParticipantChanged(participant) } @@ -343,15 +340,24 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me // start the workers once connectivity is established p.Start() + meta := &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), + } + cds := participant.GetICEConnectionDetails() + for _, cd := range cds { + if cd.Type != types.ICEConnectionTypeUnknown { + meta.ConnectionType = string(cd.Type) + break + } + } r.telemetry.ParticipantActive(context.Background(), r.ToProto(), p.ToProto(), - &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), - ConnectionType: string(p.GetICEConnectionType()), - }, + meta, false, ) + + p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) } else if state == livekit.ParticipantInfo_DISCONNECTED { // remove participant from room go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) @@ -495,24 +501,27 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livekit.ParticipantID, reason types.ParticipantCloseReason) { r.lock.Lock() p, ok := r.participants[identity] - if ok { - if pID != "" && p.ID() != pID { - // participant session has been replaced - r.lock.Unlock() - return - } + if !ok { + r.lock.Unlock() + return + } - delete(r.participants, identity) - delete(r.participantOpts, identity) - delete(r.participantRequestSources, identity) - delete(r.hasPublished, identity) - if !p.Hidden() { - r.protoRoom.NumParticipants-- - } + if pID != "" && p.ID() != pID { + // participant session has been replaced + r.lock.Unlock() + return + } + + delete(r.participants, identity) + delete(r.participantOpts, identity) + delete(r.participantRequestSources, identity) + delete(r.hasPublished, identity) + if !p.Hidden() { + r.protoRoom.NumParticipants-- } immediateChange := false - if (p != nil && p.IsRecorder()) || r.protoRoom.ActiveRecording { + if p.IsRecorder() { activeRecording := false for _, op := range r.participants { if op.IsRecorder() { @@ -524,14 +533,16 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek if r.protoRoom.ActiveRecording != activeRecording { r.protoRoom.ActiveRecording = activeRecording immediateChange = true - } } r.lock.Unlock() r.protoProxy.MarkDirty(immediateChange) - if !ok { - return + if !p.HasConnected() { + fields := append(connectionDetailsFields(p.GetICEConnectionDetails()), + "reason", reason.String(), + ) + p.GetLogger().Infow("removing participant without connection", fields...) } // send broadcast only if it's not already closed @@ -551,7 +562,6 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek p.OnSubscribeStatusChanged(nil) // close participant as well - r.Logger.Debugw("closing participant for removal", "pID", p.ID(), "participant", p.Identity()) _ = p.Close(true, reason, false) r.leftAt.Store(time.Now().Unix()) @@ -1416,3 +1426,39 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp } }) } + +func connectionDetailsFields(cds []*types.ICEConnectionDetails) []interface{} { + var fields []interface{} + connectionType := types.ICEConnectionTypeUnknown + for _, cd := range cds { + candidates := make([]string, 0, len(cd.Remote)+len(cd.Local)) + for _, c := range cd.Local { + cStr := "[local]" + if c.Selected { + cStr += "[selected]" + } else if c.Filtered { + cStr += "[filtered]" + } + cStr += " " + c.Local.String() + candidates = append(candidates, cStr) + } + for _, c := range cd.Remote { + cStr := "[remote]" + if c.Selected { + cStr += "[selected]" + } else if c.Filtered { + cStr += "[filtered]" + } + cStr += " " + c.Remote.String() + candidates = append(candidates, cStr) + } + if len(candidates) > 0 { + fields = append(fields, fmt.Sprintf("%sCandidates", strings.ToLower(cd.Transport.String())), candidates) + } + if cd.Type != types.ICEConnectionTypeUnknown { + connectionType = cd.Type + } + } + fields = append(fields, "connectionType", connectionType) + return fields +} diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 4230b9dfd..dede7cdf9 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -23,7 +23,6 @@ import ( "github.com/bep/debounce" "github.com/pion/dtls/v2/pkg/crypto/elliptic" - "github.com/pion/ice/v2" "github.com/pion/interceptor" "github.com/pion/interceptor/pkg/cc" "github.com/pion/interceptor/pkg/gcc" @@ -35,13 +34,6 @@ import ( "github.com/pkg/errors" "go.uber.org/atomic" - sutils "github.com/livekit/livekit-server/pkg/utils" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/logger/pionlogger" - lksdp "github.com/livekit/protocol/sdp" - "github.com/livekit/protocol/utils" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/pacer" @@ -49,6 +41,11 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" + lksdp "github.com/livekit/protocol/sdp" ) const ( @@ -94,7 +91,6 @@ const ( signalICEGatheringComplete signal = iota signalLocalICECandidate signalRemoteICECandidate - signalLogICECandidates signalSendOffer signalRemoteDescriptionReceived signalICERestart @@ -108,8 +104,6 @@ func (s signal) String() string { return "LOCAL_ICE_CANDIDATE" case signalRemoteICECandidate: return "REMOTE_ICE_CANDIDATE" - case signalLogICECandidates: - return "LOG_ICE_CANDIDATES" case signalSendOffer: return "SEND_OFFER" case signalRemoteDescriptionReceived: @@ -234,11 +228,7 @@ type PCTransport struct { currentOfferIceCredential string // ice user:pwd, for publish side ice restart checking pendingRestartIceOffer *webrtc.SessionDescription - // for cleaner logging - allowedLocalCandidates *utils.DedupedSlice[string] - allowedRemoteCandidates *utils.DedupedSlice[string] - filteredLocalCandidates *utils.DedupedSlice[string] - filteredRemoteCandidates *utils.DedupedSlice[string] + connectionDetails *types.ICEConnectionDetails } type TransportParams struct { @@ -251,6 +241,7 @@ type TransportParams struct { Telemetry telemetry.TelemetryService EnabledCodecs []*livekit.Codec Logger logger.Logger + Transport livekit.SignalTarget SimTracks map[uint32]SimulcastTrackInfo ClientInfo ClientInfo IsOfferer bool @@ -393,10 +384,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { eventCh: make(chan event, 100), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, - allowedLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), - allowedRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), - filteredLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), - filteredRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), + connectionDetails: types.NewICEConnectionDetails(params.Transport, params.Logger), } if params.IsSendSide { t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ @@ -564,12 +552,6 @@ func (t *PCTransport) getSelectedPair() (*webrtc.ICECandidatePair, error) { return iceTransport.GetSelectedCandidatePair() } -func (t *PCTransport) logICECandidates() { - t.postEvent(event{ - signal: signalLogICECandidates, - }) -} - func (t *PCTransport) setConnectedAt(at time.Time) bool { t.lock.Lock() t.connectedAt = at @@ -629,6 +611,14 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState switch state { case webrtc.ICEConnectionStateConnected: t.setICEConnectedAt(time.Now()) + go func() { + pair, err := t.getSelectedPair() + if err != nil { + t.params.Logger.Warnw("failed to get selected candidate pair", err) + return + } + t.connectionDetails.SetSelectedPair(pair) + }() case webrtc.ICEConnectionStateChecking: t.setICEStartedAt(time.Now()) @@ -905,6 +895,10 @@ func (t *PCTransport) HasEverConnected() bool { return !t.firstConnectedAt.IsZero() } +func (t *PCTransport) GetICEConnectionDetails() *types.ICEConnectionDetails { + return t.connectionDetails +} + func (t *PCTransport) WriteRTCP(pkts []rtcp.Packet) error { return t.pc.WriteRTCP(pkts) } @@ -1198,44 +1192,6 @@ func (t *PCTransport) SetChannelCapacityOfStreamAllocator(channelCapacity int64) t.streamAllocator.SetChannelCapacity(channelCapacity) } -func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { - unknown := types.ICEConnectionTypeUnknown - if t.pc == nil { - return unknown - } - p, err := t.getSelectedPair() - if err != nil || p == nil { - return unknown - } - - if p.Remote.Typ == webrtc.ICECandidateTypeRelay { - return types.ICEConnectionTypeTURN - } else if p.Remote.Typ == webrtc.ICECandidateTypePrflx { - // if the remote relay candidate pings us *before* we get a relay candidate, - // Pion would have created a prflx candidate with the same address as the relay candidate. - // to report an accurate connection type, we'll compare to see if existing relay candidates match - t.lock.RLock() - allowedRemoteCandidates := t.allowedRemoteCandidates.Get() - t.lock.RUnlock() - - for _, ci := range allowedRemoteCandidates { - candidateValue := strings.TrimPrefix(ci, "candidate:") - candidate, err := ice.UnmarshalCandidate(candidateValue) - if err == nil && candidate.Type() == ice.CandidateTypeRelay { - if p.Remote.Address == candidate.Address() && - p.Remote.Port == uint16(candidate.Port()) && - p.Remote.Protocol.String() == candidate.NetworkType().NetworkShort() { - return types.ICEConnectionTypeTURN - } - } - } - } - if p.Remote.Protocol == webrtc.ICEProtocolTCP { - return types.ICEConnectionTypeTCP - } - return types.ICEConnectionTypeUDP -} - func (t *PCTransport) preparePC(previousAnswer webrtc.SessionDescription) error { // sticky data channel to first m-lines, if someday we don't send sdp without media streams to // client's subscribe pc after joining, should change this step @@ -1453,7 +1409,6 @@ func (t *PCTransport) processEvents() { t.clearSignalStateCheckTimer() t.params.Logger.Debugw("leaving events processor") - t.handleLogICECandidates(nil) } func (t *PCTransport) handleEvent(e *event) error { @@ -1464,8 +1419,6 @@ func (t *PCTransport) handleEvent(e *event) error { return t.handleLocalICECandidate(e) case signalRemoteICECandidate: return t.handleRemoteICECandidate(e) - case signalLogICECandidates: - return t.handleLogICECandidates(e) case signalSendOffer: return t.handleSendOffer(e) case signalRemoteDescriptionReceived: @@ -1537,33 +1490,28 @@ func (t *PCTransport) localDescriptionSent() error { func (t *PCTransport) clearLocalDescriptionSent() { t.cacheLocalCandidates = true t.cachedLocalCandidates = nil - - t.allowedLocalCandidates.Clear() - t.lock.Lock() - t.allowedRemoteCandidates.Clear() - t.lock.Unlock() - t.filteredLocalCandidates.Clear() - t.filteredRemoteCandidates.Clear() + t.connectionDetails.Clear() } func (t *PCTransport) handleLocalICECandidate(e *event) error { c := e.data.(*webrtc.ICECandidate) filtered := false - if t.preferTCP.Load() && c != nil && c.Protocol != webrtc.ICEProtocolTCP { - cstr := c.String() - t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) - t.filteredLocalCandidates.Add(cstr) - filtered = true + if c != nil { + if t.preferTCP.Load() && c.Protocol != webrtc.ICEProtocolTCP { + t.params.Logger.Debugw("filtering out local candidate", + "candidate", func() interface{} { + return c.String() + }) + filtered = true + } + t.connectionDetails.AddLocalCandidate(c, filtered) } if filtered { return nil } - if c != nil { - t.allowedLocalCandidates.Add(c.String()) - } if t.cacheLocalCandidates { t.cachedLocalCandidates = append(t.cachedLocalCandidates, c) return nil @@ -1582,18 +1530,14 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { filtered := false if t.preferTCP.Load() && !strings.Contains(c.Candidate, "tcp") { t.params.Logger.Debugw("filtering out remote candidate", "candidate", c.Candidate) - t.filteredRemoteCandidates.Add(c.Candidate) filtered = true } + t.connectionDetails.AddRemoteCandidate(*c, filtered) if filtered { return nil } - t.lock.Lock() - t.allowedRemoteCandidates.Add(c.Candidate) - t.lock.Unlock() - if t.pc.RemoteDescription() == nil { t.pendingRemoteCandidates = append(t.pendingRemoteCandidates, c) return nil @@ -1606,30 +1550,6 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { return nil } -func (t *PCTransport) handleLogICECandidates(_ *event) error { - lc := t.allowedLocalCandidates.Get() - rc := t.allowedRemoteCandidates.Get() - var fields []interface{} - if len(lc) != 0 || len(rc) != 0 { - fields = append(fields, - "lc", lc, - "rc", rc, - "lc_filtered", t.filteredLocalCandidates.Get(), - "rc_filtered", t.filteredRemoteCandidates.Get(), - ) - - } - if pair, err := t.getSelectedPair(); err == nil { - fields = append(fields, "selected_pair", pair) - } - - if len(fields) > 0 { - t.params.Logger.Infow("ice candidates", fields...) - } - - return nil -} - func (t *PCTransport) setNegotiationState(state NegotiationState) { t.negotiationState = state if onNegotiationStateChanged := t.getOnNegotiationStateChanged(); onNegotiationStateChanged != nil { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 6805bd430..21591f3c2 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -124,6 +124,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER), SimTracks: params.SimTracks, ClientInfo: params.ClientInfo, + Transport: livekit.SignalTarget_PUBLISHER, }) if err != nil { return nil, err @@ -159,6 +160,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro IsSendSide: true, AllowPlayoutDelay: params.AllowPlayoutDelay, DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount, + Transport: livekit.SignalTarget_SUBSCRIBER, }) if err != nil { return nil, err @@ -554,8 +556,15 @@ func (t *TransportManager) SubscriberAsPrimary() bool { return t.params.SubscriberAsPrimary } -func (t *TransportManager) GetICEConnectionType() types.ICEConnectionType { - return t.getTransport(true).GetICEConnectionType() +func (t *TransportManager) GetICEConnectionDetails() []*types.ICEConnectionDetails { + details := make([]*types.ICEConnectionDetails, 0, 2) + for _, pc := range []*PCTransport{t.publisher, t.subscriber} { + cd := pc.GetICEConnectionDetails() + if cd.HasCandidates() { + details = append(details, cd.Clone()) + } + } + return details } func (t *TransportManager) getTransport(isPrimary bool) *PCTransport { diff --git a/pkg/rtc/types/ice.go b/pkg/rtc/types/ice.go new file mode 100644 index 000000000..531e1ad43 --- /dev/null +++ b/pkg/rtc/types/ice.go @@ -0,0 +1,254 @@ +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "strings" + "sync" + + "github.com/pion/ice/v2" + "github.com/pion/webrtc/v3" + "golang.org/x/exp/slices" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +type ICEConnectionType string + +const ( + ICEConnectionTypeUDP ICEConnectionType = "udp" + ICEConnectionTypeTCP ICEConnectionType = "tcp" + ICEConnectionTypeTURN ICEConnectionType = "turn" + ICEConnectionTypeUnknown ICEConnectionType = "unknown" +) + +type ICECandidateExtended struct { + // only one of local or remote is set. This is due to type foo in Pion + Local *webrtc.ICECandidate + Remote ice.Candidate + Selected bool + Filtered bool +} + +type ICEConnectionDetails struct { + Local []*ICECandidateExtended + Remote []*ICECandidateExtended + Transport livekit.SignalTarget + Type ICEConnectionType + lock sync.Mutex + logger logger.Logger +} + +func NewICEConnectionDetails(transport livekit.SignalTarget, l logger.Logger) *ICEConnectionDetails { + d := &ICEConnectionDetails{ + Transport: transport, + Type: ICEConnectionTypeUnknown, + logger: l, + } + return d +} + +func (d *ICEConnectionDetails) HasCandidates() bool { + d.lock.Lock() + defer d.lock.Unlock() + return len(d.Local) > 0 || len(d.Remote) > 0 +} + +// Clone returns a copy of the ICEConnectionDetails, where fields can be read without locking +func (d *ICEConnectionDetails) Clone() *ICEConnectionDetails { + d.lock.Lock() + defer d.lock.Unlock() + clone := &ICEConnectionDetails{ + Transport: d.Transport, + Type: d.Type, + logger: d.logger, + Local: make([]*ICECandidateExtended, 0, len(d.Local)), + Remote: make([]*ICECandidateExtended, 0, len(d.Remote)), + } + for _, c := range d.Local { + clone.Local = append(clone.Local, &ICECandidateExtended{ + Local: c.Local, + Filtered: c.Filtered, + }) + } + for _, c := range d.Remote { + clone.Remote = append(clone.Remote, &ICECandidateExtended{ + Remote: c.Remote, + Filtered: c.Filtered, + }) + } + return clone +} + +func (d *ICEConnectionDetails) AddLocalCandidate(c *webrtc.ICECandidate, filtered bool) { + d.lock.Lock() + defer d.lock.Unlock() + compFn := func(e *ICECandidateExtended) bool { + return isCandidateEqualTo(e.Local, c) + } + if slices.ContainsFunc[[]*ICECandidateExtended, *ICECandidateExtended](d.Local, compFn) { + return + } + d.Local = append(d.Local, &ICECandidateExtended{ + Local: c, + Filtered: filtered, + }) +} + +func (d *ICEConnectionDetails) AddRemoteCandidate(c webrtc.ICECandidateInit, filtered bool) { + candidate, err := unmarshalICECandidate(c) + if err != nil { + d.logger.Errorw("could not unmarshal candidate", err, "candidate", c) + return + } + + d.lock.Lock() + defer d.lock.Unlock() + compFn := func(e *ICECandidateExtended) bool { + return isICECandidateEqualTo(e.Remote, candidate) + } + if slices.ContainsFunc[[]*ICECandidateExtended, *ICECandidateExtended](d.Remote, compFn) { + return + } + d.Remote = append(d.Remote, &ICECandidateExtended{ + Remote: candidate, + Filtered: filtered, + }) +} + +func (d *ICEConnectionDetails) Clear() { + d.lock.Lock() + defer d.lock.Unlock() + d.Local = nil + d.Remote = nil + d.Type = ICEConnectionTypeUnknown +} + +func (d *ICEConnectionDetails) SetSelectedPair(pair *webrtc.ICECandidatePair) { + d.lock.Lock() + defer d.lock.Unlock() + remoteIdx := slices.IndexFunc[[]*ICECandidateExtended, *ICECandidateExtended](d.Remote, func(e *ICECandidateExtended) bool { + return isICECandidateEqualToCandidate(e.Remote, pair.Remote) + }) + if remoteIdx < 0 { + // it's possible for prflx candidates to be generated by Pion, we'll add them + candidate, err := unmarshalICECandidate(pair.Remote.ToJSON()) + if err != nil { + d.logger.Errorw("could not unmarshal remote candidate", err, "candidate", pair.Remote) + return + } + d.Remote = append(d.Remote, &ICECandidateExtended{ + Remote: candidate, + Filtered: false, + }) + remoteIdx = len(d.Remote) - 1 + } + remote := d.Remote[remoteIdx] + remote.Selected = true + + localIdx := slices.IndexFunc[[]*ICECandidateExtended, *ICECandidateExtended](d.Local, func(e *ICECandidateExtended) bool { + return isCandidateEqualTo(e.Local, pair.Local) + }) + if localIdx < 0 { + d.logger.Errorw("could not match local candidate", nil, "local", pair.Local) + // should not happen + return + } + local := d.Local[localIdx] + local.Selected = true + + d.Type = ICEConnectionTypeUDP + if pair.Remote.Protocol == webrtc.ICEProtocolTCP { + d.Type = ICEConnectionTypeTCP + } + if pair.Remote.Typ == webrtc.ICECandidateTypeRelay { + d.Type = ICEConnectionTypeTURN + } else if pair.Remote.Typ == webrtc.ICECandidateTypePrflx { + // if the remote relay candidate pings us *before* we get a relay candidate, + // Pion would have created a prflx candidate with the same address as the relay candidate. + // to report an accurate connection type, we'll compare to see if existing relay candidates match + for _, other := range d.Remote { + or := other.Remote + if or.Type() == ice.CandidateTypeRelay && + pair.Remote.Address == or.Address() && + pair.Remote.Port == uint16(or.Port()) && + pair.Remote.Protocol.String() == or.NetworkType().NetworkShort() { + d.Type = ICEConnectionTypeTURN + } + } + } +} + +func isCandidateEqualTo(c1, c2 *webrtc.ICECandidate) bool { + if c1 == nil && c2 == nil { + return true + } + if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) { + return false + } + return c1.Typ == c2.Typ && + c1.Protocol == c2.Protocol && + c1.Address == c2.Address && + c1.Port == c2.Port && + c1.Component == c2.Component && + c1.Foundation == c2.Foundation && + c1.Priority == c2.Priority && + c1.RelatedAddress == c2.RelatedAddress && + c1.RelatedPort == c2.RelatedPort && + c1.TCPType == c2.TCPType +} + +func isICECandidateEqualTo(c1, c2 ice.Candidate) bool { + if c1 == nil && c2 == nil { + return true + } + if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) { + return false + } + return c1.Type() == c2.Type() && + c1.NetworkType() == c2.NetworkType() && + c1.Address() == c2.Address() && + c1.Port() == c2.Port() && + c1.Component() == c2.Component() && + c1.Foundation() == c2.Foundation() && + c1.Priority() == c2.Priority() && + c1.RelatedAddress().Equal(c2.RelatedAddress()) && + c1.TCPType() == c2.TCPType() +} + +func isICECandidateEqualToCandidate(c1 ice.Candidate, c2 *webrtc.ICECandidate) bool { + if c1 == nil && c2 == nil { + return true + } + if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) { + return false + } + return c1.Type().String() == c2.Typ.String() && + c1.NetworkType().NetworkShort() == c2.Protocol.String() && + c1.Address() == c2.Address && + c1.Port() == int(c2.Port) && + c1.Component() == c2.Component && + c1.Foundation() == c2.Foundation && + c1.Priority() == c2.Priority && + c1.TCPType().String() == c2.TCPType +} + +func unmarshalICECandidate(c webrtc.ICECandidateInit) (ice.Candidate, error) { + candidateValue := strings.TrimPrefix(c.Candidate, "candidate:") + return ice.UnmarshalCandidate(candidateValue) +} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 266e24b8b..77e395d85 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -285,15 +285,6 @@ type Participant interface { // ------------------------------------------------------- -type ICEConnectionType string - -const ( - ICEConnectionTypeUDP ICEConnectionType = "udp" - ICEConnectionTypeTCP ICEConnectionType = "tcp" - ICEConnectionTypeTURN ICEConnectionType = "turn" - ICEConnectionTypeUnknown ICEConnectionType = "unknown" -) - type AddTrackParams struct { Stereo bool Red bool @@ -320,10 +311,11 @@ type LocalParticipant interface { SubscriberAsPrimary() bool GetClientInfo() *livekit.ClientInfo GetClientConfiguration() *livekit.ClientConfiguration - GetICEConnectionType() ICEConnectionType GetBufferFactory() *buffer.Factory GetPlayoutDelayConfig() *livekit.PlayoutDelay GetPendingTrack(trackID livekit.TrackID) *livekit.TrackInfo + GetICEConnectionDetails() []*ICEConnectionDetails + HasConnected() bool SetResponseSink(sink routing.MessageSink) CloseSignalConnection(reason SignallingCloseReason) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index a01a3834b..0cd75616a 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -233,15 +233,15 @@ type FakeLocalParticipant struct { getConnectionQualityReturnsOnCall map[int]struct { result1 *livekit.ConnectionQualityInfo } - GetICEConnectionTypeStub func() types.ICEConnectionType - getICEConnectionTypeMutex sync.RWMutex - getICEConnectionTypeArgsForCall []struct { + GetICEConnectionDetailsStub func() []*types.ICEConnectionDetails + getICEConnectionDetailsMutex sync.RWMutex + getICEConnectionDetailsArgsForCall []struct { } - getICEConnectionTypeReturns struct { - result1 types.ICEConnectionType + getICEConnectionDetailsReturns struct { + result1 []*types.ICEConnectionDetails } - getICEConnectionTypeReturnsOnCall map[int]struct { - result1 types.ICEConnectionType + getICEConnectionDetailsReturnsOnCall map[int]struct { + result1 []*types.ICEConnectionDetails } GetLoggerStub func() logger.Logger getLoggerMutex sync.RWMutex @@ -371,6 +371,16 @@ type FakeLocalParticipant struct { handleSignalSourceCloseMutex sync.RWMutex handleSignalSourceCloseArgsForCall []struct { } + HasConnectedStub func() bool + hasConnectedMutex sync.RWMutex + hasConnectedArgsForCall []struct { + } + hasConnectedReturns struct { + result1 bool + } + hasConnectedReturnsOnCall map[int]struct { + result1 bool + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -2062,15 +2072,15 @@ func (fake *FakeLocalParticipant) GetConnectionQualityReturnsOnCall(i int, resul }{result1} } -func (fake *FakeLocalParticipant) GetICEConnectionType() types.ICEConnectionType { - fake.getICEConnectionTypeMutex.Lock() - ret, specificReturn := fake.getICEConnectionTypeReturnsOnCall[len(fake.getICEConnectionTypeArgsForCall)] - fake.getICEConnectionTypeArgsForCall = append(fake.getICEConnectionTypeArgsForCall, struct { +func (fake *FakeLocalParticipant) GetICEConnectionDetails() []*types.ICEConnectionDetails { + fake.getICEConnectionDetailsMutex.Lock() + ret, specificReturn := fake.getICEConnectionDetailsReturnsOnCall[len(fake.getICEConnectionDetailsArgsForCall)] + fake.getICEConnectionDetailsArgsForCall = append(fake.getICEConnectionDetailsArgsForCall, struct { }{}) - stub := fake.GetICEConnectionTypeStub - fakeReturns := fake.getICEConnectionTypeReturns - fake.recordInvocation("GetICEConnectionType", []interface{}{}) - fake.getICEConnectionTypeMutex.Unlock() + stub := fake.GetICEConnectionDetailsStub + fakeReturns := fake.getICEConnectionDetailsReturns + fake.recordInvocation("GetICEConnectionDetails", []interface{}{}) + fake.getICEConnectionDetailsMutex.Unlock() if stub != nil { return stub() } @@ -2080,38 +2090,38 @@ func (fake *FakeLocalParticipant) GetICEConnectionType() types.ICEConnectionType return fakeReturns.result1 } -func (fake *FakeLocalParticipant) GetICEConnectionTypeCallCount() int { - fake.getICEConnectionTypeMutex.RLock() - defer fake.getICEConnectionTypeMutex.RUnlock() - return len(fake.getICEConnectionTypeArgsForCall) +func (fake *FakeLocalParticipant) GetICEConnectionDetailsCallCount() int { + fake.getICEConnectionDetailsMutex.RLock() + defer fake.getICEConnectionDetailsMutex.RUnlock() + return len(fake.getICEConnectionDetailsArgsForCall) } -func (fake *FakeLocalParticipant) GetICEConnectionTypeCalls(stub func() types.ICEConnectionType) { - fake.getICEConnectionTypeMutex.Lock() - defer fake.getICEConnectionTypeMutex.Unlock() - fake.GetICEConnectionTypeStub = stub +func (fake *FakeLocalParticipant) GetICEConnectionDetailsCalls(stub func() []*types.ICEConnectionDetails) { + fake.getICEConnectionDetailsMutex.Lock() + defer fake.getICEConnectionDetailsMutex.Unlock() + fake.GetICEConnectionDetailsStub = stub } -func (fake *FakeLocalParticipant) GetICEConnectionTypeReturns(result1 types.ICEConnectionType) { - fake.getICEConnectionTypeMutex.Lock() - defer fake.getICEConnectionTypeMutex.Unlock() - fake.GetICEConnectionTypeStub = nil - fake.getICEConnectionTypeReturns = struct { - result1 types.ICEConnectionType +func (fake *FakeLocalParticipant) GetICEConnectionDetailsReturns(result1 []*types.ICEConnectionDetails) { + fake.getICEConnectionDetailsMutex.Lock() + defer fake.getICEConnectionDetailsMutex.Unlock() + fake.GetICEConnectionDetailsStub = nil + fake.getICEConnectionDetailsReturns = struct { + result1 []*types.ICEConnectionDetails }{result1} } -func (fake *FakeLocalParticipant) GetICEConnectionTypeReturnsOnCall(i int, result1 types.ICEConnectionType) { - fake.getICEConnectionTypeMutex.Lock() - defer fake.getICEConnectionTypeMutex.Unlock() - fake.GetICEConnectionTypeStub = nil - if fake.getICEConnectionTypeReturnsOnCall == nil { - fake.getICEConnectionTypeReturnsOnCall = make(map[int]struct { - result1 types.ICEConnectionType +func (fake *FakeLocalParticipant) GetICEConnectionDetailsReturnsOnCall(i int, result1 []*types.ICEConnectionDetails) { + fake.getICEConnectionDetailsMutex.Lock() + defer fake.getICEConnectionDetailsMutex.Unlock() + fake.GetICEConnectionDetailsStub = nil + if fake.getICEConnectionDetailsReturnsOnCall == nil { + fake.getICEConnectionDetailsReturnsOnCall = make(map[int]struct { + result1 []*types.ICEConnectionDetails }) } - fake.getICEConnectionTypeReturnsOnCall[i] = struct { - result1 types.ICEConnectionType + fake.getICEConnectionDetailsReturnsOnCall[i] = struct { + result1 []*types.ICEConnectionDetails }{result1} } @@ -2811,6 +2821,59 @@ func (fake *FakeLocalParticipant) HandleSignalSourceCloseCalls(stub func()) { fake.HandleSignalSourceCloseStub = stub } +func (fake *FakeLocalParticipant) HasConnected() bool { + fake.hasConnectedMutex.Lock() + ret, specificReturn := fake.hasConnectedReturnsOnCall[len(fake.hasConnectedArgsForCall)] + fake.hasConnectedArgsForCall = append(fake.hasConnectedArgsForCall, struct { + }{}) + stub := fake.HasConnectedStub + fakeReturns := fake.hasConnectedReturns + fake.recordInvocation("HasConnected", []interface{}{}) + fake.hasConnectedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HasConnectedCallCount() int { + fake.hasConnectedMutex.RLock() + defer fake.hasConnectedMutex.RUnlock() + return len(fake.hasConnectedArgsForCall) +} + +func (fake *FakeLocalParticipant) HasConnectedCalls(stub func() bool) { + fake.hasConnectedMutex.Lock() + defer fake.hasConnectedMutex.Unlock() + fake.HasConnectedStub = stub +} + +func (fake *FakeLocalParticipant) HasConnectedReturns(result1 bool) { + fake.hasConnectedMutex.Lock() + defer fake.hasConnectedMutex.Unlock() + fake.HasConnectedStub = nil + fake.hasConnectedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) HasConnectedReturnsOnCall(i int, result1 bool) { + fake.hasConnectedMutex.Lock() + defer fake.hasConnectedMutex.Unlock() + fake.HasConnectedStub = nil + if fake.hasConnectedReturnsOnCall == nil { + fake.hasConnectedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.hasConnectedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] @@ -6156,8 +6219,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getClientInfoMutex.RUnlock() fake.getConnectionQualityMutex.RLock() defer fake.getConnectionQualityMutex.RUnlock() - fake.getICEConnectionTypeMutex.RLock() - defer fake.getICEConnectionTypeMutex.RUnlock() + fake.getICEConnectionDetailsMutex.RLock() + defer fake.getICEConnectionDetailsMutex.RUnlock() fake.getLoggerMutex.RLock() defer fake.getLoggerMutex.RUnlock() fake.getPacerMutex.RLock() @@ -6186,6 +6249,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.handleReconnectAndSendResponseMutex.RUnlock() fake.handleSignalSourceCloseMutex.RLock() defer fake.handleSignalSourceCloseMutex.RUnlock() + fake.hasConnectedMutex.RLock() + defer fake.hasConnectedMutex.RUnlock() fake.hasPermissionMutex.RLock() defer fake.hasPermissionMutex.RUnlock() fake.hiddenMutex.RLock() From 98c81b92bbd85d5e9e04567837c64b4f234d74b2 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 3 Dec 2023 10:48:33 -0800 Subject: [PATCH 010/114] Helper function to remove address from ClientInfo (#2288) --- .../staticconfiguration.go | 5 ++- pkg/rtc/participant.go | 1 - pkg/utils/protocol.go | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 pkg/utils/protocol.go diff --git a/pkg/clientconfiguration/staticconfiguration.go b/pkg/clientconfiguration/staticconfiguration.go index e0309c26c..4fb67bae4 100644 --- a/pkg/clientconfiguration/staticconfiguration.go +++ b/pkg/clientconfiguration/staticconfiguration.go @@ -17,6 +17,7 @@ package clientconfiguration import ( "google.golang.org/protobuf/proto" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -40,7 +41,9 @@ func (s *StaticClientConfigurationManager) GetConfiguration(clientInfo *livekit. for _, c := range s.confs { matched, err := c.Match.Match(clientInfo) if err != nil { - logger.Errorw("matchrule failed", err, "clientInfo", logger.Proto(clientInfo)) + logger.Errorw("matchrule failed", err, + "clientInfo", logger.Proto(utils.ClientInfoWithoutAddress(clientInfo)), + ) continue } if !matched { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index fd2d251ca..af4540a7f 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -742,7 +742,6 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea "sendLeave", sendLeave, "reason", reason.String(), "isExpectedToResume", isExpectedToResume, - "clientInfo", logger.Proto(p.params.ClientInfo), ) p.clearDisconnectTimer() p.clearMigrationTimer() diff --git a/pkg/utils/protocol.go b/pkg/utils/protocol.go new file mode 100644 index 000000000..23844742f --- /dev/null +++ b/pkg/utils/protocol.go @@ -0,0 +1,32 @@ +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "google.golang.org/protobuf/proto" + + "github.com/livekit/protocol/livekit" +) + +func ClientInfoWithoutAddress(c *livekit.ClientInfo) *livekit.ClientInfo { + if c == nil { + return nil + } + clone := proto.Clone(c).(*livekit.ClientInfo) + clone.Address = "" + return clone +} From 02c28a594633186ea8bcdc65e58c95d9925289a4 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 3 Dec 2023 22:05:23 -0800 Subject: [PATCH 011/114] Fix Selected attribute not being copied (#2289) --- pkg/rtc/types/ice.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/rtc/types/ice.go b/pkg/rtc/types/ice.go index 531e1ad43..dd16f01e4 100644 --- a/pkg/rtc/types/ice.go +++ b/pkg/rtc/types/ice.go @@ -84,12 +84,14 @@ func (d *ICEConnectionDetails) Clone() *ICEConnectionDetails { clone.Local = append(clone.Local, &ICECandidateExtended{ Local: c.Local, Filtered: c.Filtered, + Selected: c.Selected, }) } for _, c := range d.Remote { clone.Remote = append(clone.Remote, &ICECandidateExtended{ Remote: c.Remote, Filtered: c.Filtered, + Selected: c.Selected, }) } return clone From 1240c1670be02757d708e082db8cfa20c89e581c Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Mon, 4 Dec 2023 17:21:21 +0200 Subject: [PATCH 012/114] SIP: Properly handle nil trunk. (#2291) --- pkg/service/ioservice_sip.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index 076bff512..ba2286d8d 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -61,6 +61,9 @@ func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc. if err != nil { return nil, err } + if trunk == nil { + return &rpc.GetSIPTrunkAuthenticationResponse{}, nil + } return &rpc.GetSIPTrunkAuthenticationResponse{ Username: trunk.InboundUsername, Password: trunk.InboundPassword, From ec2efa2dc4bc0e1ca3b3efb7441a9c6bcf82e7fc Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Mon, 4 Dec 2023 20:03:20 +0200 Subject: [PATCH 013/114] SIP: Better debug logging. (#2293) --- pkg/service/ioservice_sip.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index ba2286d8d..540b906b5 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -18,6 +18,7 @@ import ( "context" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/sip" ) @@ -49,10 +50,16 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E if err != nil { return nil, err } + if trunk != nil { + logger.Debugw("SIP trunk matched", "trunkID", trunk.SipTrunkId, "called", req.CalledNumber, "calling", req.CallingNumber) + } else { + logger.Debugw("No SIP trunk matched", "trunkID", "", "called", req.CalledNumber, "calling", req.CallingNumber) + } best, err := s.matchSIPDispatchRule(ctx, trunk, req) if err != nil { return nil, err } + logger.Debugw("SIP dispatch rule matched", "dispatchRule", best.SipDispatchRuleId, "called", req.CalledNumber, "calling", req.CallingNumber) return sip.EvaluateDispatchRule(best, req) } @@ -62,8 +69,10 @@ func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc. return nil, err } if trunk == nil { + logger.Debugw("No SIP trunk matched for auth", "trunkID", "", "called", req.To, "calling", req.From) return &rpc.GetSIPTrunkAuthenticationResponse{}, nil } + logger.Debugw("SIP trunk matched for auth", "trunkID", trunk.SipTrunkId, "called", req.To, "calling", req.From) return &rpc.GetSIPTrunkAuthenticationResponse{ Username: trunk.InboundUsername, Password: trunk.InboundPassword, From d4c4bc1100ad2067dc2eb8b96ff4bf90e5edd8aa Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 4 Dec 2023 19:11:55 -0800 Subject: [PATCH 014/114] fix signal response delivery after session start failure (#2294) * fix signal response delivery after session start failure * tidy --- pkg/service/signal.go | 5 +- pkg/service/signal_test.go | 137 +++++++++++++++++++++++++------------ 2 files changed, 95 insertions(+), 47 deletions(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 4c9e085be..a181987e9 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -148,6 +148,7 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe "connID", ss.ConnectionId, ) + stream.Hijack() sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ Logger: l, Stream: stream, @@ -176,11 +177,9 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { + sink.Close() l.Errorw("could not handle new participant", err) - return } - - stream.Hijack() return } diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 3e202636e..83bf391e9 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -16,6 +16,7 @@ package service import ( "context" + "errors" "testing" "time" @@ -35,7 +36,6 @@ func init() { } func TestSignal(t *testing.T) { - bus := psrpc.NewLocalMessageBus() cfg := config.SignalRelayConfig{ Enabled: false, RetryTimeout: 30 * time.Second, @@ -44,56 +44,105 @@ func TestSignal(t *testing.T) { StreamBufferSize: 1000, } - reqMessageIn := &livekit.SignalRequest{ - Message: &livekit.SignalRequest_Ping{Ping: 123}, - } - resMessageIn := &livekit.SignalResponse{ - Message: &livekit.SignalResponse_Pong{Pong: 321}, - } + t.Run("messages are delivered", func(t *testing.T) { + bus := psrpc.NewLocalMessageBus() - var reqMessageOut proto.Message - var resErr error - done := make(chan struct{}) + reqMessageIn := &livekit.SignalRequest{ + Message: &livekit.SignalRequest_Ping{Ping: 123}, + } + resMessageIn := &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Pong{Pong: 321}, + } - client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) - require.NoError(t, err) + var reqMessageOut proto.Message + var resErr error + done := make(chan struct{}) - server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( - ctx context.Context, - roomName livekit.RoomName, - pi routing.ParticipantInit, - connectionID livekit.ConnectionID, - requestSource routing.MessageSource, - responseSink routing.MessageSink, - ) error { - go func() { - reqMessageOut = <-requestSource.ReadChan() - resErr = responseSink.WriteMessage(resMessageIn) - responseSink.Close() - close(done) - }() - return nil + client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) + require.NoError(t, err) + + server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + go func() { + reqMessageOut = <-requestSource.ReadChan() + resErr = responseSink.WriteMessage(resMessageIn) + responseSink.Close() + close(done) + }() + return nil + }) + require.NoError(t, err) + + err = server.Start() + require.NoError(t, err) + + _, reqSink, resSource, err := client.StartParticipantSignal( + context.Background(), + livekit.RoomName("room1"), + routing.ParticipantInit{}, + livekit.NodeID("node1"), + ) + require.NoError(t, err) + + err = reqSink.WriteMessage(reqMessageIn) + require.NoError(t, err) + + <-done + require.True(t, proto.Equal(reqMessageIn, reqMessageOut), "req message should match %s %s", protojson.Format(reqMessageIn), protojson.Format(reqMessageOut)) + require.NoError(t, resErr) + + resMessageOut := <-resSource.ReadChan() + require.True(t, proto.Equal(resMessageIn, resMessageOut), "res message should match %s %s", protojson.Format(resMessageIn), protojson.Format(resMessageOut)) }) - require.NoError(t, err) - err = server.Start() - require.NoError(t, err) + t.Run("messages are delivered when session handler fails", func(t *testing.T) { + bus := psrpc.NewLocalMessageBus() - _, reqSink, resSource, err := client.StartParticipantSignal( - context.Background(), - livekit.RoomName("room1"), - routing.ParticipantInit{}, - livekit.NodeID("node1"), - ) - require.NoError(t, err) + resMessageIn := &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Pong{Pong: 321}, + } - err = reqSink.WriteMessage(reqMessageIn) - require.NoError(t, err) + var resErr error + done := make(chan struct{}) - <-done - require.True(t, proto.Equal(reqMessageIn, reqMessageOut), "req message should match %s %s", protojson.Format(reqMessageIn), protojson.Format(reqMessageOut)) - require.NoError(t, resErr) + client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) + require.NoError(t, err) - resMessageOut := <-resSource.ReadChan() - require.True(t, proto.Equal(resMessageIn, resMessageOut), "res message should match %s %s", protojson.Format(resMessageIn), protojson.Format(resMessageOut)) + server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + defer close(done) + resErr = responseSink.WriteMessage(resMessageIn) + return errors.New("start session failed") + }) + require.NoError(t, err) + + err = server.Start() + require.NoError(t, err) + + _, _, resSource, err := client.StartParticipantSignal( + context.Background(), + livekit.RoomName("room1"), + routing.ParticipantInit{}, + livekit.NodeID("node1"), + ) + require.NoError(t, err) + + <-done + require.NoError(t, resErr) + + resMessageOut := <-resSource.ReadChan() + require.True(t, proto.Equal(resMessageIn, resMessageOut), "res message should match %s %s", protojson.Format(resMessageIn), protojson.Format(resMessageOut)) + }) } From e1cc9d6b3c75e40902d63ae36f28d4f7f1a21007 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 6 Dec 2023 00:08:48 +0800 Subject: [PATCH 015/114] Fix log marshal error (#2295) --- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/buffer/rtpstats_receiver.go | 2 +- pkg/sfu/buffer/rtpstats_receiver_test.go | 5 +++-- pkg/sfu/downtrack.go | 5 ++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 3cf102a08..fcf5ff7b5 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -352,7 +352,7 @@ func (b *Buffer) Close() error { b.rtpStats.Stop() b.logger.Debugw("rtp stats", "direction", "upstream", - "stats", func() interface{} { return b.rtpStats.ToString() }, + "stats", b.rtpStats, ) if b.onFinalRtpStats != nil { b.onFinalRtpStats(b.rtpStats.ToProto()) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index b621f42b1..53a963fcb 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -402,7 +402,7 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo { return r.deltaInfo(snapshotID, r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()) } -func (r *RTPStatsReceiver) ToString() string { +func (r *RTPStatsReceiver) String() string { r.lock.RLock() defer r.lock.RUnlock() diff --git a/pkg/sfu/buffer/rtpstats_receiver_test.go b/pkg/sfu/buffer/rtpstats_receiver_test.go index 7a39c1b54..1816b6689 100644 --- a/pkg/sfu/buffer/rtpstats_receiver_test.go +++ b/pkg/sfu/buffer/rtpstats_receiver_test.go @@ -20,9 +20,10 @@ import ( "testing" "time" - "github.com/livekit/protocol/logger" "github.com/pion/rtp" "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/logger" ) func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet { @@ -82,7 +83,7 @@ func Test_RTPStatsReceiver(t *testing.T) { } r.Stop() - fmt.Printf("%s\n", r.ToString()) + fmt.Printf("%s\n", r.String()) } func Test_RTPStatsReceiver_Update(t *testing.T) { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 70a8e8fed..ee1f9fa57 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -989,9 +989,8 @@ func (d *DownTrack) CloseWithFlush(flush bool) { "mime", d.mime, "ssrc", d.ssrc, // evaluate only if log level matches - "stats", func() interface{} { - return d.rtpStats.ToString() - }) + "stats", d.rtpStats, + ) d.maxLayerNotifierChMu.Lock() d.maxLayerNotifierChClosed = true From 83efa9258e8401f890b1edb289aa3d16dfd65866 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 6 Dec 2023 16:59:05 +0530 Subject: [PATCH 016/114] Bump up protocol for connection quality LOST. (#2297) Also log trackID/trackInfo in layer mapping. --- pkg/rtc/participant.go | 4 ++++ pkg/rtc/types/protocol_version.go | 6 +++++- pkg/sfu/buffer/videolayerutils.go | 18 +++++++++--------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index af4540a7f..cd84440a8 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -990,6 +990,10 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo } p.lock.Unlock() + if minQuality == livekit.ConnectionQuality_LOST && !p.ProtocolVersion().SupportsConnectionQualityLost() { + minQuality = livekit.ConnectionQuality_POOR + } + return &livekit.ConnectionQualityInfo{ ParticipantSid: string(p.ID()), Quality: minQuality, diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 9c4e47cc3..6dde9a976 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -16,7 +16,7 @@ package types type ProtocolVersion int -const CurrentProtocol = 10 +const CurrentProtocol = 11 func (v ProtocolVersion) SupportsPackedStreamId() bool { return v > 0 @@ -75,3 +75,7 @@ func (v ProtocolVersion) SupportHandlesDisconnectedUpdate() bool { func (v ProtocolVersion) SupportSyncStreamID() bool { return v > 9 } + +func (v ProtocolVersion) SupportsConnectionQualityLost() bool { + return v > 10 +} diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 9028283fc..895af9f18 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -36,7 +36,7 @@ func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQua if layer.Quality <= livekit.VideoQuality_HIGH { layerPresence[layer.Quality] = true } else { - logger.Warnw("unexpected quality in track info", nil, "trackInfo", logger.Proto(trackInfo)) + logger.Warnw("unexpected quality in track info", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) } } @@ -97,13 +97,13 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { return 2 case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]: - logger.Warnw("unexpected rid f with only two qualities, low and medium", nil) + logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return 1 case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, low and high", nil) + logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return 1 case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, medium and high", nil) + logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return 1 default: @@ -169,13 +169,13 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { return FullResolution case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]: - logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil) + logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return HalfResolution case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil) + logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return HalfResolution case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil) + logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) return HalfResolution default: @@ -240,7 +240,7 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek return livekit.VideoQuality_HIGH default: - logger.Errorw("invalid layer", nil, "layer", layer, "trackInfo", trackInfo) + logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo)) return livekit.VideoQuality_HIGH } @@ -250,7 +250,7 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek return livekit.VideoQuality_HIGH default: - logger.Errorw("invalid layer", nil, "layer", layer, "trackInfo", trackInfo) + logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo)) return livekit.VideoQuality_HIGH } } From 1f335dd564821aa6e8a1efdfa80086773bf8262d Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 6 Dec 2023 18:11:05 +0530 Subject: [PATCH 017/114] Convert to formatter string for lazy evaluation. (#2298) --- pkg/sfu/buffer/rtpstats_sender.go | 2 +- pkg/sfu/downtrack.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 1cb0ff1bd..6dc952e12 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -807,7 +807,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo } } -func (r *RTPStatsSender) ToString() string { +func (r *RTPStatsSender) String() string { r.lock.RLock() defer r.lock.RUnlock() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index ee1f9fa57..a2ef60926 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -133,7 +133,7 @@ type DownTrackState struct { func (d DownTrackState) String() string { return fmt.Sprintf("DownTrackState{rtpStats: %s, deltaSender: %d, forwarder: %s}", - d.RTPStats.ToString(), d.DeltaStatsSenderSnapshotId, d.ForwarderState.String()) + d.RTPStats, d.DeltaStatsSenderSnapshotId, d.ForwarderState.String()) } // ------------------------------------------------------------------- From 1bff4f387d44691812538765e03c467c1c326fea Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 6 Dec 2023 11:17:59 -0800 Subject: [PATCH 018/114] Update io service (#2300) * update io service * consistency --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/service/ioservice.go | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d78d3dd96..1f1656a10 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb - github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f + github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876 github.com/livekit/psrpc v0.5.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -89,7 +89,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect diff --git a/go.sum b/go.sum index d0a6d00c1..d80252a17 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb h1:KiGg4k+kYQD9NjKixaSDMMeYOO2//XBM4IROTI1Itjo= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f h1:6XPC53t/XEcfIe8BUwKkeFmgTLKPfF77JmQ7nydqoOs= -github.com/livekit/protocol v1.9.4-0.20231202181655-afa0350bcd0f/go.mod h1:8f342d5nvfNp9YAEfJokSR+zbNFpaivgU0h6vwaYhes= +github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876 h1:NnbpPgxDHOcSdgW0JzBkc4QzzLVAe4sOaiYqUUH0/K4= +github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876/go.mod h1:SzrmeWw8sbf99laJJNMwp+5izlvh/ynlMbVOX0JUoes= github.com/livekit/psrpc v0.5.2 h1:+MvG8Otm/J6MTg2MP/uuMbrkxOWsrj2hDhu/I1VIU1U= github.com/livekit/psrpc v0.5.2/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -233,8 +233,8 @@ github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLq github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= diff --git a/pkg/service/ioservice.go b/pkg/service/ioservice.go index 70158d5cc..62d2820ff 100644 --- a/pkg/service/ioservice.go +++ b/pkg/service/ioservice.go @@ -148,6 +148,15 @@ func (s *IOInfoService) ListEgress(ctx context.Context, req *livekit.ListEgressR return &livekit.ListEgressResponse{Items: items}, nil } +func (s *IOInfoService) UpdateMetrics(ctx context.Context, req *rpc.UpdateMetricsRequest) (*emptypb.Empty, error) { + logger.Infow("received egress metrics", + "egressID", req.Info.EgressId, + "avgCpu", req.AvgCpuUsage, + "maxCpu", req.MaxCpuUsage, + ) + return &emptypb.Empty{}, nil +} + func (s *IOInfoService) GetIngressInfo(ctx context.Context, req *rpc.GetIngressInfoRequest) (*rpc.GetIngressInfoResponse, error) { info, err := s.loadIngressFromInfoRequest(req) if err != nil { From 176f9a854cfa3ee207af09791d1150f348253d5d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:49:09 -0800 Subject: [PATCH 019/114] Update golang.org/x/exp digest to f3f8817 (#2301) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1f1656a10..ce88878bf 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.26.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20231127185646-65229373498e + golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb golang.org/x/sync v0.5.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index d80252a17..339db97d0 100644 --- a/go.sum +++ b/go.sum @@ -293,8 +293,8 @@ golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIi golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= -golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From f4acb6ceac3e270f4b8bc0039aaa1f10de3862d7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:49:47 -0800 Subject: [PATCH 020/114] Update actions/setup-go action to v5 (#2299) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/buildtest.yaml | 2 +- .github/workflows/docker.yaml | 2 +- .github/workflows/release.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 78aa75721..7fb5b2842 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -33,7 +33,7 @@ jobs: - run: redis-cli ping - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: "1.20" diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 58574e162..967b1f734 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -39,7 +39,7 @@ jobs: type=semver,pattern=v{{major}}.{{minor}} - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '>=1.18' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cd3b523fc..2f37c7a87 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -33,7 +33,7 @@ jobs: run: git fetch --force --tags - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '>=1.18' From b0b7489b0e1be36f035655826d2b52736ee020b3 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 7 Dec 2023 12:53:17 +0800 Subject: [PATCH 021/114] Fix svc freeze caused by key frame id wrap around (#2302) * Fix svc freeze caused by key frame id wrap around * rename --- .../dependencydescriptor.go | 24 +++++- .../videolayerselector/framenumberwrapper.go | 41 ++++++++++ .../framenumberwrapper_test.go | 82 +++++++++++++++++++ 3 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 pkg/sfu/videolayerselector/framenumberwrapper.go create mode 100644 pkg/sfu/videolayerselector/framenumberwrapper_test.go diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 9adc54b87..6bb2fb7ae 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -39,12 +39,14 @@ type DependencyDescriptor struct { decodeTargetsLock sync.RWMutex decodeTargets []*DecodeTarget + fnWrapper FrameNumberWrapper } func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { return &DependencyDescriptor{ Base: NewBase(logger), decisions: NewSelectorDecisionCache(256, 80), + fnWrapper: FrameNumberWrapper{logger: logger}, } } @@ -52,6 +54,7 @@ func NewDependencyDescriptorFromNull(vls VideoLayerSelector) *DependencyDescript return &DependencyDescriptor{ Base: vls.(*Null).Base, decisions: NewSelectorDecisionCache(256, 80), + fnWrapper: FrameNumberWrapper{logger: vls.(*Null).logger}, } } @@ -291,13 +294,26 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r Descriptor: dd, Structure: d.structure, } + + unWrapFn := uint16(d.fnWrapper.UpdateAndGet(extFrameNum, ddwdt.StructureUpdated)) + var ddClone *dede.DependencyDescriptor + if unWrapFn != dd.FrameNumber { + clone := *dd + ddClone = &clone + ddClone.FrameNumber = unWrapFn + ddExtension.Descriptor = ddClone + } + if dd.AttachedStructure == nil { if d.activeDecodeTargetsBitmask != nil { - // clone and override activebitmask - // DD-TODO: if the packet that contains the bitmask is acknowledged by RR, then we don't need it until it changed. - ddClone := *ddExtension.Descriptor + if ddClone == nil { + // clone and override activebitmask + // DD-TODO: if the packet that contains the bitmask is acknowledged by RR, then we don't need it until it changed. + clone := *dd + ddClone = &clone + ddExtension.Descriptor = ddClone + } ddClone.ActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask - ddExtension.Descriptor = &ddClone // d.logger.Debugw("set active decode targets bitmask", "activeDecodeTargetsBitmask", d.activeDecodeTargetsBitmask) } } diff --git a/pkg/sfu/videolayerselector/framenumberwrapper.go b/pkg/sfu/videolayerselector/framenumberwrapper.go new file mode 100644 index 000000000..7942485e0 --- /dev/null +++ b/pkg/sfu/videolayerselector/framenumberwrapper.go @@ -0,0 +1,41 @@ +package videolayerselector + +import "github.com/livekit/protocol/logger" + +type FrameNumberWrapper struct { + offset uint64 + last uint64 + inited bool + logger logger.Logger +} + +// UpdateAndGet returns the wrapped frame number from the given frame number, and updates the offset to +// make sure the returned frame number is always inorder. Should only updateOffset if the new frame is a keyframe +// because frame dependencies uses on the frame number diff so frames inside a GOP should have the same offset. +func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64 { + if !f.inited { + f.last = new + f.inited = true + return new + } + + if new <= f.last { + return new + f.offset + } + + if updateOffset { + new16 := uint16(new + f.offset) + last16 := uint16(f.last + f.offset) + // if new frame number wraps around and is considered as earlier by client, increase offset to make it later + if diff := new16 - last16; diff > 0x8000 || (diff == 0x8000 && new16 <= last16) { + // increase offset by 6000, nearly 10 seconds for 30fps video with 3 spatial layers + prevOffset := f.offset + f.offset += uint64(65535 - diff + 6000) + + // TODO: remove this + f.logger.Infow("wrap around frame number seen, update offset", "new", new, "last", f.last, "offset", f.offset, "prevOffset", prevOffset, "lastWrapFn", last16, "newWrapFn", new16) + } + } + f.last = new + return new + f.offset +} diff --git a/pkg/sfu/videolayerselector/framenumberwrapper_test.go b/pkg/sfu/videolayerselector/framenumberwrapper_test.go new file mode 100644 index 000000000..361a7de4c --- /dev/null +++ b/pkg/sfu/videolayerselector/framenumberwrapper_test.go @@ -0,0 +1,82 @@ +package videolayerselector + +import ( + "testing" + + "math/rand" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/protocol/logger" +) + +func TestFrameNumberWrapper(t *testing.T) { + + logger.InitFromConfig(&logger.Config{Level: "debug"}, t.Name()) + + fnWrap := &FrameNumberWrapper{logger: logger.GetLogger()} + + fnWrapAround := utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}) + + firstF := uint16(1000) + + testFrameOrder := func(frame uint16, isKeyFrame bool, frame2 uint16, isKeyFrame2, expectInorder bool) { + frameUnwrap := fnWrapAround.Update(frame).ExtendedVal + wrappedFrame := uint16(fnWrap.UpdateAndGet(frameUnwrap, isKeyFrame)) + + // make sure wrap around always get in order frame number + fnWrapAround.Update(frame + (frame2-frame)/2) + + frame2Unwrap := fnWrapAround.Update(frame2).ExtendedVal + wrappedFrame2 := uint16(fnWrap.UpdateAndGet(frame2Unwrap, isKeyFrame2)) + // keeps order + require.Equal(t, expectInorder, inOrder(wrappedFrame2, wrappedFrame), "frame %d, frame2 %d, wrappedFrame %d, wrapped Frame2 %d, frameUnwrap %d, frame2Unwrap %d", frame, frame2, wrappedFrame, wrappedFrame2, frameUnwrap, frame2Unwrap) + // frame number diff should be the same if frame2 is not a key frame + if !isKeyFrame2 { + require.Equal(t, frame2-frame, wrappedFrame2-wrappedFrame) + } + } + + secondF := getFrame(firstF, true) + testFrameOrder(firstF, true, secondF, false, true) + + // non key frame keeps diff and order + for i := 0; i < 100; i++ { + // frame in order + firstF = secondF + secondF = getFrame(firstF, true) + testFrameOrder(firstF, false, secondF, false, true) + + // frame out of order + firstF = secondF + secondF = getFrame(firstF, false) + testFrameOrder(firstF, false, secondF, false, false) + + // key frame in order + firstF = secondF + secondF = getFrame(firstF, true) + testFrameOrder(firstF, false, secondF, true, true) + + // frame in order + firstF = secondF + secondF = getFrame(firstF, true) + testFrameOrder(firstF, false, secondF, false, true) + + // key frame out of order but should be in order after wrap around + firstF = secondF + secondF = getFrame(firstF, false) + testFrameOrder(firstF, false, secondF, true, true) + } +} + +func inOrder(a, b uint16) bool { + return a-b < 0x8000 || (a-b == 0x8000 && a > b) +} + +func getFrame(base uint16, inorder bool) uint16 { + if inorder { + return base + uint16(rand.Intn(0x8000)) + } + return base + uint16(rand.Intn(0x8000)) + 0x8000 +} From c766676d36e6a4dac090b348500facfca350acd2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 10 Dec 2023 21:44:16 +0530 Subject: [PATCH 022/114] Handle nil pair (#2305) --- pkg/rtc/transport.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index dede7cdf9..528d14b02 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -549,7 +549,16 @@ func (t *PCTransport) getSelectedPair() (*webrtc.ICECandidatePair, error) { return nil, errors.New("no ICE transport") } - return iceTransport.GetSelectedCandidatePair() + pair, err := iceTransport.GetSelectedCandidatePair() + if err != nil { + return nil, err + } + + if pair == nil { + return nil, errors.New("no selected pair") + } + + return pair, err } func (t *PCTransport) setConnectedAt(at time.Time) bool { From dfcafff955d6b51796d0611192e57e5f83fccde7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 11 Dec 2023 11:20:25 +0530 Subject: [PATCH 023/114] Log track info when media published. (#2306) With pending track added moved to Debugw, will be good to have this when track is published. --- pkg/rtc/participant.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index cd84440a8..7339e0d2e 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1341,6 +1341,7 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w "rid", track.RID(), "SSRC", track.SSRC(), "mime", track.Codec().MimeType, + "trackInfo", logger.Proto(publishedTrack.ToProto()), ) if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { From 23b46042ccc32603f2800e5f3874f78157389a4c Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 13 Dec 2023 15:32:25 +0100 Subject: [PATCH 024/114] Populate disconnect updates with participant identity (#2310) --- pkg/rtc/participant_signal.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 863fd67d6..adaf123ce 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -212,9 +212,10 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { break } else if info.state == livekit.ParticipantInfo_DISCONNECTED { disconnectedParticipants = append(disconnectedParticipants, &livekit.ParticipantInfo{ - Sid: string(keys[i]), - Version: info.version, - State: livekit.ParticipantInfo_DISCONNECTED, + Sid: string(keys[i]), + Identity: string(p.Identity()), + Version: info.version, + State: livekit.ParticipantInfo_DISCONNECTED, }) } } From 0478af449f64257eb96d36232602870c9d3e2246 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 14 Dec 2023 15:22:57 +0530 Subject: [PATCH 025/114] Do not error on end-of-candidates candidate (#2314) --- pkg/rtc/types/ice.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/types/ice.go b/pkg/rtc/types/ice.go index dd16f01e4..f628c4a30 100644 --- a/pkg/rtc/types/ice.go +++ b/pkg/rtc/types/ice.go @@ -118,17 +118,21 @@ func (d *ICEConnectionDetails) AddRemoteCandidate(c webrtc.ICECandidateInit, fil d.logger.Errorw("could not unmarshal candidate", err, "candidate", c) return } + if candidate == nil { + // end-of-candidates candidate + return + } d.lock.Lock() defer d.lock.Unlock() compFn := func(e *ICECandidateExtended) bool { - return isICECandidateEqualTo(e.Remote, candidate) + return isICECandidateEqualTo(e.Remote, *candidate) } if slices.ContainsFunc[[]*ICECandidateExtended, *ICECandidateExtended](d.Remote, compFn) { return } d.Remote = append(d.Remote, &ICECandidateExtended{ - Remote: candidate, + Remote: *candidate, Filtered: filtered, }) } @@ -154,8 +158,11 @@ func (d *ICEConnectionDetails) SetSelectedPair(pair *webrtc.ICECandidatePair) { d.logger.Errorw("could not unmarshal remote candidate", err, "candidate", pair.Remote) return } + if candidate == nil { + return + } d.Remote = append(d.Remote, &ICECandidateExtended{ - Remote: candidate, + Remote: *candidate, Filtered: false, }) remoteIdx = len(d.Remote) - 1 @@ -250,7 +257,16 @@ func isICECandidateEqualToCandidate(c1 ice.Candidate, c2 *webrtc.ICECandidate) b c1.TCPType().String() == c2.TCPType } -func unmarshalICECandidate(c webrtc.ICECandidateInit) (ice.Candidate, error) { +func unmarshalICECandidate(c webrtc.ICECandidateInit) (*ice.Candidate, error) { candidateValue := strings.TrimPrefix(c.Candidate, "candidate:") - return ice.UnmarshalCandidate(candidateValue) + if candidateValue == "" { + return nil, nil + } + + candidate, err := ice.UnmarshalCandidate(candidateValue) + if err != nil { + return nil, err + } + + return &candidate, nil } From a150eaf697cb6c9fc0a26c9222d59ac3403775e4 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 15 Dec 2023 00:02:27 +0800 Subject: [PATCH 026/114] Fix mid info lost when migrating multi-codec simulcast track (#2315) * Fix mid info lost when migrating multi-codec simulcast track * update pion --- go.mod | 8 ++++---- go.sum | 18 +++++++++++++----- pkg/rtc/mediatrackreceiver.go | 14 ++++++++++++-- pkg/rtc/participant.go | 3 ++- .../videolayerselector/dependencydescriptor.go | 1 - 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index ce88878bf..a04bd06d6 100644 --- a/go.mod +++ b/go.mod @@ -28,13 +28,13 @@ require ( github.com/pion/dtls/v2 v2.2.8 github.com/pion/ice/v2 v2.3.11 github.com/pion/interceptor v0.1.25 - github.com/pion/rtcp v1.2.12 + github.com/pion/rtcp v1.2.13 github.com/pion/rtp v1.8.3 github.com/pion/sctp v1.8.9 github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.4 - github.com/pion/webrtc/v3 v3.2.23 + github.com/pion/webrtc/v3 v3.2.24 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/redis/go-redis/v9 v9.3.0 @@ -65,7 +65,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/subcommands v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -82,7 +82,7 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/logging v0.2.2 // indirect - github.com/pion/mdns v0.0.8 // indirect + github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/srtp/v2 v2.0.18 // indirect github.com/pion/stun v0.6.1 // indirect diff --git a/go.sum b/go.sum index 339db97d0..92cd38878 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,9 @@ github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3 github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -189,13 +190,15 @@ github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDm github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.8 h1:HhicWIg7OX5PVilyBO6plhMetInbzkVJAhbdJiAeVaI= github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= +github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= +github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtcp v1.2.13 h1:+EQijuisKwm/8VBs8nWllr0bIndR7Lf7cZG200mpbNo= +github.com/pion/rtcp v1.2.13/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= @@ -221,8 +224,8 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8= github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.23 h1:GbqEuxBbVLFhXk0GwxKAoaIJYiEa9TyoZPEZC+2HZxM= -github.com/pion/webrtc/v3 v3.2.23/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs= +github.com/pion/webrtc/v3 v3.2.24 h1:MiFL5DMo2bDaaIFWr0DDpwiV/L4EGbLZb+xoRvfEo1Y= +github.com/pion/webrtc/v3 v3.2.24/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -291,6 +294,7 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= @@ -327,6 +331,7 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -378,6 +383,7 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -389,6 +395,7 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -399,6 +406,7 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 20883e19a..b97ffbed1 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -212,7 +212,7 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority t.shadowReceiversLocked() onSetupReceiver := t.onSetupReceiver - t.params.Logger.Debugw("setup receiver", "mime", receiver.Codec().MimeType, "priority", priority, "receivers", t.receiversShadow) + t.params.Logger.Debugw("setup receiver", "mime", receiver.Codec().MimeType, "priority", priority, "receivers", t.receiversShadow, "mid", mid) t.lock.Unlock() if onSetupReceiver != nil { @@ -553,8 +553,18 @@ func (t *MediaTrackReceiver) RevokeDisallowedSubscribers(allowedSubscriberIdenti } func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + clonedInfo := proto.Clone(ti).(*livekit.TrackInfo) t.lock.Lock() - t.trackInfo = proto.Clone(ti).(*livekit.TrackInfo) + originInfo := t.trackInfo + for _, ci := range clonedInfo.Codecs { + for _, originCi := range originInfo.Codecs { + if strings.EqualFold(ci.MimeType, originCi.MimeType) && originCi.Mid != "" { + ci.Mid = originCi.Mid + break + } + } + } + t.trackInfo = clonedInfo t.lock.Unlock() if ti != nil && t.Kind() == livekit.TrackType_VIDEO { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 7339e0d2e..ded6610e4 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1777,6 +1777,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei p.pendingTracksLock.Lock() newTrack := false + mid := p.TransportManager.GetPublisherMid(rtpReceiver) p.pubLogger.Debugw( "media track received", "kind", track.Kind().String(), @@ -1784,8 +1785,8 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei "rid", track.RID(), "SSRC", track.SSRC(), "mime", track.Codec().MimeType, + "mid", mid, ) - mid := p.TransportManager.GetPublisherMid(rtpReceiver) if mid == "" { p.pendingTracksLock.Unlock() p.pubLogger.Warnw("could not get mid for track", nil, "trackID", track.ID()) diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 6bb2fb7ae..8347114bc 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -426,7 +426,6 @@ func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) { defer d.decodeTargetsLock.RUnlock() for _, dt := range d.decodeTargets { if dt.Active() && dt.Layer.Spatial == layer && dt.Valid() { - d.logger.Debugw(fmt.Sprintf("checking sync, matching decode target, layer: %d, dt: %s, dts: %+v", layer, dt, d.decodeTargets)) return true, layer } } From 05f310fef4218c95da6e992e20924e1d153273ab Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:07:58 -0800 Subject: [PATCH 027/114] Update golang.org/x/exp digest to aacd6d4 (#2316) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a04bd06d6..baad85cb5 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.26.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb + golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 golang.org/x/sync v0.5.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 92cd38878..a190f3139 100644 --- a/go.sum +++ b/go.sum @@ -297,8 +297,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= -golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From 3cf4fbc6a9907f15c3c396143c176f985e35a587 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 15 Dec 2023 15:40:10 +0530 Subject: [PATCH 028/114] Store identity in participant update cache. (#2320) Need to store identity of other partiicpant in cache so that it can be sent with the disconnected participant update. Side note: Feels like the cache can be made to hold the full proto to make things simpler, but just adding a field for now. --- pkg/rtc/participant.go | 3 ++- pkg/rtc/participant_signal.go | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ded6610e4..28863a2fa 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -76,13 +76,14 @@ type downTrackState struct { // --------------------------------------------------------------- type participantUpdateInfo struct { + identity livekit.ParticipantIdentity version uint32 state livekit.ParticipantInfo_State updatedAt time.Time } func (p participantUpdateInfo) String() string { - return fmt.Sprintf("version: %d, state: %s, updatedAt: %s", p.version, p.state.String(), p.updatedAt.String()) + return fmt.Sprintf("identity: %s, version: %d, state: %s, updatedAt: %s", p.identity, p.version, p.state.String(), p.updatedAt.String()) } // --------------------------------------------------------------- diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index adaf123ce..0a1bb31f1 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -44,7 +44,12 @@ func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) e // keep track of participant updates and versions p.updateLock.Lock() for _, op := range joinResponse.OtherParticipants { - p.updateCache.Add(livekit.ParticipantID(op.Sid), participantUpdateInfo{version: op.Version, state: op.State, updatedAt: time.Now()}) + p.updateCache.Add(livekit.ParticipantID(op.Sid), participantUpdateInfo{ + identity: livekit.ParticipantIdentity(op.Identity), + version: op.Version, + state: op.State, + updatedAt: time.Now(), + }) } p.updateLock.Unlock() @@ -104,7 +109,12 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. isValid = false } if isValid { - p.updateCache.Add(pID, participantUpdateInfo{version: pi.Version, state: pi.State, updatedAt: time.Now()}) + p.updateCache.Add(pID, participantUpdateInfo{ + identity: livekit.ParticipantIdentity(pi.Identity), + version: pi.Version, + state: pi.State, + updatedAt: time.Now(), + }) validUpdates = append(validUpdates, pi) } } @@ -213,7 +223,7 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { } else if info.state == livekit.ParticipantInfo_DISCONNECTED { disconnectedParticipants = append(disconnectedParticipants, &livekit.ParticipantInfo{ Sid: string(keys[i]), - Identity: string(p.Identity()), + Identity: string(info.identity), Version: info.version, State: livekit.ParticipantInfo_DISCONNECTED, }) @@ -222,6 +232,10 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { } p.updateLock.Unlock() + if len(disconnectedParticipants) == 0 { + return nil + } + return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Update{ Update: &livekit.ParticipantUpdate{ From 5ee307952e069769f76fe02aa507fb50251ec45f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 18 Dec 2023 14:27:55 +0530 Subject: [PATCH 029/114] Reduce a couple of logs to Debugw. Small saving. (#2322) --- pkg/rtc/transport.go | 4 ++-- pkg/sfu/streamallocator/streamallocator.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 528d14b02..d449cf3d9 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -633,9 +633,9 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState t.setICEStartedAt(time.Now()) case webrtc.ICEConnectionStateDisconnected: - fallthrough - case webrtc.ICEConnectionStateFailed: t.params.Logger.Infow("ice connection state change unexpected", "state", state.String()) + case webrtc.ICEConnectionStateFailed: + t.params.Logger.Debugw("ice connection state change unexpected", "state", state.String()) } } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 8fde6e3c7..364dd0895 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -835,7 +835,7 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "commitThreshold(bps)", commitThreshold, "channel", s.channelObserver.ToString(), ) - s.params.Logger.Infow( + s.params.Logger.Debugw( fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity: experimental", action), "rateHistory", s.rateMonitor.GetHistory(), "expectedQueuing", s.rateMonitor.GetQueuingGuess(), From 37539fdf7697c3e5c55b6f44476d356480c2d5ca Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 19 Dec 2023 11:50:48 +0530 Subject: [PATCH 030/114] Add Version to TrackInfo. (#2324) * Add Version to TrackInfo. Set when a track is published. * update protocol --- go.mod | 12 ++-- go.sum | 24 +++---- pkg/rtc/mediatrackreceiver.go | 8 +++ pkg/rtc/participant.go | 1 + pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_media_track.go | 66 +++++++++++++++++++ pkg/rtc/types/typesfakes/fake_media_track.go | 66 +++++++++++++++++++ 7 files changed, 160 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index baad85cb5..ab3d91800 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb - github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876 - github.com/livekit/psrpc v0.5.2 + github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c + github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0 @@ -70,7 +70,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.3 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -95,13 +95,13 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.16.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect - google.golang.org/grpc v1.59.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/grpc v1.60.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index a190f3139..746a2821c 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -126,10 +126,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb h1:KiGg4k+kYQD9NjKixaSDMMeYOO2//XBM4IROTI1Itjo= github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876 h1:NnbpPgxDHOcSdgW0JzBkc4QzzLVAe4sOaiYqUUH0/K4= -github.com/livekit/protocol v1.9.4-0.20231206174612-7bba17ea7876/go.mod h1:SzrmeWw8sbf99laJJNMwp+5izlvh/ynlMbVOX0JUoes= -github.com/livekit/psrpc v0.5.2 h1:+MvG8Otm/J6MTg2MP/uuMbrkxOWsrj2hDhu/I1VIU1U= -github.com/livekit/psrpc v0.5.2/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= +github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c h1:N1nhu8+N70ZuCZ2DLfqNsoLneqV0j2mbqsWSvOHY71w= +github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c/go.mod h1:acKFhqYltprWHzFV1A8ILARlJnBfwsdHw9HxWQjxTf4= +github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= +github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -295,8 +295,8 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -421,10 +421,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index b97ffbed1..358a9f615 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -644,6 +645,13 @@ func (t *MediaTrackReceiver) TrackInfo(generateLayer bool) *livekit.TrackInfo { return ti } +func (t *MediaTrackReceiver) Version() utils.TimedVersion { + t.lock.RLock() + defer t.lock.RUnlock() + + return utils.TimedVersionFromProto(t.trackInfo.Version) +} + func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { t.lock.Lock() for _, layer := range layers { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 28863a2fa..a1a01f5ba 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1824,6 +1824,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } ti.MimeType = track.Codec().MimeType + ti.Version = p.params.VersionGenerator.New().ToProto() mt = p.addMediaTrack(signalCid, track.ID(), ti) newTrack = true p.dirty.Store(true) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 77e395d85..34ac40712 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -445,6 +445,7 @@ type MediaTrack interface { Stream() string ToProto() *livekit.TrackInfo + Version() utils.TimedVersion PublisherID() livekit.ParticipantID PublisherIdentity() livekit.ParticipantIdentity diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index a606f8ef1..d056f8023 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -7,6 +7,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" ) type FakeLocalMediaTrack struct { @@ -337,6 +338,16 @@ type FakeLocalMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } + VersionStub func() utils.TimedVersion + versionMutex sync.RWMutex + versionArgsForCall []struct { + } + versionReturns struct { + result1 utils.TimedVersion + } + versionReturnsOnCall map[int]struct { + result1 utils.TimedVersion + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -2109,6 +2120,59 @@ func (fake *FakeLocalMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit. return argsForCall.arg1 } +func (fake *FakeLocalMediaTrack) Version() utils.TimedVersion { + fake.versionMutex.Lock() + ret, specificReturn := fake.versionReturnsOnCall[len(fake.versionArgsForCall)] + fake.versionArgsForCall = append(fake.versionArgsForCall, struct { + }{}) + stub := fake.VersionStub + fakeReturns := fake.versionReturns + fake.recordInvocation("Version", []interface{}{}) + fake.versionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) VersionCallCount() int { + fake.versionMutex.RLock() + defer fake.versionMutex.RUnlock() + return len(fake.versionArgsForCall) +} + +func (fake *FakeLocalMediaTrack) VersionCalls(stub func() utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = stub +} + +func (fake *FakeLocalMediaTrack) VersionReturns(result1 utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = nil + fake.versionReturns = struct { + result1 utils.TimedVersion + }{result1} +} + +func (fake *FakeLocalMediaTrack) VersionReturnsOnCall(i int, result1 utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = nil + if fake.versionReturnsOnCall == nil { + fake.versionReturnsOnCall = make(map[int]struct { + result1 utils.TimedVersion + }) + } + fake.versionReturnsOnCall[i] = struct { + result1 utils.TimedVersion + }{result1} +} + func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -2184,6 +2248,8 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.toProtoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.versionMutex.RLock() + defer fake.versionMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index fd472c52e..6f2006d6b 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -7,6 +7,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" ) type FakeMediaTrack struct { @@ -273,6 +274,16 @@ type FakeMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } + VersionStub func() utils.TimedVersion + versionMutex sync.RWMutex + versionArgsForCall []struct { + } + versionReturns struct { + result1 utils.TimedVersion + } + versionReturnsOnCall map[int]struct { + result1 utils.TimedVersion + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -1695,6 +1706,59 @@ func (fake *FakeMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.Video return argsForCall.arg1 } +func (fake *FakeMediaTrack) Version() utils.TimedVersion { + fake.versionMutex.Lock() + ret, specificReturn := fake.versionReturnsOnCall[len(fake.versionArgsForCall)] + fake.versionArgsForCall = append(fake.versionArgsForCall, struct { + }{}) + stub := fake.VersionStub + fakeReturns := fake.versionReturns + fake.recordInvocation("Version", []interface{}{}) + fake.versionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMediaTrack) VersionCallCount() int { + fake.versionMutex.RLock() + defer fake.versionMutex.RUnlock() + return len(fake.versionArgsForCall) +} + +func (fake *FakeMediaTrack) VersionCalls(stub func() utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = stub +} + +func (fake *FakeMediaTrack) VersionReturns(result1 utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = nil + fake.versionReturns = struct { + result1 utils.TimedVersion + }{result1} +} + +func (fake *FakeMediaTrack) VersionReturnsOnCall(i int, result1 utils.TimedVersion) { + fake.versionMutex.Lock() + defer fake.versionMutex.Unlock() + fake.VersionStub = nil + if fake.versionReturnsOnCall == nil { + fake.versionReturnsOnCall = make(map[int]struct { + result1 utils.TimedVersion + }) + } + fake.versionReturnsOnCall[i] = struct { + result1 utils.TimedVersion + }{result1} +} + func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -1754,6 +1818,8 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.toProtoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.versionMutex.RLock() + defer fake.versionMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value From 7c841e8895b784b59d97018790c8ea59ddbd4a79 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 20 Dec 2023 09:48:13 +0530 Subject: [PATCH 031/114] Only assign TrackInfo Version on fresh publish. (#2325) * Only assign TrackInfo Version on fresh publish. * remove redundant nil check --- pkg/rtc/participant.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a1a01f5ba..badc6b954 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1824,7 +1824,10 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } ti.MimeType = track.Codec().MimeType - ti.Version = p.params.VersionGenerator.New().ToProto() + if utils.NewTimedVersionFromProto(ti.Version).IsZero() { + // only assign version on a fresh publish, i. e. avoid updating version in scenarios like migration + ti.Version = p.params.VersionGenerator.New().ToProto() + } mt = p.addMediaTrack(signalCid, track.ID(), ti) newTrack = true p.dirty.Store(true) From d0c36aa6cce158031082242478bdb09d4e4c6df7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 20 Dec 2023 09:58:08 +0530 Subject: [PATCH 032/114] Make UpdateTrackInfo an interface. (#2327) --- pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_media_track.go | 39 +++++++++++++++++++ pkg/rtc/types/typesfakes/fake_media_track.go | 39 +++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 34ac40712..bcea1f58b 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -444,6 +444,7 @@ type MediaTrack interface { Source() livekit.TrackSource Stream() string + UpdateTrackInfo(ti *livekit.TrackInfo) ToProto() *livekit.TrackInfo Version() utils.TimedVersion diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index d056f8023..555467591 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -333,6 +333,11 @@ type FakeLocalMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateTrackInfoStub func(*livekit.TrackInfo) + updateTrackInfoMutex sync.RWMutex + updateTrackInfoArgsForCall []struct { + arg1 *livekit.TrackInfo + } UpdateVideoLayersStub func([]*livekit.VideoLayer) updateVideoLayersMutex sync.RWMutex updateVideoLayersArgsForCall []struct { @@ -2083,6 +2088,38 @@ func (fake *FakeLocalMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.Tr }{result1} } +func (fake *FakeLocalMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { + fake.updateTrackInfoMutex.Lock() + fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { + arg1 *livekit.TrackInfo + }{arg1}) + stub := fake.UpdateTrackInfoStub + fake.recordInvocation("UpdateTrackInfo", []interface{}{arg1}) + fake.updateTrackInfoMutex.Unlock() + if stub != nil { + fake.UpdateTrackInfoStub(arg1) + } +} + +func (fake *FakeLocalMediaTrack) UpdateTrackInfoCallCount() int { + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() + return len(fake.updateTrackInfoArgsForCall) +} + +func (fake *FakeLocalMediaTrack) UpdateTrackInfoCalls(stub func(*livekit.TrackInfo)) { + fake.updateTrackInfoMutex.Lock() + defer fake.updateTrackInfoMutex.Unlock() + fake.UpdateTrackInfoStub = stub +} + +func (fake *FakeLocalMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.TrackInfo { + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() + argsForCall := fake.updateTrackInfoArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { var arg1Copy []*livekit.VideoLayer if arg1 != nil { @@ -2246,6 +2283,8 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() fake.versionMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index 6f2006d6b..1f88d07a5 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -269,6 +269,11 @@ type FakeMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateTrackInfoStub func(*livekit.TrackInfo) + updateTrackInfoMutex sync.RWMutex + updateTrackInfoArgsForCall []struct { + arg1 *livekit.TrackInfo + } UpdateVideoLayersStub func([]*livekit.VideoLayer) updateVideoLayersMutex sync.RWMutex updateVideoLayersArgsForCall []struct { @@ -1669,6 +1674,38 @@ func (fake *FakeMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.TrackIn }{result1} } +func (fake *FakeMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { + fake.updateTrackInfoMutex.Lock() + fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { + arg1 *livekit.TrackInfo + }{arg1}) + stub := fake.UpdateTrackInfoStub + fake.recordInvocation("UpdateTrackInfo", []interface{}{arg1}) + fake.updateTrackInfoMutex.Unlock() + if stub != nil { + fake.UpdateTrackInfoStub(arg1) + } +} + +func (fake *FakeMediaTrack) UpdateTrackInfoCallCount() int { + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() + return len(fake.updateTrackInfoArgsForCall) +} + +func (fake *FakeMediaTrack) UpdateTrackInfoCalls(stub func(*livekit.TrackInfo)) { + fake.updateTrackInfoMutex.Lock() + defer fake.updateTrackInfoMutex.Unlock() + fake.UpdateTrackInfoStub = stub +} + +func (fake *FakeMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.TrackInfo { + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() + argsForCall := fake.updateTrackInfoArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { var arg1Copy []*livekit.VideoLayer if arg1 != nil { @@ -1816,6 +1853,8 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateTrackInfoMutex.RLock() + defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() fake.versionMutex.RLock() From be526cc43e736285846f6e1c96483154edaa9f9e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 20 Dec 2023 08:10:35 -0800 Subject: [PATCH 033/114] Adds callback to notify when workers are registered (#2328) --- pkg/service/agentservice.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index 9e7eef0e8..20636bc60 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -55,6 +55,7 @@ type AgentHandler struct { roomWorkers map[string]*worker publisherRegistered bool publisherWorkers map[string]*worker + onWorkerRegistered func(handler *AgentHandler) } type worker struct { @@ -130,6 +131,13 @@ func NewAgentHandler(agentServer rpc.AgentInternalServer, roomTopic, publisherTo } } +// OnWorkerRegistered registers a callback to be called when the first worker of each type is registered +func (s *AgentHandler) OnWorkerRegistered(handler func(handler *AgentHandler)) { + s.mu.Lock() + defer s.mu.Unlock() + s.onWorkerRegistered = handler +} + func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Conn) { sigConn := NewWSSignalConnection(conn) w := &worker{ @@ -218,6 +226,9 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor return errors.New("worker already registered") } + onRegistered := s.onWorkerRegistered + firstWorker := false + switch msg.Type { case livekit.JobType_JT_ROOM: worker.id = msg.WorkerId @@ -231,6 +242,7 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor worker.logger.Errorw("failed to register room agents", err) } else { s.roomRegistered = true + firstWorker = true } } @@ -246,6 +258,7 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor worker.logger.Errorw("failed to register publisher agents", err) } else { s.publisherRegistered = true + firstWorker = true } } default: @@ -266,6 +279,9 @@ func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWor worker.logger.Errorw("failed to write server message", err) } + if firstWorker && onRegistered != nil { + onRegistered(s) + } return nil } From faff67162bf7721f23f6a26e83375ce7b3759f36 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 21 Dec 2023 09:56:54 +0530 Subject: [PATCH 034/114] Consolidate TrackInfo. (#2331) * Consolidate TrackInfo. TrackInfo was spread across a bit. Consolidating it. * TODO comments * test * update TrackInfo on SSRC change * further consolidation * log mimes only * update receivers on SSRC set * clone proto on return * feedback: break loop on mime match * prevent data race --- pkg/rtc/mediatrack.go | 91 ++-- pkg/rtc/mediatrack_test.go | 26 +- pkg/rtc/mediatrackreceiver.go | 401 ++++++++++-------- pkg/rtc/participant.go | 5 +- pkg/rtc/participant_sdp.go | 2 +- pkg/rtc/types/interfaces.go | 1 - .../typesfakes/fake_local_media_track.go | 66 --- pkg/rtc/types/typesfakes/fake_media_track.go | 66 --- pkg/rtc/wrappedreceiver.go | 6 + pkg/sfu/buffer/videolayerutils.go | 1 + pkg/sfu/receiver.go | 41 +- pkg/sfu/streamtrackermanager.go | 26 +- 12 files changed, 325 insertions(+), 407 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 584c953c3..73796cb73 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -22,7 +22,6 @@ import ( "github.com/pion/rtcp" "github.com/pion/webrtc/v3" "go.uber.org/atomic" - "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/livekit" @@ -52,7 +51,6 @@ type MediaTrack struct { } type MediaTrackParams struct { - TrackInfo *livekit.TrackInfo SignalCid string SdpCid string ParticipantID livekit.ParticipantID @@ -71,13 +69,12 @@ type MediaTrackParams struct { SimTracks map[uint32]SimulcastTrackInfo } -func NewMediaTrack(params MediaTrackParams) *MediaTrack { +func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { t := &MediaTrack{ params: params, } t.MediaTrackReceiver = NewMediaTrackReceiver(MediaTrackReceiverParams{ - TrackInfo: params.TrackInfo, MediaTrack: t, IsRelayed: false, ParticipantID: params.ParticipantID, @@ -88,19 +85,9 @@ func NewMediaTrack(params MediaTrackParams) *MediaTrack { AudioConfig: params.AudioConfig, Telemetry: params.Telemetry, Logger: params.Logger, - }) - t.MediaTrackReceiver.OnVideoLayerUpdate(func(layers []*livekit.VideoLayer) { - t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), - &livekit.TrackInfo{ - Sid: string(t.ID()), - Type: livekit.TrackType_VIDEO, - Muted: t.IsMuted(), - Simulcast: t.IsSimulcast(), - Layers: layers, - }) - }) + }, ti) - if params.TrackInfo.Type == livekit.TrackType_AUDIO { + if ti.Type == livekit.TrackType_AUDIO { t.MediaLossProxy = NewMediaLossProxy(MediaLossProxyParams{ Logger: params.Logger, }) @@ -113,7 +100,7 @@ func NewMediaTrack(params MediaTrackParams) *MediaTrack { t.MediaTrackReceiver.OnMediaLossFeedback(t.MediaLossProxy.HandleMaxLossFeedback) } - if params.TrackInfo.Type == livekit.TrackType_VIDEO { + if ti.Type == livekit.TrackType_VIDEO { t.dynacastManager = NewDynacastManager(DynacastManagerParams{ DynacastPauseDelay: params.VideoConfig.DynacastPauseDelay, Logger: params.Logger, @@ -126,7 +113,7 @@ func NewMediaTrack(params MediaTrackParams) *MediaTrack { t.dynacastManager.NotifySubscriberMaxQuality( subscriberID, codec.MimeType, - buffer.SpatialLayerToVideoQuality(layer, t.params.TrackInfo), + buffer.SpatialLayerToVideoQuality(layer, t.MediaTrackReceiver.TrackInfo()), ) }, ) @@ -154,7 +141,7 @@ func (t *MediaTrack) OnSubscribedMaxQualityChange( for _, q := range maxSubscribedQualities { receiver := t.Receiver(q.CodecMime) if receiver != nil { - receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(q.Quality, t.params.TrackInfo)) + receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(q.Quality, t.MediaTrackReceiver.TrackInfo())) } } } @@ -177,8 +164,8 @@ func (t *MediaTrack) HasSdpCid(cid string) bool { return true } - info := t.params.TrackInfo - for _, c := range info.Codecs { + ti := t.MediaTrackReceiver.TrackInfoClone() + for _, c := range ti.Codecs { if c.Cid == cid { return true } @@ -187,24 +174,11 @@ func (t *MediaTrack) HasSdpCid(cid string) bool { } func (t *MediaTrack) ToProto() *livekit.TrackInfo { - info := t.MediaTrackReceiver.TrackInfo(true) - info.Muted = t.IsMuted() - info.Simulcast = t.IsSimulcast() - return info + return t.MediaTrackReceiver.TrackInfoClone() } -func (t *MediaTrack) SetPendingCodecSid(codecs []*livekit.SimulcastCodec) { - ti := proto.Clone(t.params.TrackInfo).(*livekit.TrackInfo) - for _, c := range codecs { - for _, origin := range ti.Codecs { - if strings.Contains(origin.MimeType, c.Codec) { - origin.Cid = c.Cid - break - } - } - } - t.params.TrackInfo = ti - t.MediaTrackReceiver.UpdateTrackInfo(ti) +func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { + t.MediaTrackReceiver.UpdateCodecCid(codecs) } // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec @@ -233,24 +207,25 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra } }) + ti := t.MediaTrackReceiver.TrackInfoClone() t.lock.Lock() mime := strings.ToLower(track.Codec().MimeType) - layer := buffer.RidToSpatialLayer(track.RID(), t.trackInfo) + layer := buffer.RidToSpatialLayer(track.RID(), ti) t.params.Logger.Debugw("AddReceiver", "mime", track.Codec().MimeType) wr := t.MediaTrackReceiver.Receiver(mime) if wr == nil { priority := -1 - for idx, c := range t.params.TrackInfo.Codecs { + for idx, c := range ti.Codecs { if strings.EqualFold(mime, c.MimeType) { priority = idx break } } - if len(t.params.TrackInfo.Codecs) == 0 { + if len(ti.Codecs) == 0 { priority = 0 } if priority < 0 { - t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mime, "track", logger.Proto(t.params.TrackInfo)) + t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mime, "track", logger.Proto(ti)) t.lock.Unlock() return false } @@ -258,7 +233,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra newWR := sfu.NewWebRTCReceiver( receiver, track, - t.params.TrackInfo, + ti, LoggerWithCodecMime(t.params.Logger, mime), twcc, t.params.VideoConfig.StreamTracker, @@ -277,16 +252,20 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra } } }) - newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { - // LK-TODO: this needs to be receiver/mime aware - key := telemetry.StatsKeyForTrack(livekit.StreamType_UPSTREAM, t.PublisherID(), t.ID(), t.params.TrackInfo.Source, t.params.TrackInfo.Type) - t.params.Telemetry.TrackStats(key, stat) - }) + // SIMULCAST-CODEC-TODO: these need to be receiver/mime aware, setting it up only for primary now + if priority == 0 { + newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { + key := telemetry.StatsKeyForTrack(livekit.StreamType_UPSTREAM, t.PublisherID(), t.ID(), ti.Source, ti.Type) + t.params.Telemetry.TrackStats(key, stat) + }) + + newWR.OnMaxLayerChange(t.onMaxLayerChange) + } if t.PrimaryReceiver() == nil { // primary codec published, set potential codecs - potentialCodecs := make([]webrtc.RTPCodecParameters, 0, len(t.params.TrackInfo.Codecs)) + potentialCodecs := make([]webrtc.RTPCodecParameters, 0, len(ti.Codecs)) parameters := receiver.GetParameters() - for _, c := range t.params.TrackInfo.Codecs { + for _, c := range ti.Codecs { for _, nc := range parameters.Codecs { if strings.EqualFold(nc.MimeType, c.MimeType) { potentialCodecs = append(potentialCodecs, nc) @@ -301,8 +280,6 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra } } - newWR.OnMaxLayerChange(t.onMaxLayerChange) - t.buffer = buff t.MediaTrackReceiver.SetupReceiver(newWR, priority, mid) @@ -367,17 +344,7 @@ func (t *MediaTrack) HasPendingCodec() bool { } func (t *MediaTrack) onMaxLayerChange(maxLayer int32) { - ti := &livekit.TrackInfo{ - Sid: t.trackInfo.Sid, - Type: t.trackInfo.Type, - } - - if layer, ok := t.MediaTrackReceiver.layerDimensions[livekit.VideoQuality(maxLayer)]; ok { - ti.Layers = []*livekit.VideoLayer{{Quality: livekit.VideoQuality(maxLayer), Width: layer.Width, Height: layer.Height}} - } else if maxLayer == -1 { - ti.Layers = []*livekit.VideoLayer{{Quality: livekit.VideoQuality_OFF}} - } - t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), ti) + t.MediaTrackReceiver.NotifyMaxLayerChange(maxLayer) } func (t *MediaTrack) Restart() { diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 9a0587447..8d96bbae4 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -35,9 +35,7 @@ func TestTrackInfo(t *testing.T) { Muted: true, } - mt := NewMediaTrack(MediaTrackParams{ - TrackInfo: &ti, - }) + mt := NewMediaTrack(MediaTrackParams{}, &ti) outInfo := mt.ToProto() require.Equal(t, ti.Muted, outInfo.Muted) require.Equal(t, ti.Name, outInfo.Name) @@ -51,17 +49,17 @@ func TestTrackInfo(t *testing.T) { require.Equal(t, ti.Simulcast, outInfo.Simulcast) // make it simulcasted - mt.simulcasted.Store(true) + mt.SetSimulcast(true) require.True(t, mt.ToProto().Simulcast) } func TestGetQualityForDimension(t *testing.T) { t.Run("landscape source", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - }}) + }) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 200)) @@ -71,11 +69,11 @@ func TestGetQualityForDimension(t *testing.T) { }) t.Run("portrait source", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 540, Height: 960, - }}) + }) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(200, 400)) require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 400)) @@ -84,7 +82,7 @@ func TestGetQualityForDimension(t *testing.T) { }) t.Run("layers provided", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, @@ -105,7 +103,7 @@ func TestGetQualityForDimension(t *testing.T) { Height: 720, }, }, - }}) + }) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) @@ -114,7 +112,7 @@ func TestGetQualityForDimension(t *testing.T) { }) t.Run("highest layer with smallest dimensions", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, @@ -135,7 +133,7 @@ func TestGetQualityForDimension(t *testing.T) { Height: 720, }, }, - }}) + }) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) @@ -143,7 +141,7 @@ func TestGetQualityForDimension(t *testing.T) { require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) - mt = NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + mt = NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, @@ -164,7 +162,7 @@ func TestGetQualityForDimension(t *testing.T) { Height: 720, }, }, - }}) + }) require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120)) require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300)) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 358a9f615..b5fefa8a3 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -15,6 +15,7 @@ package rtc import ( + "context" "errors" "fmt" "sort" @@ -23,12 +24,10 @@ import ( "github.com/pion/rtcp" "github.com/pion/webrtc/v3" - "go.uber.org/atomic" "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -74,8 +73,7 @@ func (m mediaTrackReceiverState) String() string { type simulcastReceiver struct { sfu.TrackReceiver - priority int - layerSSRCs [livekit.VideoQuality_HIGH + 1]uint32 + priority int } func (r *simulcastReceiver) Priority() int { @@ -83,7 +81,6 @@ func (r *simulcastReceiver) Priority() int { } type MediaTrackReceiverParams struct { - TrackInfo *livekit.TrackInfo MediaTrack types.MediaTrack IsRelayed bool ParticipantID livekit.ParticipantID @@ -97,32 +94,26 @@ type MediaTrackReceiverParams struct { } type MediaTrackReceiver struct { - params MediaTrackReceiverParams - muted atomic.Bool - simulcasted atomic.Bool + params MediaTrackReceiverParams lock sync.RWMutex receivers []*simulcastReceiver - receiversShadow []*simulcastReceiver trackInfo *livekit.TrackInfo - layerDimensions map[livekit.VideoQuality]*livekit.VideoLayer potentialCodecs []webrtc.RTPCodecParameters state mediaTrackReceiverState onSetupReceiver func(mime string) onMediaLossFeedback func(dt *sfu.DownTrack, report *rtcp.ReceiverReport) - onVideoLayerUpdate func(layers []*livekit.VideoLayer) onClose []func() *MediaTrackSubscriptions } -func NewMediaTrackReceiver(params MediaTrackReceiverParams) *MediaTrackReceiver { +func NewMediaTrackReceiver(params MediaTrackReceiverParams, ti *livekit.TrackInfo) *MediaTrackReceiver { t := &MediaTrackReceiver{ - params: params, - trackInfo: proto.Clone(params.TrackInfo).(*livekit.TrackInfo), - layerDimensions: make(map[livekit.VideoQuality]*livekit.VideoLayer), - state: mediaTrackReceiverStateOpen, + params: params, + trackInfo: proto.Clone(ti).(*livekit.TrackInfo), + state: mediaTrackReceiverStateOpen, } t.MediaTrackSubscriptions = NewMediaTrackSubscriptions(MediaTrackSubscriptionsParams{ @@ -138,22 +129,18 @@ func NewMediaTrackReceiver(params MediaTrackReceiverParams) *MediaTrackReceiver if t.trackInfo.Muted { t.SetMuted(true) } - - if t.trackInfo != nil && t.Kind() == livekit.TrackType_VIDEO { - t.UpdateVideoLayers(t.trackInfo.Layers) - // LK-TODO: maybe use this or simulcast flag in TrackInfo to set simulcasted here - } - return t } func (t *MediaTrackReceiver) Restart() { t.lock.Lock() - receivers := t.receiversShadow + receivers := t.receivers + ti := t.trackInfo t.lock.Unlock() + hq := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, ti) for _, receiver := range receivers { - receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.TrackInfo)) + receiver.SetMaxExpectedSpatialLayer(hq) } } @@ -210,10 +197,18 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority } } - t.shadowReceiversLocked() - + var receiverCodecs []string + for _, r := range t.receivers { + receiverCodecs = append(receiverCodecs, r.Codec().MimeType) + } + t.params.Logger.Debugw( + "setup receiver", + "mime", receiver.Codec().MimeType, + "priority", priority, + "receivers", receiverCodecs, + "mid", mid, + ) onSetupReceiver := t.onSetupReceiver - t.params.Logger.Debugw("setup receiver", "mime", receiver.Codec().MimeType, "priority", priority, "receivers", t.receiversShadow, "mid", mid) t.lock.Unlock() if onSetupReceiver != nil { @@ -255,32 +250,9 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete sort.Slice(t.receivers, func(i, j int) bool { return t.receivers[i].Priority() < t.receivers[j].Priority() }) - t.shadowReceiversLocked() t.lock.Unlock() } -func (t *MediaTrackReceiver) shadowReceiversLocked() { - t.receiversShadow = make([]*simulcastReceiver, len(t.receivers)) - copy(t.receiversShadow, t.receivers) -} - -func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) { - t.lock.Lock() - defer t.lock.Unlock() - - layer := buffer.RidToSpatialLayer(rid, t.params.TrackInfo) - if layer == buffer.InvalidLayerSpatial { - // non-simulcast case will not have `rid` - layer = 0 - } - for _, receiver := range t.receiversShadow { - if strings.EqualFold(receiver.Codec().MimeType, mime) && int(layer) < len(receiver.layerSSRCs) { - receiver.layerSSRCs[layer] = ssrc - return - } - } -} - func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { t.params.Logger.Debugw("clearing receiver", "mime", mime) t.lock.Lock() @@ -292,7 +264,6 @@ func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { } } - t.shadowReceiversLocked() t.lock.Unlock() t.removeAllSubscribersForMime(mime, willBeResumed) @@ -306,8 +277,7 @@ func (t *MediaTrackReceiver) ClearAllReceivers(willBeResumed bool) { mimes = append(mimes, receiver.Codec().MimeType) } - t.receivers = t.receivers[:0] - t.receiversShadow = nil + t.receivers = nil t.lock.Unlock() for _, mime := range mimes { @@ -319,10 +289,6 @@ func (t *MediaTrackReceiver) OnMediaLossFeedback(f func(dt *sfu.DownTrack, rr *r t.onMediaLossFeedback = f } -func (t *MediaTrackReceiver) OnVideoLayerUpdate(f func(layers []*livekit.VideoLayer)) { - t.onVideoLayerUpdate = f -} - func (t *MediaTrackReceiver) IsOpen() bool { t.lock.RLock() defer t.lock.RUnlock() @@ -353,7 +319,7 @@ func (t *MediaTrackReceiver) TryClose() bool { return true } - for _, receiver := range t.receiversShadow { + for _, receiver := range t.receivers { if dr, _ := receiver.TrackReceiver.(*DummyReceiver); dr != nil && dr.Receiver() != nil { t.lock.RUnlock() return false @@ -422,11 +388,17 @@ func (t *MediaTrackReceiver) PublisherVersion() uint32 { } func (t *MediaTrackReceiver) IsSimulcast() bool { - return t.simulcasted.Load() + t.lock.RLock() + defer t.lock.RUnlock() + + return t.trackInfo.Simulcast } func (t *MediaTrackReceiver) SetSimulcast(simulcast bool) { - t.simulcasted.Store(simulcast) + t.lock.Lock() + defer t.lock.Unlock() + + t.trackInfo.Simulcast = simulcast } func (t *MediaTrackReceiver) Name() string { @@ -437,15 +409,17 @@ func (t *MediaTrackReceiver) Name() string { } func (t *MediaTrackReceiver) IsMuted() bool { - return t.muted.Load() + t.lock.RLock() + defer t.lock.RUnlock() + + return t.trackInfo.Muted } func (t *MediaTrackReceiver) SetMuted(muted bool) { - t.muted.Store(muted) - - t.lock.RLock() - receivers := t.receiversShadow - t.lock.RUnlock() + t.lock.Lock() + t.trackInfo.Muted = muted + receivers := t.receivers + t.lock.Unlock() for _, receiver := range receivers { receiver.SetUpTrackPaused(muted) } @@ -471,7 +445,7 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su return nil, ErrNotOpen } - receivers := t.receiversShadow + receivers := t.receivers potentialCodecs := make([]webrtc.RTPCodecParameters, len(t.potentialCodecs)) copy(potentialCodecs, t.potentialCodecs) t.lock.RUnlock() @@ -553,134 +527,219 @@ func (t *MediaTrackReceiver) RevokeDisallowedSubscribers(allowedSubscriberIdenti return revokedSubscriberIdentities } -func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { - clonedInfo := proto.Clone(ti).(*livekit.TrackInfo) +func (t *MediaTrackReceiver) updateTrackInfoOfReceivers() { + t.lock.RLock() + receivers := t.receivers + ti := t.trackInfo + t.lock.RUnlock() + + for _, r := range receivers { + r.UpdateTrackInfo(ti) + } +} + +func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) { t.lock.Lock() - originInfo := t.trackInfo - for _, ci := range clonedInfo.Codecs { - for _, originCi := range originInfo.Codecs { - if strings.EqualFold(ci.MimeType, originCi.MimeType) && originCi.Mid != "" { - ci.Mid = originCi.Mid + layer := buffer.RidToSpatialLayer(rid, t.trackInfo) + if layer == buffer.InvalidLayerSpatial { + // non-simulcast case will not have `rid` + layer = 0 + } + quality := buffer.SpatialLayerToVideoQuality(layer, t.trackInfo) + // set video layer ssrc info + for _, ci := range t.trackInfo.Codecs { + if !strings.EqualFold(ci.MimeType, mime) { + continue + } + + // if origin layer has ssrc, don't override it + var matchingLayer *livekit.VideoLayer + ssrcFound := false + for _, l := range ci.Layers { + if l.Quality == quality { + matchingLayer = l + if l.Ssrc != 0 { + ssrcFound = true + } + break + } + } + if !ssrcFound && matchingLayer != nil { + matchingLayer.Ssrc = ssrc + } + break + } + + // for client don't use simulcast codecs (old client version or single codec) + if len(t.trackInfo.Codecs) == 0 { + // if origin layer has ssrc, don't override it + var matchingLayer *livekit.VideoLayer + ssrcFound := false + for _, l := range t.trackInfo.Layers { + if l.Quality == quality { + matchingLayer = l + if l.Ssrc != 0 { + ssrcFound = true + } + break + } + } + if !ssrcFound && matchingLayer != nil { + matchingLayer.Ssrc = ssrc + } + } + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + +func (t *MediaTrackReceiver) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { + t.lock.Lock() + for _, c := range codecs { + for _, origin := range t.trackInfo.Codecs { + if strings.Contains(origin.MimeType, c.Codec) { + origin.Cid = c.Cid break } } } - t.trackInfo = clonedInfo t.lock.Unlock() - if ti != nil && t.Kind() == livekit.TrackType_VIDEO { - t.UpdateVideoLayers(ti.Layers) - } + t.updateTrackInfoOfReceivers() } -func (t *MediaTrackReceiver) TrackInfo(generateLayer bool) *livekit.TrackInfo { - t.lock.RLock() - defer t.lock.RUnlock() +func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + clonedInfo := proto.Clone(ti).(*livekit.TrackInfo) - ti := proto.Clone(t.trackInfo).(*livekit.TrackInfo) - if !generateLayer { - return ti - } - - layers := t.getVideoLayersLocked() - - // set video layer ssrc info - for i, ci := range ti.Codecs { - for _, receiver := range t.receiversShadow { - if receiver.priority == i { - originLayers := ci.Layers - ci.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - ci.Layers = append(ci.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - - // if origin layer has ssrc, don't override it - ssrcFound := false - for _, l := range originLayers { - if l.Quality == ci.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - ci.Layers[layerIdx].Ssrc = l.Ssrc - ssrcFound = true - } - break - } - } - if !ssrcFound && int(layer.Quality) < len(receiver.layerSSRCs) { - ci.Layers[layerIdx].Ssrc = receiver.layerSSRCs[layer.Quality] - } - } - - if i == 0 { - ti.Layers = ci.Layers - } - break + t.lock.Lock() + // patch Mid and SSRC of codecs/layers by keeping original if available + for i, ci := range clonedInfo.Codecs { + for _, originCi := range t.trackInfo.Codecs { + if !strings.EqualFold(ci.MimeType, originCi.MimeType) { + continue } + + if originCi.Mid != "" { + ci.Mid = originCi.Mid + } + + for _, layer := range ci.Layers { + for _, originLayer := range originCi.Layers { + if layer.Quality == originLayer.Quality { + if originLayer.Ssrc != 0 { + layer.Ssrc = originLayer.Ssrc + } + break + } + } + } + break + } + + if i == 0 { + clonedInfo.Layers = ci.Layers } } // for client don't use simulcast codecs (old client version or single codec) - if len(ti.Codecs) == 0 && len(t.receiversShadow) > 0 { - receiver := t.receiversShadow[0] - originLayers := ti.Layers - ti.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - ti.Layers = append(ti.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - - // if origin layer has ssrc, don't override it - ssrcFound := false - for _, l := range originLayers { - if l.Quality == ti.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - ti.Layers[layerIdx].Ssrc = l.Ssrc - ssrcFound = true + if len(clonedInfo.Codecs) == 0 { + for _, layer := range clonedInfo.Layers { + for _, originLayer := range t.trackInfo.Layers { + if layer.Quality == originLayer.Quality { + if originLayer.Ssrc != 0 { + layer.Ssrc = originLayer.Ssrc } break } } - if !ssrcFound && int(layer.Quality) < len(receiver.layerSSRCs) { - ti.Layers[layerIdx].Ssrc = receiver.layerSSRCs[layer.Quality] - } } } - return ti -} + t.trackInfo = clonedInfo + t.lock.Unlock() -func (t *MediaTrackReceiver) Version() utils.TimedVersion { - t.lock.RLock() - defer t.lock.RUnlock() - - return utils.TimedVersionFromProto(t.trackInfo.Version) + t.updateTrackInfoOfReceivers() } func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { t.lock.Lock() - for _, layer := range layers { - t.layerDimensions[layer.Quality] = layer + // set video layer ssrc info + for i, ci := range t.trackInfo.Codecs { + originLayers := ci.Layers + ci.Layers = []*livekit.VideoLayer{} + for layerIdx, layer := range layers { + ci.Layers = append(ci.Layers, proto.Clone(layer).(*livekit.VideoLayer)) + for _, l := range originLayers { + if l.Quality == ci.Layers[layerIdx].Quality { + if l.Ssrc != 0 { + ci.Layers[layerIdx].Ssrc = l.Ssrc + } + break + } + } + } + + if i == 0 { + t.trackInfo.Layers = ci.Layers + } + } + + // for client don't use simulcast codecs (old client version or single codec) + if len(t.trackInfo.Codecs) == 0 { + originLayers := t.trackInfo.Layers + t.trackInfo.Layers = []*livekit.VideoLayer{} + for layerIdx, layer := range layers { + t.trackInfo.Layers = append(t.trackInfo.Layers, proto.Clone(layer).(*livekit.VideoLayer)) + for _, l := range originLayers { + if l.Quality == t.trackInfo.Layers[layerIdx].Quality { + if l.Ssrc != 0 { + t.trackInfo.Layers[layerIdx].Ssrc = l.Ssrc + } + break + } + } + } } t.lock.Unlock() + t.updateTrackInfoOfReceivers() t.MediaTrackSubscriptions.UpdateVideoLayers() - if t.onVideoLayerUpdate != nil { - t.onVideoLayerUpdate(layers) - } - - // TODO: this might need to trigger a participant update for clients to pick up dimension change } -func (t *MediaTrackReceiver) GetVideoLayers() []*livekit.VideoLayer { +func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { t.lock.RLock() defer t.lock.RUnlock() - return t.getVideoLayersLocked() + return t.trackInfo } -func (t *MediaTrackReceiver) getVideoLayersLocked() []*livekit.VideoLayer { - layers := make([]*livekit.VideoLayer, 0) - for _, layer := range t.layerDimensions { - layers = append(layers, proto.Clone(layer).(*livekit.VideoLayer)) - } +func (t *MediaTrackReceiver) TrackInfoClone() *livekit.TrackInfo { + t.lock.RLock() + defer t.lock.RUnlock() - return layers + return proto.Clone(t.trackInfo).(*livekit.TrackInfo) +} + +func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { + t.lock.RLock() + quality := buffer.SpatialLayerToVideoQuality(maxLayer, t.trackInfo) + ti := &livekit.TrackInfo{ + Sid: t.trackInfo.Sid, + Type: t.trackInfo.Type, + Layers: []*livekit.VideoLayer{{Quality: quality}}, + } + if quality != livekit.VideoQuality_OFF { + for _, layer := range t.trackInfo.Layers { + if layer.Quality == quality { + ti.Layers[0].Width = layer.Width + ti.Layers[0].Height = layer.Height + break + } + } + } + t.lock.RUnlock() + + t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), ti) } // GetQualityForDimension finds the closest quality to use for desired dimensions @@ -708,7 +767,7 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki // default sizes representing qualities low - high layerSizes := []uint32{180, 360, origSize} var providedSizes []uint32 - for _, layer := range t.layerDimensions { + for _, layer := range t.trackInfo.Layers { providedSizes = append(providedSizes, layer.Height) } if len(providedSizes) > 0 { @@ -757,13 +816,13 @@ func (t *MediaTrackReceiver) DebugInfo() map[string]interface{} { info := map[string]interface{}{ "ID": t.ID(), "Kind": t.Kind().String(), - "PubMuted": t.muted.Load(), + "PubMuted": t.IsMuted(), } info["DownTracks"] = t.MediaTrackSubscriptions.DebugInfo() t.lock.RLock() - receivers := t.receiversShadow + receivers := t.receivers t.lock.RUnlock() for _, receiver := range receivers { info[receiver.Codec().MimeType] = receiver.DebugInfo() @@ -776,20 +835,20 @@ func (t *MediaTrackReceiver) PrimaryReceiver() sfu.TrackReceiver { t.lock.RLock() defer t.lock.RUnlock() - if len(t.receiversShadow) == 0 { + if len(t.receivers) == 0 { return nil } - if dr, ok := t.receiversShadow[0].TrackReceiver.(*DummyReceiver); ok { + if dr, ok := t.receivers[0].TrackReceiver.(*DummyReceiver); ok { return dr.Receiver() } - return t.receiversShadow[0].TrackReceiver + return t.receivers[0].TrackReceiver } func (t *MediaTrackReceiver) Receiver(mime string) sfu.TrackReceiver { t.lock.RLock() defer t.lock.RUnlock() - for _, r := range t.receiversShadow { + for _, r := range t.receivers { if strings.EqualFold(r.Codec().MimeType, mime) { if dr, ok := r.TrackReceiver.(*DummyReceiver); ok { return dr.Receiver() @@ -804,17 +863,17 @@ func (t *MediaTrackReceiver) Receivers() []sfu.TrackReceiver { t.lock.RLock() defer t.lock.RUnlock() - receivers := make([]sfu.TrackReceiver, 0, len(t.receiversShadow)) - for _, r := range t.receiversShadow { + receivers := make([]sfu.TrackReceiver, 0, len(t.receivers)) + for _, r := range t.receivers { receivers = append(receivers, r.TrackReceiver) } return receivers } func (t *MediaTrackReceiver) SetRTT(rtt uint32) { - t.lock.Lock() - receivers := t.receiversShadow - t.lock.Unlock() + t.lock.RLock() + receivers := t.receivers + t.lock.RUnlock() for _, r := range receivers { if wr, ok := r.TrackReceiver.(*sfu.WebRTCReceiver); ok { @@ -847,9 +906,9 @@ func (t *MediaTrackReceiver) IsEncrypted() bool { } func (t *MediaTrackReceiver) GetTrackStats() *livekit.RTPStats { - t.lock.Lock() - receivers := t.receiversShadow - t.lock.Unlock() + t.lock.RLock() + receivers := t.receivers + t.lock.RUnlock() stats := make([]*livekit.RTPStats, 0, len(receivers)) for _, receiver := range receivers { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index badc6b954..6f07849b8 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1622,7 +1622,7 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l return nil } - track.(*MediaTrack).SetPendingCodecSid(req.SimulcastCodecs) + track.(*MediaTrack).UpdateCodecCid(req.SimulcastCodecs) ti := track.ToProto() return ti } @@ -1907,7 +1907,6 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *livekit.TrackInfo) *MediaTrack { mt := NewMediaTrack(MediaTrackParams{ - TrackInfo: proto.Clone(ti).(*livekit.TrackInfo), SignalCid: signalCid, SdpCid: sdpCid, ParticipantID: p.params.SID, @@ -1923,7 +1922,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv SubscriberConfig: p.params.Config.Subscriber, PLIThrottleConfig: p.params.PLIThrottleConfig, SimTracks: p.params.SimTracks, - }) + }, ti) mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange) diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 51e98c478..ed47d2da9 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -242,7 +242,7 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript _, ti, _ = p.getPendingTrack(streamID, livekit.TrackType_AUDIO) p.pendingTracksLock.RUnlock() } else { - ti = track.TrackInfo(false) + ti = track.ToProto() } break } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index bcea1f58b..05f7d5a2d 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -446,7 +446,6 @@ type MediaTrack interface { UpdateTrackInfo(ti *livekit.TrackInfo) ToProto() *livekit.TrackInfo - Version() utils.TimedVersion PublisherID() livekit.ParticipantID PublisherIdentity() livekit.ParticipantIdentity diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index 555467591..be86e74c4 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -7,7 +7,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/utils" ) type FakeLocalMediaTrack struct { @@ -343,16 +342,6 @@ type FakeLocalMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } - VersionStub func() utils.TimedVersion - versionMutex sync.RWMutex - versionArgsForCall []struct { - } - versionReturns struct { - result1 utils.TimedVersion - } - versionReturnsOnCall map[int]struct { - result1 utils.TimedVersion - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -2157,59 +2146,6 @@ func (fake *FakeLocalMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit. return argsForCall.arg1 } -func (fake *FakeLocalMediaTrack) Version() utils.TimedVersion { - fake.versionMutex.Lock() - ret, specificReturn := fake.versionReturnsOnCall[len(fake.versionArgsForCall)] - fake.versionArgsForCall = append(fake.versionArgsForCall, struct { - }{}) - stub := fake.VersionStub - fakeReturns := fake.versionReturns - fake.recordInvocation("Version", []interface{}{}) - fake.versionMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalMediaTrack) VersionCallCount() int { - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() - return len(fake.versionArgsForCall) -} - -func (fake *FakeLocalMediaTrack) VersionCalls(stub func() utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = stub -} - -func (fake *FakeLocalMediaTrack) VersionReturns(result1 utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = nil - fake.versionReturns = struct { - result1 utils.TimedVersion - }{result1} -} - -func (fake *FakeLocalMediaTrack) VersionReturnsOnCall(i int, result1 utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = nil - if fake.versionReturnsOnCall == nil { - fake.versionReturnsOnCall = make(map[int]struct { - result1 utils.TimedVersion - }) - } - fake.versionReturnsOnCall[i] = struct { - result1 utils.TimedVersion - }{result1} -} - func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -2287,8 +2223,6 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index 1f88d07a5..d4bdfc17e 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -7,7 +7,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/utils" ) type FakeMediaTrack struct { @@ -279,16 +278,6 @@ type FakeMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } - VersionStub func() utils.TimedVersion - versionMutex sync.RWMutex - versionArgsForCall []struct { - } - versionReturns struct { - result1 utils.TimedVersion - } - versionReturnsOnCall map[int]struct { - result1 utils.TimedVersion - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -1743,59 +1732,6 @@ func (fake *FakeMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.Video return argsForCall.arg1 } -func (fake *FakeMediaTrack) Version() utils.TimedVersion { - fake.versionMutex.Lock() - ret, specificReturn := fake.versionReturnsOnCall[len(fake.versionArgsForCall)] - fake.versionArgsForCall = append(fake.versionArgsForCall, struct { - }{}) - stub := fake.VersionStub - fakeReturns := fake.versionReturns - fake.recordInvocation("Version", []interface{}{}) - fake.versionMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeMediaTrack) VersionCallCount() int { - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() - return len(fake.versionArgsForCall) -} - -func (fake *FakeMediaTrack) VersionCalls(stub func() utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = stub -} - -func (fake *FakeMediaTrack) VersionReturns(result1 utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = nil - fake.versionReturns = struct { - result1 utils.TimedVersion - }{result1} -} - -func (fake *FakeMediaTrack) VersionReturnsOnCall(i int, result1 utils.TimedVersion) { - fake.versionMutex.Lock() - defer fake.versionMutex.Unlock() - fake.VersionStub = nil - if fake.versionReturnsOnCall == nil { - fake.versionReturnsOnCall = make(map[int]struct { - result1 utils.TimedVersion - }) - } - fake.versionReturnsOnCall[i] = struct { - result1 utils.TimedVersion - }{result1} -} - func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -1857,8 +1793,6 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 8df479ce7..a887ef832 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -294,6 +294,12 @@ func (d *DummyReceiver) TrackInfo() *livekit.TrackInfo { return nil } +func (d *DummyReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + r.UpdateTrackInfo(ti) + } +} + func (d *DummyReceiver) IsClosed() bool { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.IsClosed() diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 895af9f18..f099c2e7c 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -25,6 +25,7 @@ const ( FullResolution = "f" ) +// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool { if trackInfo == nil || len(trackInfo.Layers) == 0 { return nil diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 9afff8e98..e555b99f9 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -24,6 +24,7 @@ import ( "github.com/pion/rtcp" "github.com/pion/webrtc/v3" "go.uber.org/atomic" + "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil/pkg/bucket" "github.com/livekit/mediatransportutil/pkg/twcc" @@ -71,6 +72,7 @@ type TrackReceiver interface { DebugInfo() map[string]interface{} TrackInfo() *livekit.TrackInfo + UpdateTrackInfo(ti *livekit.TrackInfo) // Get primary receiver if this receiver represents a RED codec; otherwise it will return itself GetPrimaryReceiverForRed() TrackReceiver @@ -104,7 +106,7 @@ type WebRTCReceiver struct { closeOnce sync.Once closed atomic.Bool useTrackers bool - trackInfo *livekit.TrackInfo + trackInfo atomic.Pointer[livekit.TrackInfo] rtcpCh chan []rtcp.Packet @@ -200,21 +202,21 @@ func NewWebRTCReceiver( opts ...ReceiverOpts, ) *WebRTCReceiver { w := &WebRTCReceiver{ - logger: logger, - receiver: receiver, - trackID: livekit.TrackID(track.ID()), - streamID: track.StreamID(), - codec: track.Codec(), - kind: track.Kind(), - twcc: twcc, - trackInfo: trackInfo, - isSVC: IsSvcCodec(track.Codec().MimeType), - isRED: IsRedCodec(track.Codec().MimeType), + logger: logger, + receiver: receiver, + trackID: livekit.TrackID(track.ID()), + streamID: track.StreamID(), + codec: track.Codec(), + kind: track.Kind(), + twcc: twcc, + isSVC: IsSvcCodec(track.Codec().MimeType), + isRED: IsRedCodec(track.Codec().MimeType), } for _, opt := range opts { w = opt(w) } + w.trackInfo.Store(proto.Clone(trackInfo).(*livekit.TrackInfo)) w.downTrackSpreader = NewDownTrackSpreader(DownTrackSpreaderParams{ Threshold: w.lbThreshold, @@ -232,7 +234,7 @@ func NewWebRTCReceiver( w.onStatsUpdate(w, stat) } }) - w.connectionStats.Start(w.trackInfo) + w.connectionStats.Start(trackInfo) w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, trackersConfig) w.streamTrackerManager.SetListener(w) @@ -250,7 +252,12 @@ func NewWebRTCReceiver( } func (w *WebRTCReceiver) TrackInfo() *livekit.TrackInfo { - return w.trackInfo + return w.trackInfo.Load() +} + +func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + w.trackInfo.Store(proto.Clone(ti).(*livekit.TrackInfo)) + w.streamTrackerManager.UpdateTrackInfo(ti) } func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) { @@ -328,7 +335,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff layer := int32(0) if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { - layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo) + layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load()) } buff.SetLogger(w.logger.WithValues("layer", layer)) buff.SetTWCC(w.twcc) @@ -700,9 +707,13 @@ func (w *WebRTCReceiver) closeTracks() { } func (w *WebRTCReceiver) DebugInfo() map[string]interface{} { + isSimulcast := !w.isSVC + if ti := w.trackInfo.Load(); ti != nil { + isSimulcast = isSimulcast && len(ti.Layers) > 1 + } info := map[string]interface{}{ "SVC": w.isSVC, - "Simulcast": !w.isSVC && len(w.trackInfo.Layers) > 1, + "Simulcast": isSimulcast, } w.upTrackMu.RLock() diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index c9a22bff9..a934b9e93 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -22,6 +22,8 @@ import ( "time" "github.com/frostbyte73/core" + "go.uber.org/atomic" + "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -57,7 +59,7 @@ type endsSenderReport struct { type StreamTrackerManager struct { logger logger.Logger - trackInfo *livekit.TrackInfo + trackInfo atomic.Pointer[livekit.TrackInfo] isSVC bool clockRate uint32 @@ -92,15 +94,15 @@ func NewStreamTrackerManager( ) *StreamTrackerManager { s := &StreamTrackerManager{ logger: logger, - trackInfo: trackInfo, isSVC: isSVC, maxPublishedLayer: buffer.InvalidLayerSpatial, maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, closed: core.NewFuse(), } + s.trackInfo.Store(proto.Clone(trackInfo).(*livekit.TrackInfo)) - switch s.trackInfo.Source { + switch trackInfo.Source { case livekit.TrackSource_SCREEN_SHARE: s.trackerConfig = trackersConfig.Screenshare case livekit.TrackSource_CAMERA: @@ -111,7 +113,7 @@ func NewStreamTrackerManager( s.maxExpectedLayerFromTrackInfo() - if s.trackInfo.Type == livekit.TrackType_VIDEO { + if trackInfo.Type == livekit.TrackType_VIDEO { go s.bitrateReporter() } return s @@ -316,6 +318,11 @@ func (s *StreamTrackerManager) IsPaused() bool { return s.paused } +func (s *StreamTrackerManager) UpdateTrackInfo(ti *livekit.TrackInfo) { + s.trackInfo.Store(proto.Clone(ti).(*livekit.TrackInfo)) + s.maxExpectedLayerFromTrackInfo() +} + func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 { s.lock.Lock() prev := s.maxExpectedLayer @@ -540,10 +547,13 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { s.maxExpectedLayer = buffer.InvalidLayerSpatial - for _, layer := range s.trackInfo.Layers { - spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, s.trackInfo) - if spatialLayer > s.maxExpectedLayer { - s.maxExpectedLayer = spatialLayer + ti := s.trackInfo.Load() + if ti != nil { + for _, layer := range ti.Layers { + spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, ti) + if spatialLayer > s.maxExpectedLayer { + s.maxExpectedLayer = spatialLayer + } } } } From 5dc87d7ac521c3fb2914c5393337951da4d87406 Mon Sep 17 00:00:00 2001 From: Kou Date: Thu, 21 Dec 2023 16:18:28 +0900 Subject: [PATCH 035/114] Fix panic occurs when starting livekit-server with key-file option (#2312) (#2313) --- pkg/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index a0c882f14..be9920cce 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -652,6 +652,7 @@ func (conf *Config) ValidateKeys() error { _ = f.Close() }() decoder := yaml.NewDecoder(f) + conf.Keys = map[string]string{} if err = decoder.Decode(conf.Keys); err != nil { return err } From a4888fcf8fada3aecfac42cfe2668514ea875c85 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 21 Dec 2023 16:02:10 +0530 Subject: [PATCH 036/114] Prevent unsafe access (hopefully). (#2332) * Prevent unsafe access (hopefully). Thank you @paulwe for catching it. * prevent recursive locks --- pkg/rtc/mediatrackreceiver.go | 54 +++++++++++++++-------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index b5fefa8a3..d04db1a02 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -133,13 +133,11 @@ func NewMediaTrackReceiver(params MediaTrackReceiverParams, ti *livekit.TrackInf } func (t *MediaTrackReceiver) Restart() { - t.lock.Lock() - receivers := t.receivers - ti := t.trackInfo - t.lock.Unlock() + t.lock.RLock() + hq := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.trackInfo) + t.lock.RUnlock() - hq := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, ti) - for _, receiver := range receivers { + for _, receiver := range t.Receivers() { receiver.SetMaxExpectedSpatialLayer(hq) } } @@ -263,7 +261,6 @@ func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { break } } - t.lock.Unlock() t.removeAllSubscribersForMime(mime, willBeResumed) @@ -271,14 +268,12 @@ func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { func (t *MediaTrackReceiver) ClearAllReceivers(willBeResumed bool) { t.params.Logger.Debugw("clearing all receivers") - t.lock.Lock() + t.lock.RLock() var mimes []string for _, receiver := range t.receivers { mimes = append(mimes, receiver.Codec().MimeType) } - - t.receivers = nil - t.lock.Unlock() + t.lock.RUnlock() for _, mime := range mimes { t.ClearReceiver(mime, willBeResumed) @@ -418,9 +413,9 @@ func (t *MediaTrackReceiver) IsMuted() bool { func (t *MediaTrackReceiver) SetMuted(muted bool) { t.lock.Lock() t.trackInfo.Muted = muted - receivers := t.receivers t.lock.Unlock() - for _, receiver := range receivers { + + for _, receiver := range t.Receivers() { receiver.SetUpTrackPaused(muted) } @@ -445,7 +440,7 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su return nil, ErrNotOpen } - receivers := t.receivers + receivers := t.simulcastReceiversLocked() potentialCodecs := make([]webrtc.RTPCodecParameters, len(t.potentialCodecs)) copy(potentialCodecs, t.potentialCodecs) t.lock.RUnlock() @@ -529,11 +524,10 @@ func (t *MediaTrackReceiver) RevokeDisallowedSubscribers(allowedSubscriberIdenti func (t *MediaTrackReceiver) updateTrackInfoOfReceivers() { t.lock.RLock() - receivers := t.receivers - ti := t.trackInfo + ti := proto.Clone(t.trackInfo).(*livekit.TrackInfo) t.lock.RUnlock() - for _, r := range receivers { + for _, r := range t.Receivers() { r.UpdateTrackInfo(ti) } } @@ -821,10 +815,7 @@ func (t *MediaTrackReceiver) DebugInfo() map[string]interface{} { info["DownTracks"] = t.MediaTrackSubscriptions.DebugInfo() - t.lock.RLock() - receivers := t.receivers - t.lock.RUnlock() - for _, receiver := range receivers { + for _, receiver := range t.Receivers() { info[receiver.Codec().MimeType] = receiver.DebugInfo() } @@ -870,13 +861,17 @@ func (t *MediaTrackReceiver) Receivers() []sfu.TrackReceiver { return receivers } -func (t *MediaTrackReceiver) SetRTT(rtt uint32) { - t.lock.RLock() - receivers := t.receivers - t.lock.RUnlock() +func (t *MediaTrackReceiver) simulcastReceiversLocked() []*simulcastReceiver { + receivers := make([]*simulcastReceiver, 0, len(t.receivers)) + for _, r := range t.receivers { + receivers = append(receivers, r) + } + return receivers +} - for _, r := range receivers { - if wr, ok := r.TrackReceiver.(*sfu.WebRTCReceiver); ok { +func (t *MediaTrackReceiver) SetRTT(rtt uint32) { + for _, r := range t.Receivers() { + if wr, ok := r.(*sfu.WebRTCReceiver); ok { wr.SetRTT(rtt) } } @@ -906,10 +901,7 @@ func (t *MediaTrackReceiver) IsEncrypted() bool { } func (t *MediaTrackReceiver) GetTrackStats() *livekit.RTPStats { - t.lock.RLock() - receivers := t.receivers - t.lock.RUnlock() - + receivers := t.Receivers() stats := make([]*livekit.RTPStats, 0, len(receivers)) for _, receiver := range receivers { receiverStats := receiver.GetTrackStats() From 4c1047d8c3b52f9902e4a1f5bfc4776bffcc8a9a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 21 Dec 2023 18:52:49 +0530 Subject: [PATCH 037/114] Populate simulcast codec layers. (#2334) Previously, it was done on read. Missed populating it on write in the TrackInfo consolidation effort. Fix by populating layers when adding pending track itself. As all codecs will have same layers, clone the top level layers and add it all codecs. --- pkg/rtc/participant.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 6f07849b8..b9c4d4942 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1670,9 +1670,15 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l continue } seenCodecs[mime] = struct{}{} + + clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) + for _, l := range req.Layers { + clonedLayers = append(clonedLayers, proto.Clone(l).(*livekit.VideoLayer)) + } ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ MimeType: mime, Cid: codec.Cid, + Layers: clonedLayers, }) } From 01f90d185f5a2de7fcfece86a261bf5b1f1ca295 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 21 Dec 2023 08:23:22 -0800 Subject: [PATCH 038/114] copy receivers on write (#2336) * copy receivers on write * cleanup * cleanup * test --- pkg/rtc/mediatrackreceiver.go | 85 ++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index d04db1a02..3942264ef 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -24,6 +24,7 @@ import ( "github.com/pion/rtcp" "github.com/pion/webrtc/v3" + "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" @@ -137,7 +138,7 @@ func (t *MediaTrackReceiver) Restart() { hq := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.trackInfo) t.lock.RUnlock() - for _, receiver := range t.Receivers() { + for _, receiver := range t.loadReceivers() { receiver.SetMaxExpectedSpatialLayer(hq) } } @@ -156,9 +157,11 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority return } + receivers := slices.Clone(t.receivers) + // codec position maybe taken by DummyReceiver, check and upgrade to WebRTCReceiver var upgradeReceiver bool - for _, r := range t.receivers { + for _, r := range receivers { if strings.EqualFold(r.Codec().MimeType, receiver.Codec().MimeType) { if d, ok := r.TrackReceiver.(*DummyReceiver); ok { d.Upgrade(receiver) @@ -168,11 +171,11 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority } } if !upgradeReceiver { - t.receivers = append(t.receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) + receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) } - sort.Slice(t.receivers, func(i, j int) bool { - return t.receivers[i].Priority() < t.receivers[j].Priority() + sort.Slice(receivers, func(i, j int) bool { + return receivers[i].Priority() < receivers[j].Priority() }) if mid != "" { @@ -195,8 +198,12 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority } } + t.receivers = receivers + onSetupReceiver := t.onSetupReceiver + t.lock.Unlock() + var receiverCodecs []string - for _, r := range t.receivers { + for _, r := range receivers { receiverCodecs = append(receiverCodecs, r.Codec().MimeType) } t.params.Logger.Debugw( @@ -206,8 +213,6 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority "receivers", receiverCodecs, "mid", mid, ) - onSetupReceiver := t.onSetupReceiver - t.lock.Unlock() if onSetupReceiver != nil { onSetupReceiver(receiver.Codec().MimeType) @@ -225,10 +230,11 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete } } t.lock.Lock() + receivers := slices.Clone(t.receivers) t.potentialCodecs = codecs for i, c := range codecs { var exist bool - for _, r := range t.receivers { + for _, r := range receivers { if strings.EqualFold(c.MimeType, r.Codec().MimeType) { exist = true break @@ -239,28 +245,30 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete if !sfu.IsSvcCodec(c.MimeType) { extHeaders = headersWithoutDD } - t.receivers = append(t.receivers, &simulcastReceiver{ + receivers = append(receivers, &simulcastReceiver{ TrackReceiver: NewDummyReceiver(livekit.TrackID(t.trackInfo.Sid), string(t.PublisherID()), c, extHeaders), priority: i, }) } } - sort.Slice(t.receivers, func(i, j int) bool { - return t.receivers[i].Priority() < t.receivers[j].Priority() + sort.Slice(receivers, func(i, j int) bool { + return receivers[i].Priority() < receivers[j].Priority() }) + t.receivers = receivers t.lock.Unlock() } func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { - t.params.Logger.Debugw("clearing receiver", "mime", mime) t.lock.Lock() - for idx, receiver := range t.receivers { + receivers := slices.Clone(t.receivers) + for idx, receiver := range receivers { if strings.EqualFold(receiver.Codec().MimeType, mime) { - t.receivers[idx] = t.receivers[len(t.receivers)-1] - t.receivers = t.receivers[:len(t.receivers)-1] + receivers[idx] = receivers[len(receivers)-1] + receivers = receivers[:len(receivers)-1] break } } + t.receivers = receivers t.lock.Unlock() t.removeAllSubscribersForMime(mime, willBeResumed) @@ -268,15 +276,13 @@ func (t *MediaTrackReceiver) ClearReceiver(mime string, willBeResumed bool) { func (t *MediaTrackReceiver) ClearAllReceivers(willBeResumed bool) { t.params.Logger.Debugw("clearing all receivers") - t.lock.RLock() - var mimes []string - for _, receiver := range t.receivers { - mimes = append(mimes, receiver.Codec().MimeType) - } - t.lock.RUnlock() + t.lock.Lock() + receivers := t.receivers + t.receivers = nil + t.lock.Unlock() - for _, mime := range mimes { - t.ClearReceiver(mime, willBeResumed) + for _, r := range receivers { + t.removeAllSubscribersForMime(r.Codec().MimeType, willBeResumed) } } @@ -415,7 +421,7 @@ func (t *MediaTrackReceiver) SetMuted(muted bool) { t.trackInfo.Muted = muted t.lock.Unlock() - for _, receiver := range t.Receivers() { + for _, receiver := range t.loadReceivers() { receiver.SetUpTrackPaused(muted) } @@ -440,7 +446,7 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su return nil, ErrNotOpen } - receivers := t.simulcastReceiversLocked() + receivers := t.receivers potentialCodecs := make([]webrtc.RTPCodecParameters, len(t.potentialCodecs)) copy(potentialCodecs, t.potentialCodecs) t.lock.RUnlock() @@ -527,7 +533,7 @@ func (t *MediaTrackReceiver) updateTrackInfoOfReceivers() { ti := proto.Clone(t.trackInfo).(*livekit.TrackInfo) t.lock.RUnlock() - for _, r := range t.Receivers() { + for _, r := range t.loadReceivers() { r.UpdateTrackInfo(ti) } } @@ -815,7 +821,7 @@ func (t *MediaTrackReceiver) DebugInfo() map[string]interface{} { info["DownTracks"] = t.MediaTrackSubscriptions.DebugInfo() - for _, receiver := range t.Receivers() { + for _, receiver := range t.loadReceivers() { info[receiver.Codec().MimeType] = receiver.DebugInfo() } @@ -853,25 +859,22 @@ func (t *MediaTrackReceiver) Receiver(mime string) sfu.TrackReceiver { func (t *MediaTrackReceiver) Receivers() []sfu.TrackReceiver { t.lock.RLock() defer t.lock.RUnlock() - - receivers := make([]sfu.TrackReceiver, 0, len(t.receivers)) - for _, r := range t.receivers { - receivers = append(receivers, r.TrackReceiver) + receivers := make([]sfu.TrackReceiver, len(t.receivers)) + for i, r := range t.receivers { + receivers[i] = r.TrackReceiver } return receivers } -func (t *MediaTrackReceiver) simulcastReceiversLocked() []*simulcastReceiver { - receivers := make([]*simulcastReceiver, 0, len(t.receivers)) - for _, r := range t.receivers { - receivers = append(receivers, r) - } - return receivers +func (t *MediaTrackReceiver) loadReceivers() []*simulcastReceiver { + t.lock.RLock() + defer t.lock.RUnlock() + return t.receivers } func (t *MediaTrackReceiver) SetRTT(rtt uint32) { - for _, r := range t.Receivers() { - if wr, ok := r.(*sfu.WebRTCReceiver); ok { + for _, r := range t.loadReceivers() { + if wr, ok := r.TrackReceiver.(*sfu.WebRTCReceiver); ok { wr.SetRTT(rtt) } } @@ -901,7 +904,7 @@ func (t *MediaTrackReceiver) IsEncrypted() bool { } func (t *MediaTrackReceiver) GetTrackStats() *livekit.RTPStats { - receivers := t.Receivers() + receivers := t.loadReceivers() stats := make([]*livekit.RTPStats, 0, len(receivers)) for _, receiver := range receivers { receiverStats := receiver.GetTrackStats() From 7008fdf78b3f1c84a7db24cbfe78517713bb8b3c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:58:20 -0800 Subject: [PATCH 039/114] Update livekit deps (#2274) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index ab3d91800..5e88b072b 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb - github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c + github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f + github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -37,7 +37,7 @@ require ( github.com/pion/webrtc/v3 v3.2.24 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 - github.com/redis/go-redis/v9 v9.3.0 + github.com/redis/go-redis/v9 v9.3.1 github.com/rs/cors v1.10.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.26.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 + golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 golang.org/x/sync v0.5.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 746a2821c..ae4de7f61 100644 --- a/go.sum +++ b/go.sum @@ -124,10 +124,10 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb h1:KiGg4k+kYQD9NjKixaSDMMeYOO2//XBM4IROTI1Itjo= -github.com/livekit/mediatransportutil v0.0.0-20231128042044-05525c8278cb/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c h1:N1nhu8+N70ZuCZ2DLfqNsoLneqV0j2mbqsWSvOHY71w= -github.com/livekit/protocol v1.9.4-0.20231219061222-8fb7e763249c/go.mod h1:acKFhqYltprWHzFV1A8ILARlJnBfwsdHw9HxWQjxTf4= +github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= +github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= +github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371 h1:mF2FpLIPs2CDGQ1QB99Z2USItNhDd+tg4m5+flt4I+Y= +github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371/go.mod h1:pdyn8m58RfiRl/0nA612rAD9eat5/kJI91UFqB/rYEU= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -238,8 +238,8 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= -github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= +github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= @@ -297,8 +297,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE= +golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From 28ae092ea887a598bdacb4657ed150e50657d212 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 21 Dec 2023 23:04:19 -0800 Subject: [PATCH 040/114] version 1.5.2 (#2338) --- CHANGELOG | 26 ++++++++++++++++++++++++++ version/version.go | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 1947140a6..bb789bceb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,32 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.2] - 2023-12-21 + +Support for LiveKit SIP Bridge + +### Added +- Add SIP Support (#2240 #2241 #2244 #2250 #2263 #2291 #2293) +- Introduce `LOST` connection quality. (#2265 #2276) +- Expose detailed connection info with ICEConnectionDetails (#2287) +- Add Version to TrackInfo. (#2324 #2325) + +### Fixed +- Guard against bad quality in trackInfo (#2271) +- Group SDES items for one SSRC in the same chunk. (#2280) +- Avoid dropping data packets on local router (#2270) +- Fix signal response delivery after session start failure (#2294) +- Populate disconnect updates with participant identity (#2310) +- Fix mid info lost when migrating multi-codec simulcast track (#2315) +- Store identity in participant update cache. (#2320) +- Fix panic occurs when starting livekit-server with key-file option (#2312) (#2313) + +### Changed +- INFO logging reduction (#2243 #2273 #2275 #2281 #2283 #2285 #2322) +- Clean up restart a bit. (#2247) +- Use a worker to report signal/data stats. (#2260) +- Consolidate TrackInfo. (#2331) + ## [1.5.1] - 2023-11-09 Support for the Agent framework. diff --git a/version/version.go b/version/version.go index cbd85195e..e821eae6b 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.5.1" +const Version = "1.5.2" From 26c96ec283412a1006897beff533e7382bb71258 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 22 Dec 2023 17:09:49 +0530 Subject: [PATCH 041/114] Synthesise codec when adding pending track for no simulcast case also. (#2339) * Synthesise codec when adding pending track for no simulcast case also. Older clients not using simulcast codecs were failing e2e migration tests. Problem is that they did not have layer information and hence SSRC could not be set on migration. A codec was getting added later (when OnTrack was received). I missed adding layers in that code. Could have cloned layers there and added it. But, simplifying and adding at the start itself. Also, cleaning up code in `MediaTrackReceiver` for no codecs case as it should not happen any more. * clone per layer * fix priority determination --- pkg/rtc/mediatrack.go | 13 ++++++- pkg/rtc/mediatrackreceiver.go | 63 ++++-------------------------- pkg/rtc/participant.go | 73 ++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 89 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 73796cb73..7a3ca64fc 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -221,8 +221,17 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra break } } - if len(ti.Codecs) == 0 { - priority = 0 + if priority < 0 { + switch len(ti.Codecs) { + case 0: + // audio track + priority = 0 + case 1: + // older clients or non simulcast-codec, mime type only set later + if ti.Codecs[0].MimeType == "" { + priority = 0 + } + } } if priority < 0 { t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mime, "track", logger.Proto(ti)) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 3942264ef..b4128f690 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -182,11 +182,6 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority if priority == 0 { t.trackInfo.MimeType = receiver.Codec().MimeType t.trackInfo.Mid = mid - - // for clients don't have simulcast codecs (old version or single codec), add the primary codec - if len(t.trackInfo.Codecs) == 0 && t.trackInfo.Type == livekit.TrackType_VIDEO { - t.trackInfo.Codecs = append(t.trackInfo.Codecs, &livekit.SimulcastCodecInfo{}) - } } for i, ci := range t.trackInfo.Codecs { @@ -547,7 +542,7 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) } quality := buffer.SpatialLayerToVideoQuality(layer, t.trackInfo) // set video layer ssrc info - for _, ci := range t.trackInfo.Codecs { + for i, ci := range t.trackInfo.Codecs { if !strings.EqualFold(ci.MimeType, mime) { continue } @@ -567,26 +562,12 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) if !ssrcFound && matchingLayer != nil { matchingLayer.Ssrc = ssrc } - break - } - // for client don't use simulcast codecs (old client version or single codec) - if len(t.trackInfo.Codecs) == 0 { - // if origin layer has ssrc, don't override it - var matchingLayer *livekit.VideoLayer - ssrcFound := false - for _, l := range t.trackInfo.Layers { - if l.Quality == quality { - matchingLayer = l - if l.Ssrc != 0 { - ssrcFound = true - } - break - } - } - if !ssrcFound && matchingLayer != nil { - matchingLayer.Ssrc = ssrc + // for client don't use simulcast codecs (old client version or single codec) + if i == 0 { + t.trackInfo.Layers = ci.Layers } + break } t.lock.Unlock() @@ -636,25 +617,11 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { break } + // for client don't use simulcast codecs (old client version or single codec) if i == 0 { clonedInfo.Layers = ci.Layers } } - - // for client don't use simulcast codecs (old client version or single codec) - if len(clonedInfo.Codecs) == 0 { - for _, layer := range clonedInfo.Layers { - for _, originLayer := range t.trackInfo.Layers { - if layer.Quality == originLayer.Quality { - if originLayer.Ssrc != 0 { - layer.Ssrc = originLayer.Ssrc - } - break - } - } - } - } - t.trackInfo = clonedInfo t.lock.Unlock() @@ -679,27 +646,11 @@ func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { } } + // for client don't use simulcast codecs (old client version or single codec) if i == 0 { t.trackInfo.Layers = ci.Layers } } - - // for client don't use simulcast codecs (old client version or single codec) - if len(t.trackInfo.Codecs) == 0 { - originLayers := t.trackInfo.Layers - t.trackInfo.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - t.trackInfo.Layers = append(t.trackInfo.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - for _, l := range originLayers { - if l.Quality == t.trackInfo.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - t.trackInfo.Layers[layerIdx].Ssrc = l.Ssrc - } - break - } - } - } - } t.lock.Unlock() t.updateTrackInfoOfReceivers() diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b9c4d4942..154d7c794 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1645,41 +1645,52 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l ti.Stream = StreamFromTrackSource(ti.Source) } p.setStableTrackID(req.Cid, ti) - seenCodecs := make(map[string]struct{}) - for _, codec := range req.SimulcastCodecs { - mime := codec.Codec + + if len(req.SimulcastCodecs) == 0 { if req.Type == livekit.TrackType_VIDEO { - if !strings.HasPrefix(mime, "video/") { - mime = "video/" + mime + // clients not supporting simulcast codecs, synthesise a codec + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + Cid: req.Cid, + Layers: req.Layers, + }) + } + } else { + seenCodecs := make(map[string]struct{}) + for _, codec := range req.SimulcastCodecs { + mime := codec.Codec + if req.Type == livekit.TrackType_VIDEO { + if !strings.HasPrefix(mime, "video/") { + mime = "video/" + mime + } + if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { + altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) + p.pubLogger.Infow("falling back to alternative codec", + "codec", mime, + "altCodec", altCodec, + "trackID", ti.Sid, + ) + // select an alternative MIME type that's generally supported + mime = altCodec + } + } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { + mime = "audio/" + mime } - if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { - altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) - p.pubLogger.Infow("falling back to alternative codec", - "codec", mime, - "altCodec", altCodec, - "trackID", ti.Sid, - ) - // select an alternative MIME type that's generally supported - mime = altCodec + + if _, ok := seenCodecs[mime]; ok || mime == "" { + continue } - } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { - mime = "audio/" + mime - } + seenCodecs[mime] = struct{}{} - if _, ok := seenCodecs[mime]; ok || mime == "" { - continue + clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) + for _, l := range req.Layers { + clonedLayers = append(clonedLayers, proto.Clone(l).(*livekit.VideoLayer)) + } + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + MimeType: mime, + Cid: codec.Cid, + Layers: clonedLayers, + }) } - seenCodecs[mime] = struct{}{} - - clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) - for _, l := range req.Layers { - clonedLayers = append(clonedLayers, proto.Clone(l).(*livekit.VideoLayer)) - } - ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mime, - Cid: codec.Cid, - Layers: clonedLayers, - }) } p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), ti) @@ -1901,7 +1912,7 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo for _, codec := range ti.Codecs { for ssrc, info := range p.params.SimTracks { if info.Mid == codec.Mid { - mt.MediaTrackReceiver.SetLayerSsrc(codec.MimeType, info.Rid, ssrc) + mt.SetLayerSsrc(codec.MimeType, info.Rid, ssrc) } } } From 3770fbce6412cfd9bc7f7b629f9af3544c4dfa1a Mon Sep 17 00:00:00 2001 From: shishirng Date: Fri, 22 Dec 2023 18:59:04 -0500 Subject: [PATCH 042/114] Analytics: send local node room state/info (#2335) * Analytics: send local node room state/info Signed-off-by: shishir gowda --- go.mod | 6 +- go.sum | 12 +-- pkg/telemetry/analyticsservice.go | 27 +++++- .../telemetryfakes/fake_analytics_service.go | 41 ++++++++++ .../telemetryfakes/fake_telemetry_service.go | 82 +++++++++++++++++++ pkg/telemetry/telemetryservice.go | 7 ++ 6 files changed, 162 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 5e88b072b..0b61459ab 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371 + github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -48,7 +48,7 @@ require ( go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 golang.org/x/sync v0.5.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -61,7 +61,7 @@ require ( github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.1 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/subcommands v1.2.0 // indirect diff --git a/go.sum b/go.sum index ae4de7f61..abcfd8611 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44 github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371 h1:mF2FpLIPs2CDGQ1QB99Z2USItNhDd+tg4m5+flt4I+Y= -github.com/livekit/protocol v1.9.4-0.20231221100346-e9e5fcd7d371/go.mod h1:pdyn8m58RfiRl/0nA612rAD9eat5/kJI91UFqB/rYEU= +github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa h1:qIzXYbpCR01Czwe/j2HYFzHp3j3VSCavdR+R2+qSmS4= +github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa/go.mod h1:qL0J9HZaUrimDs29b/uRARvWn1cqjbvXUhayZ02RF9U= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -433,8 +433,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index 8611337f1..3f1955873 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -17,6 +17,9 @@ package telemetry import ( "context" + "go.uber.org/atomic" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -28,14 +31,17 @@ import ( type AnalyticsService interface { SendStats(ctx context.Context, stats []*livekit.AnalyticsStat) SendEvent(ctx context.Context, events *livekit.AnalyticsEvent) + SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms) } type analyticsService struct { - analyticsKey string - nodeID string + analyticsKey string + nodeID string + sequenceNumber atomic.Uint64 - events livekit.AnalyticsRecorderService_IngestEventsClient - stats livekit.AnalyticsRecorderService_IngestStatsClient + events livekit.AnalyticsRecorderService_IngestEventsClient + stats livekit.AnalyticsRecorderService_IngestStatsClient + nodeRooms livekit.AnalyticsRecorderService_IngestNodeRoomStatesClient } func NewAnalyticsService(_ *config.Config, currentNode routing.LocalNode) AnalyticsService { @@ -71,3 +77,16 @@ func (a *analyticsService) SendEvent(_ context.Context, event *livekit.Analytics logger.Errorw("failed to send event", err, "eventType", event.Type.String()) } } + +func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *livekit.AnalyticsNodeRooms) { + if a.nodeRooms == nil { + return + } + + nodeRooms.NodeId = a.nodeID + nodeRooms.SequenceNumber = a.sequenceNumber.Add(1) + nodeRooms.Timestamp = timestamppb.Now() + if err := a.nodeRooms.Send(nodeRooms); err != nil { + logger.Errorw("failed to send node room states", err) + } +} diff --git a/pkg/telemetry/telemetryfakes/fake_analytics_service.go b/pkg/telemetry/telemetryfakes/fake_analytics_service.go index 5af3a3513..21b2bb6a1 100644 --- a/pkg/telemetry/telemetryfakes/fake_analytics_service.go +++ b/pkg/telemetry/telemetryfakes/fake_analytics_service.go @@ -16,6 +16,12 @@ type FakeAnalyticsService struct { arg1 context.Context arg2 *livekit.AnalyticsEvent } + SendNodeRoomStatesStub func(context.Context, *livekit.AnalyticsNodeRooms) + sendNodeRoomStatesMutex sync.RWMutex + sendNodeRoomStatesArgsForCall []struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + } SendStatsStub func(context.Context, []*livekit.AnalyticsStat) sendStatsMutex sync.RWMutex sendStatsArgsForCall []struct { @@ -59,6 +65,39 @@ func (fake *FakeAnalyticsService) SendEventArgsForCall(i int) (context.Context, return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeAnalyticsService) SendNodeRoomStates(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) { + fake.sendNodeRoomStatesMutex.Lock() + fake.sendNodeRoomStatesArgsForCall = append(fake.sendNodeRoomStatesArgsForCall, struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + }{arg1, arg2}) + stub := fake.SendNodeRoomStatesStub + fake.recordInvocation("SendNodeRoomStates", []interface{}{arg1, arg2}) + fake.sendNodeRoomStatesMutex.Unlock() + if stub != nil { + fake.SendNodeRoomStatesStub(arg1, arg2) + } +} + +func (fake *FakeAnalyticsService) SendNodeRoomStatesCallCount() int { + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() + return len(fake.sendNodeRoomStatesArgsForCall) +} + +func (fake *FakeAnalyticsService) SendNodeRoomStatesCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) { + fake.sendNodeRoomStatesMutex.Lock() + defer fake.sendNodeRoomStatesMutex.Unlock() + fake.SendNodeRoomStatesStub = stub +} + +func (fake *FakeAnalyticsService) SendNodeRoomStatesArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) { + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() + argsForCall := fake.sendNodeRoomStatesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeAnalyticsService) SendStats(arg1 context.Context, arg2 []*livekit.AnalyticsStat) { var arg2Copy []*livekit.AnalyticsStat if arg2 != nil { @@ -102,6 +141,8 @@ func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.sendEventMutex.RLock() defer fake.sendEventMutex.RUnlock() + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() fake.sendStatsMutex.RLock() defer fake.sendStatsMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index fd3ae6ff5..3be71676b 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -62,6 +62,12 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.IngressInfo } + LocalRoomStateStub func(context.Context, *livekit.AnalyticsNodeRooms) + localRoomStateMutex sync.RWMutex + localRoomStateArgsForCall []struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + } NotifyEventStub func(context.Context, *livekit.WebhookEvent) notifyEventMutex sync.RWMutex notifyEventArgsForCall []struct { @@ -122,6 +128,12 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.AnalyticsEvent } + SendNodeRoomStatesStub func(context.Context, *livekit.AnalyticsNodeRooms) + sendNodeRoomStatesMutex sync.RWMutex + sendNodeRoomStatesArgsForCall []struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + } SendStatsStub func(context.Context, []*livekit.AnalyticsStat) sendStatsMutex sync.RWMutex sendStatsArgsForCall []struct { @@ -533,6 +545,39 @@ func (fake *FakeTelemetryService) IngressUpdatedArgsForCall(i int) (context.Cont return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) LocalRoomState(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) { + fake.localRoomStateMutex.Lock() + fake.localRoomStateArgsForCall = append(fake.localRoomStateArgsForCall, struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + }{arg1, arg2}) + stub := fake.LocalRoomStateStub + fake.recordInvocation("LocalRoomState", []interface{}{arg1, arg2}) + fake.localRoomStateMutex.Unlock() + if stub != nil { + fake.LocalRoomStateStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) LocalRoomStateCallCount() int { + fake.localRoomStateMutex.RLock() + defer fake.localRoomStateMutex.RUnlock() + return len(fake.localRoomStateArgsForCall) +} + +func (fake *FakeTelemetryService) LocalRoomStateCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) { + fake.localRoomStateMutex.Lock() + defer fake.localRoomStateMutex.Unlock() + fake.LocalRoomStateStub = stub +} + +func (fake *FakeTelemetryService) LocalRoomStateArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) { + fake.localRoomStateMutex.RLock() + defer fake.localRoomStateMutex.RUnlock() + argsForCall := fake.localRoomStateArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) NotifyEvent(arg1 context.Context, arg2 *livekit.WebhookEvent) { fake.notifyEventMutex.Lock() fake.notifyEventArgsForCall = append(fake.notifyEventArgsForCall, struct { @@ -809,6 +854,39 @@ func (fake *FakeTelemetryService) SendEventArgsForCall(i int) (context.Context, return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) SendNodeRoomStates(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) { + fake.sendNodeRoomStatesMutex.Lock() + fake.sendNodeRoomStatesArgsForCall = append(fake.sendNodeRoomStatesArgsForCall, struct { + arg1 context.Context + arg2 *livekit.AnalyticsNodeRooms + }{arg1, arg2}) + stub := fake.SendNodeRoomStatesStub + fake.recordInvocation("SendNodeRoomStates", []interface{}{arg1, arg2}) + fake.sendNodeRoomStatesMutex.Unlock() + if stub != nil { + fake.SendNodeRoomStatesStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) SendNodeRoomStatesCallCount() int { + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() + return len(fake.sendNodeRoomStatesArgsForCall) +} + +func (fake *FakeTelemetryService) SendNodeRoomStatesCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) { + fake.sendNodeRoomStatesMutex.Lock() + defer fake.sendNodeRoomStatesMutex.Unlock() + fake.SendNodeRoomStatesStub = stub +} + +func (fake *FakeTelemetryService) SendNodeRoomStatesArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) { + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() + argsForCall := fake.sendNodeRoomStatesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) SendStats(arg1 context.Context, arg2 []*livekit.AnalyticsStat) { var arg2Copy []*livekit.AnalyticsStat if arg2 != nil { @@ -1359,6 +1437,8 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.ingressStartedMutex.RUnlock() fake.ingressUpdatedMutex.RLock() defer fake.ingressUpdatedMutex.RUnlock() + fake.localRoomStateMutex.RLock() + defer fake.localRoomStateMutex.RUnlock() fake.notifyEventMutex.RLock() defer fake.notifyEventMutex.RUnlock() fake.participantActiveMutex.RLock() @@ -1375,6 +1455,8 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.roomStartedMutex.RUnlock() fake.sendEventMutex.RLock() defer fake.sendEventMutex.RUnlock() + fake.sendNodeRoomStatesMutex.RLock() + defer fake.sendNodeRoomStatesMutex.RUnlock() fake.sendStatsMutex.RLock() defer fake.sendStatsMutex.RUnlock() fake.trackMaxSubscribedVideoQualityMutex.RLock() diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index c74619eda..572bba118 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -73,6 +73,7 @@ type TelemetryService interface { IngressStarted(ctx context.Context, info *livekit.IngressInfo) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) IngressEnded(ctx context.Context, info *livekit.IngressInfo) + LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) // helpers AnalyticsService @@ -187,3 +188,9 @@ func (t *telemetryService) cleanupWorkers() { } } } + +func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) { + t.enqueue(func() { + t.SendNodeRoomStates(ctx, info) + }) +} From 6cac17affe52d498c8b07ce3108044fe19027e2a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 23 Dec 2023 18:55:24 +0530 Subject: [PATCH 043/114] Add some debug logs around track publish (#2340) --- pkg/rtc/mediatrack.go | 8 +++++++- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/participant.go | 5 +++-- pkg/sfu/downtrack.go | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 7a3ca64fc..3cc45dba6 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -211,7 +211,13 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra t.lock.Lock() mime := strings.ToLower(track.Codec().MimeType) layer := buffer.RidToSpatialLayer(track.RID(), ti) - t.params.Logger.Debugw("AddReceiver", "mime", track.Codec().MimeType) + t.params.Logger.Debugw( + "AddReceiver", + "mime", track.Codec().MimeType, + "rid", track.RID(), + "layer", layer, + "ssrc", track.SSRC(), + ) wr := t.MediaTrackReceiver.Receiver(mime) if wr == nil { priority := -1 diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index b4128f690..3c9db5616 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -186,8 +186,8 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority for i, ci := range t.trackInfo.Codecs { if i == priority { - ci.Mid = mid ci.MimeType = receiver.Codec().MimeType + ci.Mid = mid break } } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 154d7c794..41e7a31a7 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1833,7 +1833,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } } if codecFound != len(ti.Codecs) { - p.params.Logger.Warnw("migrated track codec mismatched", nil, "track", logger.Proto(ti), "webrtcCodec", parameters) + p.pubLogger.Warnw("migrated track codec mismatched", nil, "track", logger.Proto(ti), "webrtcCodec", parameters) p.pendingTracksLock.Unlock() p.IssueFullReconnect(types.ParticipantCloseReasonMigrateCodecMismatch) return nil, false @@ -1862,6 +1862,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei if mt.AddReceiver(rtpReceiver, track, p.twcc, mid) { p.removeMutedTrackNotFired(mt) if newTrack { + p.pubLogger.Debugw("track published", nil, "trackID", mt.ID(), "track", logger.Proto(mt.ToProto())) go p.handleTrackPublished(mt) } } @@ -1993,7 +1994,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv if !p.IsClosed() { // unpublished events aren't necessary when participant is closed - p.pubLogger.Infow("unpublished track", "trackID", ti.Sid, "trackInfo", ti) + p.pubLogger.Debugw("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti)) p.lock.RLock() onTrackUnpublished := p.onTrackUnpublished p.lock.RUnlock() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index a2ef60926..4a0533a73 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -458,7 +458,7 @@ func (d *DownTrack) SetStreamAllocatorListener(listener DownTrackStreamAllocator d.transportWideExtID = 0 } - // kick of a gratuitous allocation + // kick off a gratuitous allocation listener.OnSubscriptionChanged(d) } } From ee1a167c3e7393fba90ba71507d664c623383829 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 24 Dec 2023 12:25:05 +0530 Subject: [PATCH 044/114] Correct logger field (#2341) --- pkg/rtc/participant.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 41e7a31a7..285963286 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1862,7 +1862,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei if mt.AddReceiver(rtpReceiver, track, p.twcc, mid) { p.removeMutedTrackNotFired(mt) if newTrack { - p.pubLogger.Debugw("track published", nil, "trackID", mt.ID(), "track", logger.Proto(mt.ToProto())) + p.pubLogger.Debugw("track published", "trackID", mt.ID(), "track", logger.Proto(mt.ToProto())) go p.handleTrackPublished(mt) } } From f5e2c1da15cd971ee0818fb8fe7635600cb699fb Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 24 Dec 2023 14:03:51 +0530 Subject: [PATCH 045/114] Debug log downtrack life cycle a bit (#2342) --- pkg/sfu/downtrack.go | 1 + pkg/sfu/receiver.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 4a0533a73..57da4a30f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1579,6 +1579,7 @@ func (d *DownTrack) SetConnected() { if !d.connected.Swap(true) { d.onBindAndConnectedChange() } + d.params.Logger.Debugw("downtrack connected") } // SetActivePaddingOnMuteUpTrack will enable padding on the track when its uptrack is muted. diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index e555b99f9..ae368b42f 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -421,6 +421,7 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen()) w.downTrackSpreader.Store(track) + w.logger.Debugw("downtrack added", "subscriberID", track.SubscriberID()) return nil } @@ -505,6 +506,7 @@ func (w *WebRTCReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) { } w.downTrackSpreader.Free(subscriberID) + w.logger.Debugw("downtrack deleted", "subscriberID", subscriberID) } func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) { From bdcd142c0d951fecc92414958ed188ae3e5b4b42 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 25 Dec 2023 14:12:08 +0530 Subject: [PATCH 046/114] Adding some logs in subscribe path. (#2343) Trying to chase down an older client failing to subscribe some times. --- pkg/rtc/mediatracksubscriptions.go | 12 ++++++++++++ pkg/rtc/subscriptionmanager.go | 21 ++++++++++++++++++--- pkg/sfu/downtrack.go | 1 + 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 15db1a8a3..0a619bb99 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -207,6 +207,12 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * replacedTrack := false existingTransceiver, dtState = sub.GetCachedDownTrack(trackID) if existingTransceiver != nil { + sub.GetLogger().Debugw( + "trying to use existing transceiver", + "publisher", subTrack.PublisherIdentity(), + "publisherID", subTrack.PublisherID(), + "trackID", trackID, + ) reusingTransceiver.Store(true) rtpSender := existingTransceiver.Sender() if rtpSender != nil { @@ -217,6 +223,12 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * sender = rtpSender transceiver = existingTransceiver replacedTrack = true + sub.GetLogger().Debugw( + "track replaced", + "publisher", subTrack.PublisherIdentity(), + "publisherID", subTrack.PublisherID(), + "trackID", trackID, + ) } } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index eec97e026..78ce71c6b 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -505,8 +505,17 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { subTrack, err := track.AddSubscriber(m.params.Participant) if err != nil && err != errAlreadySubscribed { // ignore already subscribed error + m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) return err } + if err == errAlreadySubscribed { + m.params.Logger.Debugw( + "already subscribed to track", + "trackID", trackID, + "subscribedAudioCount", m.subscribedAudioCount.Load(), + "subscribedVideoCount", m.subscribedVideoCount.Load(), + ) + } if err == nil && subTrack != nil { // subTrack could be nil if already subscribed subTrack.OnClose(func(willBeResumed bool) { m.handleSubscribedTrackClose(s, willBeResumed) @@ -536,9 +545,14 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { } go m.params.OnTrackSubscribed(subTrack) - } - m.params.Logger.Debugw("subscribed to track", "trackID", trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) + m.params.Logger.Debugw( + "subscribed to track", + "trackID", trackID, + "subscribedAudioCount", m.subscribedAudioCount.Load(), + "subscribedVideoCount", m.subscribedVideoCount.Load(), + ) + } // add mark the participant as someone we've subscribed to firstSubscribe := false @@ -596,7 +610,8 @@ func (m *SubscriptionManager) handleSourceTrackRemoved(trackID livekit.TrackID) // - UpTrack was closed // - publisher revoked permissions for the participant func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, willBeResumed bool) { - s.logger.Debugw("subscribed track closed", + s.logger.Debugw( + "subscribed track closed", "willBeResumed", willBeResumed, ) wasBound := s.isBound() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 57da4a30f..696b3a8c8 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -351,6 +351,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { go d.maxLayerNotifierWorker() go d.keyFrameRequester() } + d.params.Logger.Debugw("downtrack created") return d, nil } From 5429edd476c06848af68542c3395573251083b1c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 25 Dec 2023 23:03:39 +0530 Subject: [PATCH 047/114] Record node selection reason. (#2346) --- pkg/routing/interfaces.go | 9 +++++---- pkg/service/rtcservice.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 9411ed2c3..37985bb9d 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -108,10 +108,11 @@ type Router interface { } type StartParticipantSignalResults struct { - ConnectionID livekit.ConnectionID - RequestSink MessageSink - ResponseSource MessageSource - NodeID livekit.NodeID + ConnectionID livekit.ConnectionID + RequestSink MessageSink + ResponseSource MessageSource + NodeID livekit.NodeID + NodeSelectionReason string } type MessageRouter interface { diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index a273edecf..064fc3101 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -312,6 +312,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { "reconnectReason", pi.ReconnectReason, "adaptiveStream", pi.AdaptiveStream, "selectedNodeID", cr.NodeID, + "nodeSelectionReason", cr.NodeSelectionReason, ) // handle responses From bdfc684cd7b4d7bd828c4e1f6bf7c808649ab8ba Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 25 Dec 2023 23:05:59 +0530 Subject: [PATCH 048/114] Prevent race of new track and new receiver. (#2345) * Prevent race of new track and new receiver. Two different concepts 1. Creation of a new media track 2. Creation of a new receiver inside the media track collided and caused track published to not be fired. Unify to mark creation of new receiver as the source of truth. With simulcast codecs, creation of a new receiver should be treated as a new published track. * Fire onTrackPublished only on new track --- pkg/rtc/participant.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 285963286..b6d63e334 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1861,10 +1861,17 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei if mt.AddReceiver(rtpReceiver, track, p.twcc, mid) { p.removeMutedTrackNotFired(mt) - if newTrack { - p.pubLogger.Debugw("track published", "trackID", mt.ID(), "track", logger.Proto(mt.ToProto())) - go p.handleTrackPublished(mt) - } + } + + if newTrack { + go func() { + p.pubLogger.Debugw( + "track published", + "trackID", mt.ID(), + "track", logger.Proto(mt.ToProto()), + ) + p.handleTrackPublished(mt) + }() } return mt, newTrack From 28ee3f4007926cdc9345779ad86ac1d985cdc031 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 18:01:42 -0800 Subject: [PATCH 049/114] Update golang.org/x/exp digest to 02704c9 (#2326) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0b61459ab..956ef39da 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.26.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 + golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/sync v0.5.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index abcfd8611..2bf32b52f 100644 --- a/go.sum +++ b/go.sum @@ -297,8 +297,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE= -golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From fcee6edcb246435f20b2f5776dc14a50a5d1906e Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 26 Dec 2023 11:40:34 +0800 Subject: [PATCH 050/114] change svc frame number log to debug (#2347) --- pkg/sfu/videolayerselector/framenumberwrapper.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/sfu/videolayerselector/framenumberwrapper.go b/pkg/sfu/videolayerselector/framenumberwrapper.go index 7942485e0..d2821db80 100644 --- a/pkg/sfu/videolayerselector/framenumberwrapper.go +++ b/pkg/sfu/videolayerselector/framenumberwrapper.go @@ -32,8 +32,7 @@ func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64 prevOffset := f.offset f.offset += uint64(65535 - diff + 6000) - // TODO: remove this - f.logger.Infow("wrap around frame number seen, update offset", "new", new, "last", f.last, "offset", f.offset, "prevOffset", prevOffset, "lastWrapFn", last16, "newWrapFn", new16) + f.logger.Debugw("wrap around frame number seen, update offset", "new", new, "last", f.last, "offset", f.offset, "prevOffset", prevOffset, "lastWrapFn", last16, "newWrapFn", new16) } } f.last = new From ffac7561c894f9a26f0ae0d8cbd370ee749302ab Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:41:32 -0800 Subject: [PATCH 051/114] Update module github.com/urfave/cli/v2 to v2.27.0 (#2348) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 956ef39da..9c0d0b0f5 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc - github.com/urfave/cli/v2 v2.26.0 + github.com/urfave/cli/v2 v2.27.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b diff --git a/go.sum b/go.sum index 2bf32b52f..3ae94fb80 100644 --- a/go.sum +++ b/go.sum @@ -267,8 +267,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc h1:iT5lwxf894PiMq7cnMMQg/7VOD1pxmu//gQuHWAFy4s= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= -github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= -github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.0 h1:uNs1K8JwTFL84X68j5Fjny6hfANh9nTlJ6dRtZAFAHY= +github.com/urfave/cli/v2 v2.27.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 15500d8a1824b0029557774548afe08c5e3fa90a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 27 Dec 2023 17:40:59 +0530 Subject: [PATCH 052/114] Add padding and lost packets to traffic stats (#2349) * Add padding and lost packets to traffic stats * aggregate padding packets --- pkg/rtc/participant_traffic_load.go | 2 +- pkg/rtc/types/trafficstats.go | 50 +++++++++++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/pkg/rtc/participant_traffic_load.go b/pkg/rtc/participant_traffic_load.go index eef44ad5e..87b8b9ef2 100644 --- a/pkg/rtc/participant_traffic_load.go +++ b/pkg/rtc/participant_traffic_load.go @@ -144,7 +144,7 @@ func (p *ParticipantTrafficLoad) updateTrafficLoad() *types.TrafficLoad { trafficTypeStats := make([]*types.TrafficTypeStats, 0, 6) addTypeStats := func(statsList []*types.TrafficStats, trackType livekit.TrackType, streamType livekit.StreamType) { - agg := types.AggregateTrafficStats(statsList) + agg := types.AggregateTrafficStats(statsList...) if agg != nil { trafficTypeStats = append(trafficTypeStats, &types.TrafficTypeStats{ TrackType: trackType, diff --git a/pkg/rtc/types/trafficstats.go b/pkg/rtc/types/trafficstats.go index 70b269db8..d6021958e 100644 --- a/pkg/rtc/types/trafficstats.go +++ b/pkg/rtc/types/trafficstats.go @@ -21,10 +21,12 @@ import ( ) type TrafficStats struct { - StartTime time.Time - EndTime time.Time - Packets uint32 - Bytes uint64 + StartTime time.Time + EndTime time.Time + Packets uint32 + PacketsLost uint32 + PacketsPadding uint32 + Bytes uint64 } type TrafficTypeStats struct { @@ -49,22 +51,30 @@ func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { if before == nil { return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets, - Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets, + PacketsLost: after.PacketsLost, + PacketsPadding: after.PacketsPadding, + Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, } } + packetsLost := uint32(0) + if after.PacketsLost >= before.PacketsLost { + packetsLost = after.PacketsLost - before.PacketsLost + } return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets - before.Packets, - Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets - before.Packets, + PacketsLost: packetsLost, + PacketsPadding: after.PacketsPadding - before.PacketsPadding, + Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), } } -func AggregateTrafficStats(statsList []*TrafficStats) *TrafficStats { +func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { if len(statsList) == 0 { return nil } @@ -73,6 +83,8 @@ func AggregateTrafficStats(statsList []*TrafficStats) *TrafficStats { endTime := time.Time{} packets := uint32(0) + packetsLost := uint32(0) + packetsPadding := uint32(0) bytes := uint64(0) for _, stats := range statsList { @@ -85,6 +97,8 @@ func AggregateTrafficStats(statsList []*TrafficStats) *TrafficStats { } packets += stats.Packets + packetsLost += stats.PacketsLost + packetsPadding += stats.PacketsPadding bytes += stats.Bytes } @@ -92,10 +106,12 @@ func AggregateTrafficStats(statsList []*TrafficStats) *TrafficStats { endTime = time.Now() } return &TrafficStats{ - StartTime: startTime, - EndTime: endTime, - Packets: packets, - Bytes: bytes, + StartTime: startTime, + EndTime: endTime, + Packets: packets, + PacketsLost: packetsLost, + PacketsPadding: packetsPadding, + Bytes: bytes, } } From 5eea679589d3ad0c58ab501d03b3593e37f045dd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 27 Dec 2023 23:15:24 +0530 Subject: [PATCH 053/114] Include packets out-of-order in TrafficStats. (#2350) PacketsLost may not provide useful if repairs are discounting the loss. So, out-of-order packets are an indication of loss and maybe subsequent repair. Note that out-of-order could be just out-of-order by a short amount of time, but a lot of that happening is not good either. So, out-of-order could provide a decent view of link quality. --- pkg/rtc/types/trafficstats.go | 54 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/pkg/rtc/types/trafficstats.go b/pkg/rtc/types/trafficstats.go index d6021958e..06e43786a 100644 --- a/pkg/rtc/types/trafficstats.go +++ b/pkg/rtc/types/trafficstats.go @@ -21,12 +21,13 @@ import ( ) type TrafficStats struct { - StartTime time.Time - EndTime time.Time - Packets uint32 - PacketsLost uint32 - PacketsPadding uint32 - Bytes uint64 + StartTime time.Time + EndTime time.Time + Packets uint32 + PacketsLost uint32 + PacketsPadding uint32 + PacketsOutOfOrder uint32 + Bytes uint64 } type TrafficTypeStats struct { @@ -51,12 +52,13 @@ func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { if before == nil { return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets, - PacketsLost: after.PacketsLost, - PacketsPadding: after.PacketsPadding, - Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets, + PacketsLost: after.PacketsLost, + PacketsPadding: after.PacketsPadding, + PacketsOutOfOrder: after.PacketsOutOfOrder, + Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, } } @@ -65,12 +67,13 @@ func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { packetsLost = after.PacketsLost - before.PacketsLost } return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets - before.Packets, - PacketsLost: packetsLost, - PacketsPadding: after.PacketsPadding - before.PacketsPadding, - Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets - before.Packets, + PacketsLost: packetsLost, + PacketsPadding: after.PacketsPadding - before.PacketsPadding, + PacketsOutOfOrder: after.PacketsOutOfOrder - before.PacketsOutOfOrder, + Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), } } @@ -85,6 +88,7 @@ func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { packets := uint32(0) packetsLost := uint32(0) packetsPadding := uint32(0) + packetsOutOfOrder := uint32(0) bytes := uint64(0) for _, stats := range statsList { @@ -99,6 +103,7 @@ func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { packets += stats.Packets packetsLost += stats.PacketsLost packetsPadding += stats.PacketsPadding + packetsOutOfOrder += stats.PacketsOutOfOrder bytes += stats.Bytes } @@ -106,12 +111,13 @@ func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { endTime = time.Now() } return &TrafficStats{ - StartTime: startTime, - EndTime: endTime, - Packets: packets, - PacketsLost: packetsLost, - PacketsPadding: packetsPadding, - Bytes: bytes, + StartTime: startTime, + EndTime: endTime, + Packets: packets, + PacketsLost: packetsLost, + PacketsPadding: packetsPadding, + PacketsOutOfOrder: packetsOutOfOrder, + Bytes: bytes, } } From 2f1a2ff39da53340d6e828e58f2c493ef02309e1 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 28 Dec 2023 01:16:59 +0530 Subject: [PATCH 054/114] Protect against stats getting reset. (#2351) A reset would make `after` look like it is `before` and the diff will be large unsigned numbers. --- pkg/rtc/types/trafficstats.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/types/trafficstats.go b/pkg/rtc/types/trafficstats.go index 06e43786a..f5c0b386d 100644 --- a/pkg/rtc/types/trafficstats.go +++ b/pkg/rtc/types/trafficstats.go @@ -50,7 +50,7 @@ func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { startTime = before.EndTime } - if before == nil { + getAfter := func() *TrafficStats { return &TrafficStats{ StartTime: startTime.AsTime(), EndTime: after.EndTime.AsTime(), @@ -62,6 +62,19 @@ func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { } } + if before == nil { + return getAfter() + } + + if (after.Packets - before.Packets) > (1 << 31) { + // after packets < before packets, probably got reset, just return after + return getAfter() + } + if ((after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding)) > (1 << 63) { + // after bytes < before bytes, probably got reset, just return after + return getAfter() + } + packetsLost := uint32(0) if after.PacketsLost >= before.PacketsLost { packetsLost = after.PacketsLost - before.PacketsLost From 73af5da956343a16536b08fc2e622a8f2bbac069 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 28 Dec 2023 15:50:51 +0530 Subject: [PATCH 055/114] Notify TrackInfo available from red receivers. (#2354) * Notify TrackInfo available from red receivers. That kicks off the down track scorer. * test --- pkg/sfu/redprimaryreceiver.go | 4 ++++ pkg/sfu/redreceiver.go | 4 ++++ pkg/sfu/redreceiver_test.go | 41 +++++++++++++++++++++++++++++------ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index e7d33099f..e9e2642f3 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -96,7 +96,10 @@ func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error { r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID()) } + track.TrackInfoAvailable() + r.downTrackSpreader.Store(track) + r.logger.Debugw("red primary receiver downtrack added", "subscriberID", track.SubscriberID()) return nil } @@ -106,6 +109,7 @@ func (r *RedPrimaryReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) } r.downTrackSpreader.Free(subscriberID) + r.logger.Debugw("red primary receiver downtrack deleted", "subscriberID", subscriberID) } func (r *RedPrimaryReceiver) IsClosed() bool { diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index 9254be438..fe254a972 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -87,7 +87,10 @@ func (r *RedReceiver) AddDownTrack(track TrackSender) error { r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID()) } + track.TrackInfoAvailable() + r.downTrackSpreader.Store(track) + r.logger.Debugw("red receiver downtrack added", "subscriberID", track.SubscriberID()) return nil } @@ -97,6 +100,7 @@ func (r *RedReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) { } r.downTrackSpreader.Free(subscriberID) + r.logger.Debugw("red receiver downtrack deleted", "subscriberID", subscriberID) } func (r *RedReceiver) CanClose() bool { diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 2aae30182..72261f7db 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" ) const tsStep = uint32(48000 / 1000 * 10) @@ -38,11 +39,17 @@ func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) error { return nil } +func (dt *dummyDowntrack) TrackInfoAvailable() {} + func TestRedReceiver(t *testing.T) { dt := &dummyDowntrack{TrackSender: &DownTrack{}} t.Run("normal", func(t *testing.T) { - w := &WebRTCReceiver{isRED: true, kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + isRED: true, + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } require.Equal(t, w.GetRedReceiver(), w) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) @@ -64,7 +71,10 @@ func TestRedReceiver(t *testing.T) { }) t.Run("packet lost and jump", func(t *testing.T) { - w := &WebRTCReceiver{kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } red := w.GetRedReceiver().(*RedReceiver) require.NoError(t, red.AddDownTrack(dt)) @@ -112,7 +122,10 @@ func TestRedReceiver(t *testing.T) { }) t.Run("unorder and repeat", func(t *testing.T) { - w := &WebRTCReceiver{kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } red := w.GetRedReceiver().(*RedReceiver) require.NoError(t, red.AddDownTrack(dt)) @@ -141,7 +154,11 @@ func TestRedReceiver(t *testing.T) { }) t.Run("encoding exceed space", func(t *testing.T) { - w := &WebRTCReceiver{isRED: true, kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + isRED: true, + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } require.Equal(t, w.GetRedReceiver(), w) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) @@ -162,7 +179,11 @@ func TestRedReceiver(t *testing.T) { }) t.Run("large timestamp gap", func(t *testing.T) { - w := &WebRTCReceiver{isRED: true, kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + isRED: true, + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } require.Equal(t, w.GetRedReceiver(), w) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) @@ -257,7 +278,10 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) { dt := &dummyDowntrack{TrackSender: &DownTrack{}} - w := &WebRTCReceiver{kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } require.Equal(t, w.GetPrimaryReceiverForRed(), w) w.isRED = true red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver) @@ -283,7 +307,10 @@ func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktI } func TestRedPrimaryReceiver(t *testing.T) { - w := &WebRTCReceiver{kind: webrtc.RTPCodecTypeAudio} + w := &WebRTCReceiver{ + kind: webrtc.RTPCodecTypeAudio, + logger: logger.GetLogger(), + } require.Equal(t, w.GetPrimaryReceiverForRed(), w) w.isRED = true red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver) From 7c8b60431dd054019bfb63b057f2afa8b567deb3 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 Jan 2024 23:32:43 +0530 Subject: [PATCH 056/114] Update mute if necessary when updating track info. (#2357) --- pkg/rtc/mediatrackreceiver.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 3c9db5616..d0818ceb1 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -590,6 +590,7 @@ func (t *MediaTrackReceiver) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { } func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + updateMute := false clonedInfo := proto.Clone(ti).(*livekit.TrackInfo) t.lock.Lock() @@ -622,9 +623,16 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { clonedInfo.Layers = ci.Layers } } + if t.trackInfo.Muted != clonedInfo.Muted { + updateMute = true + } t.trackInfo = clonedInfo t.lock.Unlock() + if updateMute { + t.SetMuted(clonedInfo.Muted) + } + t.updateTrackInfoOfReceivers() } From a10830a9952691dcd0d35e4c0fb05ba4e9319c9d Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 3 Jan 2024 00:06:43 -0800 Subject: [PATCH 057/114] add protocol version 12 helper (#2358) --- pkg/rtc/types/protocol_version.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 6dde9a976..ac445e9d8 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -16,7 +16,7 @@ package types type ProtocolVersion int -const CurrentProtocol = 11 +const CurrentProtocol = 12 func (v ProtocolVersion) SupportsPackedStreamId() bool { return v > 0 @@ -79,3 +79,7 @@ func (v ProtocolVersion) SupportSyncStreamID() bool { func (v ProtocolVersion) SupportsConnectionQualityLost() bool { return v > 10 } + +func (v ProtocolVersion) SupportsAsyncRoomID() bool { + return v > 11 +} From a738062099363d28ab36d907ad29f5cc54cb15fd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:27:27 -0800 Subject: [PATCH 058/114] Update go deps (#2352) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 12 ++++++------ go.sum | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 9c0d0b0f5..083f856b4 100644 --- a/go.mod +++ b/go.mod @@ -36,17 +36,17 @@ require ( github.com/pion/turn/v2 v2.1.4 github.com/pion/webrtc/v3 v3.2.24 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/redis/go-redis/v9 v9.3.1 github.com/rs/cors v1.10.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc - github.com/urfave/cli/v2 v2.27.0 + github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20231226003508-02704c960a9b + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/sync v0.5.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 @@ -74,7 +74,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect github.com/nats-io/nats.go v1.31.0 // indirect @@ -87,8 +87,8 @@ require ( github.com/pion/srtp/v2 v2.0.18 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect diff --git a/go.sum b/go.sum index 3ae94fb80..c6b4aa7f5 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0 h1:z0CfPybq3CxaJvrrpf7Gme1psZTqHhJxf83q6apkSpI= github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0/go.mod h1:RVP6/F85JyxTrbJxWIdKU2vlSvK48iCMnMXRkSz7xtg= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= @@ -230,12 +230,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= @@ -267,8 +267,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc h1:iT5lwxf894PiMq7cnMMQg/7VOD1pxmu//gQuHWAFy4s= github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= -github.com/urfave/cli/v2 v2.27.0 h1:uNs1K8JwTFL84X68j5Fjny6hfANh9nTlJ6dRtZAFAHY= -github.com/urfave/cli/v2 v2.27.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -297,8 +297,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -335,7 +335,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 7049627767d47e60d36e610ab1508fba79153077 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:16:41 -0800 Subject: [PATCH 059/114] Update module golang.org/x/sync to v0.6.0 (#2361) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 083f856b4..d4b2739b8 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index c6b4aa7f5..51a1c4075 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 32bd75648f3bc28e83a2d4f39b39894b9d737923 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 5 Jan 2024 17:07:44 +0530 Subject: [PATCH 060/114] Wait for metadata update. (#2363) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- pkg/rtc/room.go | 4 ++-- pkg/service/roommanager.go | 4 +++- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index d4b2739b8..aaad09ddf 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa + github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -78,7 +78,7 @@ require ( github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect github.com/nats-io/nats.go v1.31.0 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/logging v0.2.2 // indirect @@ -98,10 +98,10 @@ require ( golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/grpc v1.60.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 51a1c4075..856494975 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa h1:qIzXYbpCR01Czwe/j2HYFzHp3j3VSCavdR+R2+qSmS4= -github.com/livekit/protocol v1.9.4-0.20231222234445-80e8b5a2d1fa/go.mod h1:qL0J9HZaUrimDs29b/uRARvWn1cqjbvXUhayZ02RF9U= +github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83 h1:iYur8jpRGdpoFH2IZAXUEu/l4Gsp5sQticAXnSeUHGA= +github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83/go.mod h1:8db9KAaD8iYZLyewdgtJBQ70A63srl4AI9hBD9JJ0ps= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -163,8 +163,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -383,8 +383,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -420,8 +420,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index d832a61a3..8412788e9 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -767,11 +767,11 @@ func (r *Room) SendDataPacket(up *livekit.UserPacket, kind livekit.DataPacket_Ki r.onDataPacket(nil, dp) } -func (r *Room) SetMetadata(metadata string) { +func (r *Room) SetMetadata(metadata string) <-chan struct{} { r.lock.Lock() r.protoRoom.Metadata = metadata r.lock.Unlock() - r.protoProxy.MarkDirty(true) + return r.protoProxy.MarkDirty(true) } func (r *Room) UpdateParticipantMetadata(participant types.LocalParticipant, name string, metadata string) { diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index a6fa679f3..24feab876 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -776,7 +776,9 @@ func (r *RoomManager) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat } room.Logger.Debugw("updating room") - room.SetMetadata(req.Metadata) + done := room.SetMetadata(req.Metadata) + // wait till the update is applied + <-done return room.ToProto(), nil } From b9f9a6b35ea1227d96f2587e925cf25504355b71 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 5 Jan 2024 04:16:02 -0800 Subject: [PATCH 061/114] force shutdown after second signal (#2364) --- cmd/server/main.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index c53f01640..b9bfea418 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -292,9 +292,12 @@ func startServer(c *cli.Context) error { signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { - sig := <-sigChan - logger.Infow("exit requested, shutting down", "signal", sig) - server.Stop(false) + for i := 0; i < 2; i++ { + sig := <-sigChan + force := i > 0 + logger.Infow("exit requested, shutting down", "signal", sig, "force", force) + go server.Stop(force) + } }() return server.Start() From d8df0f7727e5c4e338cbed8ece6c555b7696f746 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 22:21:37 -0800 Subject: [PATCH 062/114] Update module github.com/florianl/go-tc to v0.4.3 (#2365) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index aaad09ddf..b342bd81c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/d5/tengo/v2 v2.16.1 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 - github.com/florianl/go-tc v0.4.2 + github.com/florianl/go-tc v0.4.3 github.com/frostbyte73/core v0.0.9 github.com/gammazero/deque v0.2.1 github.com/gammazero/workerpool v1.1.3 @@ -63,7 +63,7 @@ require ( github.com/go-jose/go-jose/v3 v3.0.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index 856494975..e26ea1b33 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= -github.com/florianl/go-tc v0.4.2 h1:jan5zcOWCLhA9SRBHZhQ0SSAq7cmDUagiRPngAi5AOQ= -github.com/florianl/go-tc v0.4.2/go.mod h1:2W1jSMFryiYlpQigr4ZpSSpE9XNze+bW7cTsCXWbMwo= +github.com/florianl/go-tc v0.4.3 h1:xpobG2gFNvEqbclU07zjddALSjqTQTWJkxg5/kRYDpw= +github.com/florianl/go-tc v0.4.3/go.mod h1:uvp6pIlOw7Z8hhfnT5M4+V1hHVgZWRZwwMS8Z0JsRxc= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frostbyte73/core v0.0.9 h1:AmE9GjgGpPsWk9ZkmY3HsYUs2hf2tZt+/W6r49URBQI= @@ -66,8 +66,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= From a1ca41a4e1cf051f1e04669857cd2ee66d8d2c73 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 8 Jan 2024 13:11:04 +0530 Subject: [PATCH 063/114] Skip reporting skew for out-of-order reports (#2369) --- pkg/sfu/buffer/rtpstats_sender.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 6dc952e12..618341e75 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -629,7 +629,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, calculatedClockRate ui RTPTimestampExt: nowRTPExt, At: now, } - if r.srNewest != nil { + if r.srNewest != nil && nowRTPExt >= r.srNewest.RTPTimestampExt { timeSinceLastReport := nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RTPTimestampExt windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds() From 4e30e1a86da3d64b9b32ab8fa5399616991b390d Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Mon, 8 Jan 2024 16:24:53 -0800 Subject: [PATCH 064/114] Send telemetry events when switching to COMPLETE state (#2371) --- pkg/service/ioservice.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/service/ioservice.go b/pkg/service/ioservice.go index 62d2820ff..67f995033 100644 --- a/pkg/service/ioservice.go +++ b/pkg/service/ioservice.go @@ -193,7 +193,8 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI switch req.State.Status { case livekit.IngressState_ENDPOINT_ERROR, - livekit.IngressState_ENDPOINT_INACTIVE: + livekit.IngressState_ENDPOINT_INACTIVE, + livekit.IngressState_ENDPOINT_COMPLETE: s.telemetry.IngressEnded(ctx, info) if req.State.Error != "" { @@ -218,7 +219,7 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI s.telemetry.IngressUpdated(ctx, info) - logger.Infow("ingress updated", "ingressID", req.IngressId) + logger.Infow("ingress updated", "ingressID", req.IngressId, "status", info.State.Status) } return &emptypb.Empty{}, nil From d70a8e366c20280a4e95f7ee75c12a146ed54aa0 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 10 Jan 2024 03:31:54 -0800 Subject: [PATCH 065/114] inject logger constructor into signal server (#2372) * inject logger constructor into signal server * tidy * tidy * test --- .../servicefakes/fake_session_handler.go | 199 ++++++++++++++++++ pkg/service/signal.go | 95 +++++---- pkg/service/signal_test.go | 69 +++--- 3 files changed, 295 insertions(+), 68 deletions(-) create mode 100644 pkg/service/servicefakes/fake_session_handler.go diff --git a/pkg/service/servicefakes/fake_session_handler.go b/pkg/service/servicefakes/fake_session_handler.go new file mode 100644 index 000000000..552386918 --- /dev/null +++ b/pkg/service/servicefakes/fake_session_handler.go @@ -0,0 +1,199 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package servicefakes + +import ( + "context" + "sync" + + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +type FakeSessionHandler struct { + HandleSessionStub func(context.Context, livekit.RoomName, routing.ParticipantInit, livekit.ConnectionID, routing.MessageSource, routing.MessageSink) error + handleSessionMutex sync.RWMutex + handleSessionArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomName + arg3 routing.ParticipantInit + arg4 livekit.ConnectionID + arg5 routing.MessageSource + arg6 routing.MessageSink + } + handleSessionReturns struct { + result1 error + } + handleSessionReturnsOnCall map[int]struct { + result1 error + } + LoggerStub func(context.Context) logger.Logger + loggerMutex sync.RWMutex + loggerArgsForCall []struct { + arg1 context.Context + } + loggerReturns struct { + result1 logger.Logger + } + loggerReturnsOnCall map[int]struct { + result1 logger.Logger + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeSessionHandler) HandleSession(arg1 context.Context, arg2 livekit.RoomName, arg3 routing.ParticipantInit, arg4 livekit.ConnectionID, arg5 routing.MessageSource, arg6 routing.MessageSink) error { + fake.handleSessionMutex.Lock() + ret, specificReturn := fake.handleSessionReturnsOnCall[len(fake.handleSessionArgsForCall)] + fake.handleSessionArgsForCall = append(fake.handleSessionArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomName + arg3 routing.ParticipantInit + arg4 livekit.ConnectionID + arg5 routing.MessageSource + arg6 routing.MessageSink + }{arg1, arg2, arg3, arg4, arg5, arg6}) + stub := fake.HandleSessionStub + fakeReturns := fake.handleSessionReturns + fake.recordInvocation("HandleSession", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) + fake.handleSessionMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4, arg5, arg6) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSessionHandler) HandleSessionCallCount() int { + fake.handleSessionMutex.RLock() + defer fake.handleSessionMutex.RUnlock() + return len(fake.handleSessionArgsForCall) +} + +func (fake *FakeSessionHandler) HandleSessionCalls(stub func(context.Context, livekit.RoomName, routing.ParticipantInit, livekit.ConnectionID, routing.MessageSource, routing.MessageSink) error) { + fake.handleSessionMutex.Lock() + defer fake.handleSessionMutex.Unlock() + fake.HandleSessionStub = stub +} + +func (fake *FakeSessionHandler) HandleSessionArgsForCall(i int) (context.Context, livekit.RoomName, routing.ParticipantInit, livekit.ConnectionID, routing.MessageSource, routing.MessageSink) { + fake.handleSessionMutex.RLock() + defer fake.handleSessionMutex.RUnlock() + argsForCall := fake.handleSessionArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 +} + +func (fake *FakeSessionHandler) HandleSessionReturns(result1 error) { + fake.handleSessionMutex.Lock() + defer fake.handleSessionMutex.Unlock() + fake.HandleSessionStub = nil + fake.handleSessionReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeSessionHandler) HandleSessionReturnsOnCall(i int, result1 error) { + fake.handleSessionMutex.Lock() + defer fake.handleSessionMutex.Unlock() + fake.HandleSessionStub = nil + if fake.handleSessionReturnsOnCall == nil { + fake.handleSessionReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleSessionReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeSessionHandler) Logger(arg1 context.Context) logger.Logger { + fake.loggerMutex.Lock() + ret, specificReturn := fake.loggerReturnsOnCall[len(fake.loggerArgsForCall)] + fake.loggerArgsForCall = append(fake.loggerArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.LoggerStub + fakeReturns := fake.loggerReturns + fake.recordInvocation("Logger", []interface{}{arg1}) + fake.loggerMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSessionHandler) LoggerCallCount() int { + fake.loggerMutex.RLock() + defer fake.loggerMutex.RUnlock() + return len(fake.loggerArgsForCall) +} + +func (fake *FakeSessionHandler) LoggerCalls(stub func(context.Context) logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = stub +} + +func (fake *FakeSessionHandler) LoggerArgsForCall(i int) context.Context { + fake.loggerMutex.RLock() + defer fake.loggerMutex.RUnlock() + argsForCall := fake.loggerArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSessionHandler) LoggerReturns(result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + fake.loggerReturns = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeSessionHandler) LoggerReturnsOnCall(i int, result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + if fake.loggerReturnsOnCall == nil { + fake.loggerReturnsOnCall = make(map[int]struct { + result1 logger.Logger + }) + } + fake.loggerReturnsOnCall[i] = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeSessionHandler) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.handleSessionMutex.RLock() + defer fake.handleSessionMutex.RUnlock() + fake.loggerMutex.RLock() + defer fake.loggerMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeSessionHandler) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ service.SessionHandler = new(FakeSessionHandler) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index a181987e9..a0abc5952 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -31,14 +31,21 @@ import ( "github.com/livekit/psrpc/pkg/middleware" ) -type SessionHandler func( - ctx context.Context, - roomName livekit.RoomName, - pi routing.ParticipantInit, - connectionID livekit.ConnectionID, - requestSource routing.MessageSource, - responseSink routing.MessageSink, -) error +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . SessionHandler +type SessionHandler interface { + Logger(ctx context.Context) logger.Logger + + HandleSession( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error +} type SignalServer struct { server rpc.TypedSignalServer @@ -72,43 +79,53 @@ func NewDefaultSignalServer( router routing.Router, roomManager *RoomManager, ) (r *SignalServer, err error) { - sessionHandler := func( - ctx context.Context, - roomName livekit.RoomName, - pi routing.ParticipantInit, - connectionID livekit.ConnectionID, - requestSource routing.MessageSource, - responseSink routing.MessageSink, - ) error { - prometheus.IncrementParticipantRtcInit(1) + return NewSignalServer(livekit.NodeID(currentNode.Id), currentNode.Region, bus, config, &defaultSessionHandler{currentNode, router, roomManager}) +} - if rr, ok := router.(*routing.RedisRouter); ok { - rtcNode, err := router.GetNodeForRoom(ctx, roomName) - if err != nil { - return err - } +type defaultSessionHandler struct { + currentNode routing.LocalNode + router routing.Router + roomManager *RoomManager +} - if rtcNode.Id != currentNode.Id { - err = routing.ErrIncorrectRTCNode - logger.Errorw("called participant on incorrect node", err, - "rtcNode", rtcNode, - ) - return err - } +func (s *defaultSessionHandler) Logger(ctx context.Context) logger.Logger { + return logger.GetLogger() +} - pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) - pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) +func (s *defaultSessionHandler) HandleSession( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, +) error { + prometheus.IncrementParticipantRtcInit(1) - // RTC session should start on this node - if err := rr.SetParticipantRTCNode(pKey, pKeyB62, currentNode.Id); err != nil { - return err - } + if rr, ok := s.router.(*routing.RedisRouter); ok { + rtcNode, err := s.router.GetNodeForRoom(ctx, roomName) + if err != nil { + return err } - return roomManager.StartSession(ctx, roomName, pi, requestSource, responseSink) + if rtcNode.Id != s.currentNode.Id { + err = routing.ErrIncorrectRTCNode + logger.Errorw("called participant on incorrect node", err, + "rtcNode", rtcNode, + ) + return err + } + + pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) + pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) + + // RTC session should start on this node + if err := rr.SetParticipantRTCNode(pKey, pKeyB62, s.currentNode.Id); err != nil { + return err + } } - return NewSignalServer(livekit.NodeID(currentNode.Id), currentNode.Region, bus, config, sessionHandler) + return s.roomManager.StartSession(ctx, roomName, pi, requestSource, responseSink) } func (s *SignalServer) Start() error { @@ -142,7 +159,7 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe return errors.Wrap(err, "failed to read participant from session") } - l := logger.GetLogger().WithValues( + l := r.sessionHandler.Logger(stream.Context()).WithValues( "room", ss.RoomName, "participant", ss.Identity, "connID", ss.ConnectionId, @@ -175,7 +192,7 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe // copy the incoming rpc headers to avoid dropping any session vars. ctx := metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context())) - err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) + err = r.sessionHandler.HandleSession(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { sink.Close() l.Errorw("could not handle new participant", err) diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 83bf391e9..a80935182 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package service +package service_test import ( "context" @@ -26,8 +26,11 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/livekit-server/pkg/service/servicefakes" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/psrpc" ) @@ -61,22 +64,26 @@ func TestSignal(t *testing.T) { client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) require.NoError(t, err) - server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( - ctx context.Context, - roomName livekit.RoomName, - pi routing.ParticipantInit, - connectionID livekit.ConnectionID, - requestSource routing.MessageSource, - responseSink routing.MessageSink, - ) error { - go func() { - reqMessageOut = <-requestSource.ReadChan() - resErr = responseSink.WriteMessage(resMessageIn) - responseSink.Close() - close(done) - }() - return nil - }) + handler := &servicefakes.FakeSessionHandler{ + LoggerStub: func(context.Context) logger.Logger { return logger.GetLogger() }, + HandleSessionStub: func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + go func() { + reqMessageOut = <-requestSource.ReadChan() + resErr = responseSink.WriteMessage(resMessageIn) + responseSink.Close() + close(done) + }() + return nil + }, + } + server, err := service.NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, handler) require.NoError(t, err) err = server.Start() @@ -114,18 +121,22 @@ func TestSignal(t *testing.T) { client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) require.NoError(t, err) - server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( - ctx context.Context, - roomName livekit.RoomName, - pi routing.ParticipantInit, - connectionID livekit.ConnectionID, - requestSource routing.MessageSource, - responseSink routing.MessageSink, - ) error { - defer close(done) - resErr = responseSink.WriteMessage(resMessageIn) - return errors.New("start session failed") - }) + handler := &servicefakes.FakeSessionHandler{ + LoggerStub: func(context.Context) logger.Logger { return logger.GetLogger() }, + HandleSessionStub: func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + defer close(done) + resErr = responseSink.WriteMessage(resMessageIn) + return errors.New("start session failed") + }, + } + server, err := service.NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, handler) require.NoError(t, err) err = server.Start() From 6ff7b0fabad4136de9081828f49f7896a04acc2f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 11 Jan 2024 07:54:26 +0530 Subject: [PATCH 066/114] Pass mock track in track update callback (#2373) --- pkg/rtc/testutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/testutils.go b/pkg/rtc/testutils.go index 845cc4629..f80230f0c 100644 --- a/pkg/rtc/testutils.go +++ b/pkg/rtc/testutils.go @@ -62,7 +62,7 @@ func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro f = p.OnTrackUpdatedArgsForCall(p.OnTrackUpdatedCallCount() - 1) } if f != nil { - f(p, nil) + f(p, NewMockTrack(livekit.TrackType_VIDEO, "testcam")) } } From dc1b09c757f05f7b5d05059d4663a53909292801 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 11 Jan 2024 13:57:29 +0530 Subject: [PATCH 067/114] Update pion/ice to pick some more trace logging (#2374) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b342bd81c..4429562c1 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.8 - github.com/pion/ice/v2 v2.3.11 + github.com/pion/ice/v2 v2.3.12 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.13 github.com/pion/rtp v1.8.3 diff --git a/go.sum b/go.sum index e26ea1b33..a0e9f5359 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,9 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= +github.com/pion/ice/v2 v2.3.12 h1:NWKW2b3+oSZS3klbQMIEWQ0i52Kuo0KBg505a5kQv4s= +github.com/pion/ice/v2 v2.3.12/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= From 3687396d8414000810d5f7acc9bb9db6d9c85d40 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 12 Jan 2024 12:16:19 +0530 Subject: [PATCH 068/114] Squelch error logs while waiting for track resolve. (#2376) --- pkg/rtc/subscriptionmanager.go | 5 +++-- pkg/rtc/transportmanager.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 78ce71c6b..eb00a15c9 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -18,6 +18,7 @@ package rtc import ( "context" + "errors" "sync" "time" @@ -503,8 +504,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { } subTrack, err := track.AddSubscriber(m.params.Participant) - if err != nil && err != errAlreadySubscribed { - // ignore already subscribed error + if err != nil && !errors.Is(err, errAlreadySubscribed) && !errors.Is(err, ErrTrackNotAttached) && !errors.Is(err, ErrNoReceiver) { + // ignore errors: already subscribed OR waiting for track resolve m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) return err } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 21591f3c2..489f834e7 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -305,7 +305,7 @@ func (t *TransportManager) RemoveSubscribedTrack(subTrack types.SubscribedTrack) } func (t *TransportManager) OnDataMessage(f func(kind livekit.DataPacket_Kind, data []byte)) { - // upstream data is always comes in via publisher peer connection irrespective of which is primary + // upstream data always comes in via publisher peer connection irrespective of which is primary t.publisher.OnDataPacket(f) } From 2fe2a9c9f2110ddff149f58b3b3f1d15b4efb702 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 11 Jan 2024 23:23:51 -0800 Subject: [PATCH 069/114] add session start time metric (#2377) --- pkg/rtc/participant.go | 2 ++ pkg/rtc/participant_internal_test.go | 1 + pkg/service/roommanager.go | 3 +++ pkg/telemetry/prometheus/rooms.go | 14 ++++++++++++++ 4 files changed, 20 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b6d63e334..06171a837 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -97,6 +97,7 @@ type ParticipantParams struct { AudioConfig config.AudioConfig VideoConfig config.VideoConfig ProtocolVersion types.ProtocolVersion + SessionStartTime time.Time Telemetry telemetry.TelemetryService Trailer []byte PLIThrottleConfig config.PLIThrottleConfig @@ -1421,6 +1422,7 @@ func (p *ParticipantImpl) onPrimaryTransportInitialConnected() { } func (p *ParticipantImpl) onPrimaryTransportFullyEstablished() { + prometheus.RecordSessionStartTime(int(p.ProtocolVersion()), time.Since(p.params.SessionStartTime)) p.updateState(livekit.ParticipantInfo_ACTIVE) } diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index afd312bc8..a1b758ff8 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -753,6 +753,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p Config: rtcConf, Sink: &routingfakes.FakeMessageSink{}, ProtocolVersion: opts.protocolVersion, + SessionStartTime: time.Now(), PLIThrottleConfig: conf.RTC.PLIThrottle, Grants: grants, PublishEnabledCodecs: enabledCodecs, diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 24feab876..6d122fd6e 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -245,6 +245,8 @@ func (r *RoomManager) StartSession( requestSource routing.MessageSource, responseSink routing.MessageSink, ) error { + sessionStartTime := time.Now() + room, err := r.getOrCreateRoom(ctx, roomName) if err != nil { return err @@ -390,6 +392,7 @@ func (r *RoomManager) StartSession( AudioConfig: r.config.Audio, VideoConfig: r.config.Video, ProtocolVersion: pv, + SessionStartTime: sessionStartTime, Telemetry: r.telemetry, Trailer: room.Trailer(), PLIThrottleConfig: r.config.RTC.PLIThrottle, diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index ef320ad73..2aaff88e2 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -15,6 +15,7 @@ package prometheus import ( + "strconv" "time" "github.com/prometheus/client_golang/prometheus" @@ -43,6 +44,7 @@ var ( promTrackSubscribedCurrent *prometheus.GaugeVec promTrackPublishCounter *prometheus.CounterVec promTrackSubscribeCounter *prometheus.CounterVec + promSessionStartTime prometheus.HistogramVec ) func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { @@ -91,6 +93,13 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { Name: "subscribe_counter", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"state", "error"}) + promSessionStartTime = *prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "session", + Name: "start_time_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: prometheus.ExponentialBucketsRange(100, 5000, 10), + }, []string{"protocol_version"}) prometheus.MustRegister(promRoomCurrent) prometheus.MustRegister(promRoomDuration) @@ -99,6 +108,7 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { prometheus.MustRegister(promTrackSubscribedCurrent) prometheus.MustRegister(promTrackPublishCounter) prometheus.MustRegister(promTrackSubscribeCounter) + prometheus.MustRegister(promSessionStartTime) } func RoomStarted() { @@ -172,3 +182,7 @@ func RecordTrackSubscribeFailure(err error, isUserError bool) { trackSubscribeUserError.Inc() } } + +func RecordSessionStartTime(protocolVersion int, d time.Duration) { + promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds())) +} From bf0e88dea4e44183affe402951c2e228b9a2a892 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 12 Jan 2024 16:58:23 +0530 Subject: [PATCH 070/114] Squelch only the log, not the error return. (#2379) --- pkg/rtc/subscriptionmanager.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index eb00a15c9..5cbbb216f 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -504,9 +504,12 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { } subTrack, err := track.AddSubscriber(m.params.Participant) - if err != nil && !errors.Is(err, errAlreadySubscribed) && !errors.Is(err, ErrTrackNotAttached) && !errors.Is(err, ErrNoReceiver) { - // ignore errors: already subscribed OR waiting for track resolve - m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) + if err != nil && !errors.Is(err, errAlreadySubscribed) { + // ignore error(s): already subscribed + if !errors.Is(err, ErrTrackNotAttached) && !errors.Is(err, ErrNoReceiver) { + // as track resolution could take some time, not logging errors due to waiting for track resolution + m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) + } return err } if err == errAlreadySubscribed { From c726cbf2ba80c96b35ea9ba2ca4457e924c3588e Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 12 Jan 2024 03:49:23 -0800 Subject: [PATCH 071/114] increase max session start time bin size (#2380) --- pkg/telemetry/prometheus/rooms.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index 2aaff88e2..91c7cc2ba 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -44,7 +44,7 @@ var ( promTrackSubscribedCurrent *prometheus.GaugeVec promTrackPublishCounter *prometheus.CounterVec promTrackSubscribeCounter *prometheus.CounterVec - promSessionStartTime prometheus.HistogramVec + promSessionStartTime *prometheus.HistogramVec ) func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { @@ -93,12 +93,12 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { Name: "subscribe_counter", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"state", "error"}) - promSessionStartTime = *prometheus.NewHistogramVec(prometheus.HistogramOpts{ + promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "session", Name: "start_time_ms", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, - Buckets: prometheus.ExponentialBucketsRange(100, 5000, 10), + Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15), }, []string{"protocol_version"}) prometheus.MustRegister(promRoomCurrent) From 78bf642d670d4771a59266bf1fa0e3a457bbb0a6 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 12 Jan 2024 05:55:56 -0800 Subject: [PATCH 072/114] record session once (#2381) --- pkg/rtc/participant.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 06171a837..bfb3112ca 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -145,6 +145,7 @@ type ParticipantImpl struct { hidden atomic.Bool isPublisher atomic.Bool + sessionStartRecorded atomic.Bool // when first connected connectedAt time.Time // timer that's set when disconnect is detected on primary PC @@ -1422,7 +1423,9 @@ func (p *ParticipantImpl) onPrimaryTransportInitialConnected() { } func (p *ParticipantImpl) onPrimaryTransportFullyEstablished() { - prometheus.RecordSessionStartTime(int(p.ProtocolVersion()), time.Since(p.params.SessionStartTime)) + if !p.sessionStartRecorded.Swap(true) { + prometheus.RecordSessionStartTime(int(p.ProtocolVersion()), time.Since(p.params.SessionStartTime)) + } p.updateState(livekit.ParticipantInfo_ACTIVE) } From 2ba4e5c070c30f4ca2d8f096b08b3f3aaa271fd0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Jan 2024 10:30:22 -0800 Subject: [PATCH 073/114] Update go deps (#2366) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 4429562c1..88118d713 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pion/webrtc/v3 v3.2.24 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 - github.com/redis/go-redis/v9 v9.3.1 + github.com/redis/go-redis/v9 v9.4.0 github.com/rs/cors v1.10.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc + golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 @@ -95,12 +95,12 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.18.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/grpc v1.60.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a0e9f5359..2b4eb0f15 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= -github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= @@ -296,10 +296,10 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -333,8 +333,8 @@ golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -415,8 +415,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 1cb4b3e585f46347afe7f0355cafbafcbe729a98 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 17:42:34 -0800 Subject: [PATCH 074/114] Update go deps (#2382) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 88118d713..9387bf323 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 - github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0 + github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.8 @@ -42,7 +42,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible - github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc + github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 diff --git a/go.sum b/go.sum index 2b4eb0f15..60a8c98fc 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0 h1:z0CfPybq3CxaJvrrpf7Gme1psZTqHhJxf83q6apkSpI= -github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0/go.mod h1:RVP6/F85JyxTrbJxWIdKU2vlSvK48iCMnMXRkSz7xtg= +github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= +github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= @@ -178,7 +178,7 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -266,8 +266,8 @@ github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= -github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc h1:iT5lwxf894PiMq7cnMMQg/7VOD1pxmu//gQuHWAFy4s= -github.com/ua-parser/uap-go v0.0.0-20230823213814-f77b3e91e9dc/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= +github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 h1:F04Na0QJP9GJrwmK3vQDuDrCuGllrrfngW8CIeF1aag= +github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= From 3f2f850bdb5cbf87b79f7d549872ed512f2109ec Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 14 Jan 2024 01:49:26 -0800 Subject: [PATCH 075/114] clean up legacy rpc (#2384) * clean up legacy rpc * cleanup * cleanup * cleanup * tidy * cleanup * cleanup --- config-sample.yaml | 4 - pkg/config/config.go | 2 - pkg/routing/interfaces.go | 15 +- pkg/routing/localrouter.go | 129 +--------- pkg/routing/redisrouter.go | 310 ++---------------------- pkg/routing/routingfakes/fake_router.go | 236 ------------------ pkg/service/roommanager.go | 29 +-- pkg/service/roomservice.go | 201 +-------------- pkg/service/roomservice_test.go | 20 -- pkg/service/signal.go | 30 +-- pkg/service/signal_test.go | 1 - pkg/service/wire_gen.go | 4 +- test/integration_helpers.go | 1 - 13 files changed, 45 insertions(+), 937 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index 4a70f2ae8..0d001bd10 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -183,8 +183,6 @@ keys: # since v1.4.0, a more reliable, psrpc based signal relay is available # this gives us the ability to reliably proxy messages between a signal server and RTC node # signal_relay: -# # enabled by default as of v1.5.0, legacy signal proxy will be removed in v1.6 -# enabled: true # # amount of time a message delivery is tried before giving up # retry_timeout: 30s # # minimum amount of time to wait for RTC node to ack, @@ -199,8 +197,6 @@ keys: # PSRPC # since v1.5.1, a more reliable, psrpc based internal rpc # psrpc: -# # enable the psrpc internal api client for roomservice calls -# enabled: true # # maximum number of rpc attempts # max_attempts: 3 # # initial time to wait for calls to complete diff --git a/pkg/config/config.go b/pkg/config/config.go index be9920cce..1a515f86f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -268,7 +268,6 @@ type NodeSelectorConfig struct { } type SignalRelayConfig struct { - Enabled bool `yaml:"enabled,omitempty"` RetryTimeout time.Duration `yaml:"retry_timeout,omitempty"` MinRetryInterval time.Duration `yaml:"min_retry_interval,omitempty"` MaxRetryInterval time.Duration `yaml:"max_retry_interval,omitempty"` @@ -487,7 +486,6 @@ var DefaultConfig = Config{ CPULoadLimit: 0.9, }, SignalRelay: SignalRelayConfig{ - Enabled: true, RetryTimeout: 7500 * time.Millisecond, MinRetryInterval: 500 * time.Millisecond, MaxRetryInterval: 4 * time.Second, diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 37985bb9d..0b4f31c41 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -21,7 +21,6 @@ import ( "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -99,12 +98,6 @@ type Router interface { Start() error Drain() Stop() - - // OnNewParticipantRTC is called to start a new participant's RTC connection - OnNewParticipantRTC(callback NewParticipantCallback) - - // OnRTCMessage is called to execute actions on the RTC node - OnRTCMessage(callback RTCMessageCallback) } type StartParticipantSignalResults struct { @@ -118,17 +111,13 @@ type StartParticipantSignalResults struct { type MessageRouter interface { // StartParticipantSignal participant signal connection is ready to start StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) - - // Write a message to a participant or room - WriteParticipantRTC(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error - WriteRoomRTC(ctx context.Context, roomName livekit.RoomName, msg *livekit.RTCNodeMessage) error } -func CreateRouter(config *config.Config, rc redis.UniversalClient, node LocalNode, signalClient SignalClient) Router { +func CreateRouter(rc redis.UniversalClient, node LocalNode, signalClient SignalClient) Router { lr := NewLocalRouter(node, signalClient) if rc != nil { - return NewRedisRouter(config, lr, rc) + return NewRedisRouter(lr, rc) } // local routing and store diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index d83f5ea60..85279ead0 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -26,8 +26,7 @@ import ( "github.com/livekit/protocol/logger" ) -// aggregated channel for all participants -const localRTCChannelSize = 10000 +var _ Router = (*LocalRouter)(nil) // a router of messages on the same node, basic implementation for local testing type LocalRouter struct { @@ -39,11 +38,6 @@ type LocalRouter struct { requestChannels map[string]*MessageChannel responseChannels map[string]*MessageChannel isStarted atomic.Bool - - rtcMessageChan *MessageChannel - - onNewParticipant NewParticipantCallback - onRTCMessage RTCMessageCallback } func NewLocalRouter(currentNode LocalNode, signalClient SignalClient) *LocalRouter { @@ -52,7 +46,6 @@ func NewLocalRouter(currentNode LocalNode, signalClient SignalClient) *LocalRout signalClient: signalClient, requestChannels: make(map[string]*MessageChannel), responseChannels: make(map[string]*MessageChannel), - rtcMessageChan: NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize), } } @@ -68,7 +61,6 @@ func (r *LocalRouter) SetNodeForRoom(_ context.Context, _ livekit.RoomName, _ li } func (r *LocalRouter) ClearRoomState(_ context.Context, _ livekit.RoomName) error { - // do nothing return nil } @@ -120,65 +112,22 @@ func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, room return } -func (r *LocalRouter) WriteParticipantRTC(_ context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { - r.lock.Lock() - if r.rtcMessageChan.IsClosed() { - // create a new one - r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) - } - r.lock.Unlock() - msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) - return r.writeRTCMessage(r.rtcMessageChan, msg) -} - -func (r *LocalRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomName, msg *livekit.RTCNodeMessage) error { - msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) - return r.WriteNodeRTC(ctx, r.currentNode.Id, msg) -} - -func (r *LocalRouter) WriteNodeRTC(_ context.Context, _ string, msg *livekit.RTCNodeMessage) error { - r.lock.Lock() - if r.rtcMessageChan.IsClosed() { - // create a new one - r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) - } - r.lock.Unlock() - return r.writeRTCMessage(r.rtcMessageChan, msg) -} - -func (r *LocalRouter) writeRTCMessage(sink MessageSink, msg *livekit.RTCNodeMessage) error { - msg.SenderTime = time.Now().Unix() - return sink.WriteMessage(msg) -} - -func (r *LocalRouter) OnNewParticipantRTC(callback NewParticipantCallback) { - r.onNewParticipant = callback -} - -func (r *LocalRouter) OnRTCMessage(callback RTCMessageCallback) { - r.onRTCMessage = callback -} - func (r *LocalRouter) Start() error { if r.isStarted.Swap(true) { return nil } go r.statsWorker() // go r.memStatsWorker() - // on local routers, Start doesn't do anything, websocket connections initiate the connections - go r.rtcMessageWorker() return nil } func (r *LocalRouter) Drain() { + r.lock.Lock() + defer r.lock.Unlock() r.currentNode.State = livekit.NodeState_SHUTTING_DOWN } -func (r *LocalRouter) Stop() { - r.rtcMessageChan.Close() -} +func (r *LocalRouter) Stop() {} func (r *LocalRouter) GetRegion() string { return r.currentNode.Region @@ -214,73 +163,3 @@ func (r *LocalRouter) statsWorker() { } } */ -func (r *LocalRouter) rtcMessageWorker() { - // is a new channel available? if so swap to that one - if !r.isStarted.Load() { - return - } - - // start a new worker after this finished - defer func() { - go r.rtcMessageWorker() - }() - - r.lock.RLock() - isClosed := r.rtcMessageChan.IsClosed() - r.lock.RUnlock() - if isClosed { - // sleep and retry - time.Sleep(time.Second) - } - - r.lock.RLock() - msgChan := r.rtcMessageChan.ReadChan() - r.lock.RUnlock() - // consume messages from - for msg := range msgChan { - if rtcMsg, ok := msg.(*livekit.RTCNodeMessage); ok { - var room livekit.RoomName - var identity livekit.ParticipantIdentity - var err error - if rtcMsg.ParticipantKeyB62 != "" { - room, identity, err = parseParticipantKey(livekit.ParticipantKey(rtcMsg.ParticipantKeyB62)) - } - if err != nil { - room, identity, err = parseParticipantKeyLegacy(livekit.ParticipantKey(rtcMsg.ParticipantKey)) - } - if err != nil { - logger.Errorw("could not process RTC message", err) - continue - } - if r.onRTCMessage != nil { - r.onRTCMessage(context.Background(), room, identity, rtcMsg) - } - } - } -} - -func (r *LocalRouter) getMessageChannel(target map[string]*MessageChannel, key string) *MessageChannel { - r.lock.RLock() - defer r.lock.RUnlock() - return target[key] -} - -func (r *LocalRouter) getOrCreateMessageChannel(target map[string]*MessageChannel, key string) *MessageChannel { - r.lock.Lock() - defer r.lock.Unlock() - mc := target[key] - - if mc != nil { - return mc - } - - mc = NewMessageChannel(livekit.ConnectionID(key), DefaultMessageChannelSize) - mc.OnClose(func() { - r.lock.Lock() - delete(target, key) - r.lock.Unlock() - }) - target[key] = mc - - return mc -} diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index 5a920b9c5..743d63d1c 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -28,9 +28,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing/selector" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -42,17 +40,18 @@ const ( statsMaxDelaySeconds = 30 ) +var _ Router = (*RedisRouter)(nil) + // RedisRouter uses Redis pub/sub to route signaling messages across different nodes // It relies on the RTC node to be the primary driver of the participant connection. // Because type RedisRouter struct { *LocalRouter - rc redis.UniversalClient - usePSRPCSignal bool - ctx context.Context - isStarted atomic.Bool - nodeMu sync.RWMutex + rc redis.UniversalClient + ctx context.Context + isStarted atomic.Bool + nodeMu sync.RWMutex // previous stats for computing averages prevStats *livekit.NodeStats @@ -60,11 +59,10 @@ type RedisRouter struct { cancel func() } -func NewRedisRouter(config *config.Config, lr *LocalRouter, rc redis.UniversalClient) *RedisRouter { +func NewRedisRouter(lr *LocalRouter, rc redis.UniversalClient) *RedisRouter { rr := &RedisRouter{ - LocalRouter: lr, - rc: rc, - usePSRPCSignal: config.SignalRelay.Enabled, + LocalRouter: lr, + rc: rc, } rr.ctx, rr.cancel = context.WithCancel(context.Background()) return rr @@ -163,72 +161,7 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } - if r.usePSRPCSignal { - res, err = r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) - if err != nil { - return - } - - // map signal & rtc nodes - err = r.setParticipantSignalNode(res.ConnectionID, r.currentNode.Id) - return - } - - res.ConnectionID = livekit.ConnectionID(utils.NewGuid("CO_")) - pKey := ParticipantKeyLegacy(roomName, pi.Identity) - pKeyB62 := ParticipantKey(roomName, pi.Identity) - - // map signal & rtc nodes - if err = r.setParticipantSignalNode(res.ConnectionID, r.currentNode.Id); err != nil { - return - } - - // index by connectionID, since there may be multiple connections for the participant - // set up response channel before sending StartSession and be ready to receive responses. - resChan := r.getOrCreateMessageChannel(r.responseChannels, string(res.ConnectionID)) - - sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), res.ConnectionID, pKey, pKeyB62) - - // serialize claims - ss, err := pi.ToStartSession(roomName, res.ConnectionID) - if err != nil { - return - } - - // sends a message to start session - err = sink.WriteMessage(ss) - if err != nil { - return - } - - res.RequestSink = sink - res.ResponseSource = resChan - return res, nil -} - -func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { - pkey := ParticipantKeyLegacy(roomName, identity) - pkeyB62 := ParticipantKey(roomName, identity) - rtcNode, err := r.getParticipantRTCNode(pkey, pkeyB62) - if err != nil { - return err - } - - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), "ephemeral", pkey, pkeyB62) - msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) - defer rtcSink.Close() - return r.writeRTCMessage(rtcSink, msg) -} - -func (r *RedisRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomName, msg *livekit.RTCNodeMessage) error { - node, err := r.GetNodeForRoom(ctx, roomName) - if err != nil { - return err - } - msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) - return r.WriteNodeRTC(ctx, node.Id, msg) + return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) } func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *livekit.RTCNodeMessage) error { @@ -237,82 +170,9 @@ func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *liv return r.writeRTCMessage(rtcSink, msg) } -func (r *RedisRouter) startParticipantRTC(ss *livekit.StartSession, participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey) error { - prometheus.IncrementParticipantRtcInit(1) - // find the node where the room is hosted at - rtcNode, err := r.GetNodeForRoom(r.ctx, livekit.RoomName(ss.RoomName)) - if err != nil { - return err - } - - if rtcNode.Id != r.currentNode.Id { - err = ErrIncorrectRTCNode - logger.Errorw("called participant on incorrect node", err, - "rtcNode", rtcNode, - ) - return err - } - - if err := r.SetParticipantRTCNode(participantKey, participantKeyB62, rtcNode.Id); err != nil { - return err - } - - // find signal node to send responses back - signalNode, err := r.getParticipantSignalNode(livekit.ConnectionID(ss.ConnectionId)) - if err != nil { - return err - } - - // treat it as a new participant connecting - if r.onNewParticipant == nil { - return ErrHandlerNotDefined - } - - // we do not want to re-use the same response sink - // the previous rtc worker thread is still consuming off of it. - // we'll want to sever the connection and switch to the new one - r.lock.RLock() - var requestChan *MessageChannel - var ok bool - var pkey livekit.ParticipantKey - if participantKeyB62 != "" { - requestChan, ok = r.requestChannels[string(participantKeyB62)] - pkey = participantKeyB62 - } else { - requestChan, ok = r.requestChannels[string(participantKey)] - pkey = participantKey - } - r.lock.RUnlock() - if ok { - requestChan.Close() - } - - pi, err := ParticipantInitFromStartSession(ss, r.currentNode.Region) - if err != nil { - return err - } - - reqChan := r.getOrCreateMessageChannel(r.requestChannels, string(pkey)) - resSink := NewSignalNodeSink(r.rc, livekit.NodeID(signalNode), livekit.ConnectionID(ss.ConnectionId)) - go func() { - err := r.onNewParticipant( - r.ctx, - livekit.RoomName(ss.RoomName), - *pi, - reqChan, - resSink, - ) - if err != nil { - logger.Errorw("could not handle new participant", err, - "room", ss.RoomName, - "participant", ss.Identity, - ) - // cleanup request channels - reqChan.Close() - resSink.Close() - } - }() - return nil +func (r *LocalRouter) writeRTCMessage(sink MessageSink, msg *livekit.RTCNodeMessage) error { + msg.SenderTime = time.Now().Unix() + return sink.WriteMessage(msg) } func (r *RedisRouter) Start() error { @@ -352,58 +212,6 @@ func (r *RedisRouter) Stop() { r.cancel() } -func (r *RedisRouter) SetParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, nodeID string) error { - var err error - if participantKey != "" { - err1 := r.rc.Set(r.ctx, participantRTCKey(participantKey), nodeID, participantMappingTTL).Err() - if err1 != nil { - err = errors.Wrap(err, "could not set rtc node") - } - } - if participantKeyB62 != "" { - err2 := r.rc.Set(r.ctx, participantRTCKey(participantKeyB62), nodeID, participantMappingTTL).Err() - if err2 != nil { - err = errors.Wrap(err, "could not set rtc node") - } - } - return err -} - -func (r *RedisRouter) setParticipantSignalNode(connectionID livekit.ConnectionID, nodeID string) error { - if err := r.rc.Set(r.ctx, participantSignalKey(connectionID), nodeID, participantMappingTTL).Err(); err != nil { - return errors.Wrap(err, "could not set signal node") - } - return nil -} - -func (r *RedisRouter) getParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey) (string, error) { - var val string - var err error - if participantKeyB62 != "" { - val, err = r.rc.Get(r.ctx, participantRTCKey(participantKeyB62)).Result() - if err == redis.Nil { - val, err = r.rc.Get(r.ctx, participantRTCKey(participantKey)).Result() - if err == redis.Nil { - err = ErrNodeNotFound - } - } - } else { - val, err = r.rc.Get(r.ctx, participantRTCKey(participantKey)).Result() - if err == redis.Nil { - err = ErrNodeNotFound - } - } - return val, err -} - -func (r *RedisRouter) getParticipantSignalNode(connectionID livekit.ConnectionID) (nodeID string, err error) { - val, err := r.rc.Get(r.ctx, participantSignalKey(connectionID)).Result() - if err == redis.Nil { - err = ErrNodeNotFound - } - return val, err -} - // update node stats and cleanup func (r *RedisRouter) statsWorker() { goroutineDumped := false @@ -444,9 +252,8 @@ func (r *RedisRouter) redisWorker(startedChan chan struct{}) { }() logger.Debugw("starting redisWorker", "nodeID", r.currentNode.Id) - sigChannel := signalNodeChannel(livekit.NodeID(r.currentNode.Id)) rtcChannel := rtcNodeChannel(livekit.NodeID(r.currentNode.Id)) - r.pubsub = r.rc.Subscribe(r.ctx, sigChannel, rtcChannel) + r.pubsub = r.rc.Subscribe(r.ctx, rtcChannel) close(startedChan) for msg := range r.pubsub.Channel() { @@ -454,20 +261,7 @@ func (r *RedisRouter) redisWorker(startedChan chan struct{}) { return } - if msg.Channel == sigChannel { - sm := livekit.SignalNodeMessage{} - if err := proto.Unmarshal([]byte(msg.Payload), &sm); err != nil { - logger.Errorw("could not unmarshal signal message on sigchan", err) - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) - continue - } - if err := r.handleSignalMessage(&sm); err != nil { - logger.Errorw("error processing signal message", err) - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) - continue - } - prometheus.MessageCounter.WithLabelValues("signal", "success").Add(1) - } else if msg.Channel == rtcChannel { + if msg.Channel == rtcChannel { rm := livekit.RTCNodeMessage{} if err := proto.Unmarshal([]byte(msg.Payload), &rm); err != nil { logger.Errorw("could not unmarshal RTC message on rtcchan", err) @@ -484,62 +278,8 @@ func (r *RedisRouter) redisWorker(startedChan chan struct{}) { } } -func (r *RedisRouter) handleSignalMessage(sm *livekit.SignalNodeMessage) error { - connectionID := sm.ConnectionId - - r.lock.RLock() - resSink := r.responseChannels[connectionID] - r.lock.RUnlock() - - // if a client closed the channel, then sent more messages after that, - if resSink == nil { - return nil - } - - switch rmb := sm.Message.(type) { - case *livekit.SignalNodeMessage_Response: - // logger.Debugw("forwarding signal message", - // "connID", connectionID, - // "type", fmt.Sprintf("%T", rmb.Response.Message)) - if err := resSink.WriteMessage(rmb.Response); err != nil { - return err - } - - case *livekit.SignalNodeMessage_EndSession: - // logger.Debugw("received EndSession, closing signal connection", - // "connID", connectionID) - resSink.Close() - } - return nil -} - func (r *RedisRouter) handleRTCMessage(rm *livekit.RTCNodeMessage) error { - pKey := livekit.ParticipantKey(rm.ParticipantKey) - pKeyB62 := livekit.ParticipantKey(rm.ParticipantKeyB62) - - switch rmb := rm.Message.(type) { - case *livekit.RTCNodeMessage_StartSession: - // RTC session should start on this node - if err := r.startParticipantRTC(rmb.StartSession, pKey, pKeyB62); err != nil { - return errors.Wrap(err, "could not start participant") - } - - case *livekit.RTCNodeMessage_Request: - r.lock.RLock() - var requestChan *MessageChannel - if pKeyB62 != "" { - requestChan = r.requestChannels[string(pKeyB62)] - } else { - requestChan = r.requestChannels[string(pKey)] - } - r.lock.RUnlock() - if requestChan == nil { - return ErrChannelClosed - } - if err := requestChan.WriteMessage(rmb.Request); err != nil { - return err - } - + switch rm.Message.(type) { case *livekit.RTCNodeMessage_KeepAlive: if time.Since(time.Unix(rm.SenderTime, 0)) > statsUpdateInterval { logger.Infow("keep alive too old, skipping", "senderTime", rm.SenderTime) @@ -566,24 +306,6 @@ func (r *RedisRouter) handleRTCMessage(rm *livekit.RTCNodeMessage) error { if err := r.RegisterNode(); err != nil { logger.Errorw("could not update node", err) } - - default: - // route it to handler - if r.onRTCMessage != nil { - var roomName livekit.RoomName - var identity livekit.ParticipantIdentity - var err error - if pKeyB62 != "" { - roomName, identity, err = parseParticipantKey(pKeyB62) - } - if err != nil || pKeyB62 == "" { - roomName, identity, err = parseParticipantKeyLegacy(pKey) - } - if err != nil { - return err - } - r.onRTCMessage(r.ctx, roomName, identity, rm) - } } return nil } diff --git a/pkg/routing/routingfakes/fake_router.go b/pkg/routing/routingfakes/fake_router.go index b52f36bb0..88fa9bf58 100644 --- a/pkg/routing/routingfakes/fake_router.go +++ b/pkg/routing/routingfakes/fake_router.go @@ -62,16 +62,6 @@ type FakeRouter struct { result1 []*livekit.Node result2 error } - OnNewParticipantRTCStub func(routing.NewParticipantCallback) - onNewParticipantRTCMutex sync.RWMutex - onNewParticipantRTCArgsForCall []struct { - arg1 routing.NewParticipantCallback - } - OnRTCMessageStub func(routing.RTCMessageCallback) - onRTCMessageMutex sync.RWMutex - onRTCMessageArgsForCall []struct { - arg1 routing.RTCMessageCallback - } RegisterNodeStub func() error registerNodeMutex sync.RWMutex registerNodeArgsForCall []struct { @@ -144,33 +134,6 @@ type FakeRouter struct { unregisterNodeReturnsOnCall map[int]struct { result1 error } - WriteParticipantRTCStub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity, *livekit.RTCNodeMessage) error - writeParticipantRTCMutex sync.RWMutex - writeParticipantRTCArgsForCall []struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 livekit.ParticipantIdentity - arg4 *livekit.RTCNodeMessage - } - writeParticipantRTCReturns struct { - result1 error - } - writeParticipantRTCReturnsOnCall map[int]struct { - result1 error - } - WriteRoomRTCStub func(context.Context, livekit.RoomName, *livekit.RTCNodeMessage) error - writeRoomRTCMutex sync.RWMutex - writeRoomRTCArgsForCall []struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 *livekit.RTCNodeMessage - } - writeRoomRTCReturns struct { - result1 error - } - writeRoomRTCReturnsOnCall map[int]struct { - result1 error - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -435,70 +398,6 @@ func (fake *FakeRouter) ListNodesReturnsOnCall(i int, result1 []*livekit.Node, r }{result1, result2} } -func (fake *FakeRouter) OnNewParticipantRTC(arg1 routing.NewParticipantCallback) { - fake.onNewParticipantRTCMutex.Lock() - fake.onNewParticipantRTCArgsForCall = append(fake.onNewParticipantRTCArgsForCall, struct { - arg1 routing.NewParticipantCallback - }{arg1}) - stub := fake.OnNewParticipantRTCStub - fake.recordInvocation("OnNewParticipantRTC", []interface{}{arg1}) - fake.onNewParticipantRTCMutex.Unlock() - if stub != nil { - fake.OnNewParticipantRTCStub(arg1) - } -} - -func (fake *FakeRouter) OnNewParticipantRTCCallCount() int { - fake.onNewParticipantRTCMutex.RLock() - defer fake.onNewParticipantRTCMutex.RUnlock() - return len(fake.onNewParticipantRTCArgsForCall) -} - -func (fake *FakeRouter) OnNewParticipantRTCCalls(stub func(routing.NewParticipantCallback)) { - fake.onNewParticipantRTCMutex.Lock() - defer fake.onNewParticipantRTCMutex.Unlock() - fake.OnNewParticipantRTCStub = stub -} - -func (fake *FakeRouter) OnNewParticipantRTCArgsForCall(i int) routing.NewParticipantCallback { - fake.onNewParticipantRTCMutex.RLock() - defer fake.onNewParticipantRTCMutex.RUnlock() - argsForCall := fake.onNewParticipantRTCArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeRouter) OnRTCMessage(arg1 routing.RTCMessageCallback) { - fake.onRTCMessageMutex.Lock() - fake.onRTCMessageArgsForCall = append(fake.onRTCMessageArgsForCall, struct { - arg1 routing.RTCMessageCallback - }{arg1}) - stub := fake.OnRTCMessageStub - fake.recordInvocation("OnRTCMessage", []interface{}{arg1}) - fake.onRTCMessageMutex.Unlock() - if stub != nil { - fake.OnRTCMessageStub(arg1) - } -} - -func (fake *FakeRouter) OnRTCMessageCallCount() int { - fake.onRTCMessageMutex.RLock() - defer fake.onRTCMessageMutex.RUnlock() - return len(fake.onRTCMessageArgsForCall) -} - -func (fake *FakeRouter) OnRTCMessageCalls(stub func(routing.RTCMessageCallback)) { - fake.onRTCMessageMutex.Lock() - defer fake.onRTCMessageMutex.Unlock() - fake.OnRTCMessageStub = stub -} - -func (fake *FakeRouter) OnRTCMessageArgsForCall(i int) routing.RTCMessageCallback { - fake.onRTCMessageMutex.RLock() - defer fake.onRTCMessageMutex.RUnlock() - argsForCall := fake.onRTCMessageArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeRouter) RegisterNode() error { fake.registerNodeMutex.Lock() ret, specificReturn := fake.registerNodeReturnsOnCall[len(fake.registerNodeArgsForCall)] @@ -864,133 +763,6 @@ func (fake *FakeRouter) UnregisterNodeReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeRouter) WriteParticipantRTC(arg1 context.Context, arg2 livekit.RoomName, arg3 livekit.ParticipantIdentity, arg4 *livekit.RTCNodeMessage) error { - fake.writeParticipantRTCMutex.Lock() - ret, specificReturn := fake.writeParticipantRTCReturnsOnCall[len(fake.writeParticipantRTCArgsForCall)] - fake.writeParticipantRTCArgsForCall = append(fake.writeParticipantRTCArgsForCall, struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 livekit.ParticipantIdentity - arg4 *livekit.RTCNodeMessage - }{arg1, arg2, arg3, arg4}) - stub := fake.WriteParticipantRTCStub - fakeReturns := fake.writeParticipantRTCReturns - fake.recordInvocation("WriteParticipantRTC", []interface{}{arg1, arg2, arg3, arg4}) - fake.writeParticipantRTCMutex.Unlock() - if stub != nil { - return stub(arg1, arg2, arg3, arg4) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRouter) WriteParticipantRTCCallCount() int { - fake.writeParticipantRTCMutex.RLock() - defer fake.writeParticipantRTCMutex.RUnlock() - return len(fake.writeParticipantRTCArgsForCall) -} - -func (fake *FakeRouter) WriteParticipantRTCCalls(stub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity, *livekit.RTCNodeMessage) error) { - fake.writeParticipantRTCMutex.Lock() - defer fake.writeParticipantRTCMutex.Unlock() - fake.WriteParticipantRTCStub = stub -} - -func (fake *FakeRouter) WriteParticipantRTCArgsForCall(i int) (context.Context, livekit.RoomName, livekit.ParticipantIdentity, *livekit.RTCNodeMessage) { - fake.writeParticipantRTCMutex.RLock() - defer fake.writeParticipantRTCMutex.RUnlock() - argsForCall := fake.writeParticipantRTCArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 -} - -func (fake *FakeRouter) WriteParticipantRTCReturns(result1 error) { - fake.writeParticipantRTCMutex.Lock() - defer fake.writeParticipantRTCMutex.Unlock() - fake.WriteParticipantRTCStub = nil - fake.writeParticipantRTCReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRouter) WriteParticipantRTCReturnsOnCall(i int, result1 error) { - fake.writeParticipantRTCMutex.Lock() - defer fake.writeParticipantRTCMutex.Unlock() - fake.WriteParticipantRTCStub = nil - if fake.writeParticipantRTCReturnsOnCall == nil { - fake.writeParticipantRTCReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.writeParticipantRTCReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeRouter) WriteRoomRTC(arg1 context.Context, arg2 livekit.RoomName, arg3 *livekit.RTCNodeMessage) error { - fake.writeRoomRTCMutex.Lock() - ret, specificReturn := fake.writeRoomRTCReturnsOnCall[len(fake.writeRoomRTCArgsForCall)] - fake.writeRoomRTCArgsForCall = append(fake.writeRoomRTCArgsForCall, struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 *livekit.RTCNodeMessage - }{arg1, arg2, arg3}) - stub := fake.WriteRoomRTCStub - fakeReturns := fake.writeRoomRTCReturns - fake.recordInvocation("WriteRoomRTC", []interface{}{arg1, arg2, arg3}) - fake.writeRoomRTCMutex.Unlock() - if stub != nil { - return stub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRouter) WriteRoomRTCCallCount() int { - fake.writeRoomRTCMutex.RLock() - defer fake.writeRoomRTCMutex.RUnlock() - return len(fake.writeRoomRTCArgsForCall) -} - -func (fake *FakeRouter) WriteRoomRTCCalls(stub func(context.Context, livekit.RoomName, *livekit.RTCNodeMessage) error) { - fake.writeRoomRTCMutex.Lock() - defer fake.writeRoomRTCMutex.Unlock() - fake.WriteRoomRTCStub = stub -} - -func (fake *FakeRouter) WriteRoomRTCArgsForCall(i int) (context.Context, livekit.RoomName, *livekit.RTCNodeMessage) { - fake.writeRoomRTCMutex.RLock() - defer fake.writeRoomRTCMutex.RUnlock() - argsForCall := fake.writeRoomRTCArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeRouter) WriteRoomRTCReturns(result1 error) { - fake.writeRoomRTCMutex.Lock() - defer fake.writeRoomRTCMutex.Unlock() - fake.WriteRoomRTCStub = nil - fake.writeRoomRTCReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRouter) WriteRoomRTCReturnsOnCall(i int, result1 error) { - fake.writeRoomRTCMutex.Lock() - defer fake.writeRoomRTCMutex.Unlock() - fake.WriteRoomRTCStub = nil - if fake.writeRoomRTCReturnsOnCall == nil { - fake.writeRoomRTCReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.writeRoomRTCReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeRouter) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -1004,10 +776,6 @@ func (fake *FakeRouter) Invocations() map[string][][]interface{} { defer fake.getRegionMutex.RUnlock() fake.listNodesMutex.RLock() defer fake.listNodesMutex.RUnlock() - fake.onNewParticipantRTCMutex.RLock() - defer fake.onNewParticipantRTCMutex.RUnlock() - fake.onRTCMessageMutex.RLock() - defer fake.onRTCMessageMutex.RUnlock() fake.registerNodeMutex.RLock() defer fake.registerNodeMutex.RUnlock() fake.removeDeadNodesMutex.RLock() @@ -1022,10 +790,6 @@ func (fake *FakeRouter) Invocations() map[string][][]interface{} { defer fake.stopMutex.RUnlock() fake.unregisterNodeMutex.RLock() defer fake.unregisterNodeMutex.RUnlock() - fake.writeParticipantRTCMutex.RLock() - defer fake.writeParticipantRTCMutex.RUnlock() - fake.writeRoomRTCMutex.RLock() - defer fake.writeRoomRTCMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 6d122fd6e..51ab919a0 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -101,7 +101,7 @@ func NewLocalRoomManager( return nil, err } - r := &RoomManager{ + return &RoomManager{ config: conf, rtcConfig: rtcConf, currentNode: currentNode, @@ -126,12 +126,7 @@ func NewLocalRoomManager( Region: conf.Region, NodeId: currentNode.Id, }, - } - - // hook up to router - router.OnNewParticipantRTC(r.StartSession) - router.OnRTCMessage(r.handleRTCMessage) - return r, nil + }, nil } func (r *RoomManager) GetRoom(_ context.Context, roomName livekit.RoomName) *rtc.Room { @@ -638,26 +633,6 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa } } -// handles RTC messages resulted from Room API calls -func (r *RoomManager) handleRTCMessage(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) { - switch rm := msg.Message.(type) { - case *livekit.RTCNodeMessage_RemoveParticipant: - r.RemoveParticipant(ctx, rm.RemoveParticipant) - case *livekit.RTCNodeMessage_MuteTrack: - r.MutePublishedTrack(ctx, rm.MuteTrack) - case *livekit.RTCNodeMessage_UpdateParticipant: - r.UpdateParticipant(ctx, rm.UpdateParticipant) - case *livekit.RTCNodeMessage_DeleteRoom: - r.DeleteRoom(ctx, rm.DeleteRoom) - case *livekit.RTCNodeMessage_UpdateSubscriptions: - r.UpdateSubscriptions(ctx, rm.UpdateSubscriptions) - case *livekit.RTCNodeMessage_SendData: - r.SendData(ctx, rm.SendData) - case *livekit.RTCNodeMessage_UpdateRoomMetadata: - r.UpdateRoomMetadata(ctx, rm.UpdateRoomMetadata) - } -} - type participantReq interface { GetRoom() string GetIdentity() string diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 2076b5ca4..003da7f19 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -16,20 +16,16 @@ package service import ( "context" - "fmt" "strconv" "time" "github.com/pkg/errors" - "github.com/thoas/go-funk" "github.com/twitchtv/twirp" - "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" ) @@ -170,39 +166,7 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, twirpAuthError(err) } - if s.psrpcConf.Enabled { - return s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) - } - - if _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false); err == ErrRoomNotFound { - return nil, twirp.NotFoundError("room not found") - } - - err := s.router.WriteRoomRTC(ctx, livekit.RoomName(req.Room), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_DeleteRoom{ - DeleteRoom: req, - }, - }) - if err != nil { - return nil, err - } - - // we should not return until when the room is confirmed deleted - err = s.confirmExecution(func() error { - _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false) - if err == nil { - return ErrOperationFailed - } else if err != ErrRoomNotFound { - return err - } else { - return nil - } - }) - if err != nil { - return nil, err - } - - return &livekit.DeleteRoomResponse{}, nil + return s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) } func (s *RoomService) ListParticipants(ctx context.Context, req *livekit.ListParticipantsRequest) (*livekit.ListParticipantsResponse, error) { @@ -247,34 +211,7 @@ func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomPa return nil, twirp.NotFoundError("participant not found") } - if s.psrpcConf.Enabled { - return s.participantClient.RemoveParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) - } - - err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_RemoveParticipant{ - RemoveParticipant: req, - }, - }) - if err != nil { - return nil, err - } - - err = s.confirmExecution(func() error { - _, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) - if err == ErrParticipantNotFound { - return nil - } else if err != nil { - return err - } else { - return ErrOperationFailed - } - }) - if err != nil { - return nil, err - } - - return &livekit.RemoveParticipantResponse{}, nil + return s.participantClient.RemoveParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) } func (s *RoomService) MutePublishedTrack(ctx context.Context, req *livekit.MuteRoomTrackRequest) (*livekit.MuteRoomTrackResponse, error) { @@ -283,47 +220,7 @@ func (s *RoomService) MutePublishedTrack(ctx context.Context, req *livekit.MuteR return nil, twirpAuthError(err) } - if s.psrpcConf.Enabled { - return s.participantClient.MutePublishedTrack(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) - } - - err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_MuteTrack{ - MuteTrack: req, - }, - }) - if err != nil { - return nil, err - } - - var track *livekit.TrackInfo - err = s.confirmExecution(func() error { - p, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) - if err != nil { - return err - } - // ensure track is muted - t := funk.Find(p.Tracks, func(t *livekit.TrackInfo) bool { - return t.Sid == req.TrackSid - }) - var ok bool - track, ok = t.(*livekit.TrackInfo) - if !ok { - return ErrTrackNotFound - } - if track.Muted != req.Muted { - return ErrOperationFailed - } - return nil - }) - if err != nil { - return nil, err - } - - res := &livekit.MuteRoomTrackResponse{ - Track: track, - } - return res, nil + return s.participantClient.MutePublishedTrack(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) } func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.UpdateParticipantRequest) (*livekit.ParticipantInfo, error) { @@ -337,42 +234,7 @@ func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.Update return nil, twirpAuthError(err) } - if s.psrpcConf.Enabled { - return s.participantClient.UpdateParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) - } - - err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_UpdateParticipant{ - UpdateParticipant: req, - }, - }) - if err != nil { - return nil, err - } - - var participant *livekit.ParticipantInfo - var detailedError error - err = s.confirmExecution(func() error { - participant, err = s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) - if err != nil { - return err - } - if req.Metadata != "" && participant.Metadata != req.Metadata { - detailedError = fmt.Errorf("metadata does not match") - return ErrOperationFailed - } - if req.Permission != nil && !proto.Equal(req.Permission, participant.Permission) { - detailedError = fmt.Errorf("permissions do not match, expected: %v, actual: %v", req.Permission, participant.Permission) - return ErrOperationFailed - } - return nil - }) - if err != nil { - logger.Warnw("could not confirm participant update", detailedError) - return nil, err - } - - return participant, nil + return s.participantClient.UpdateParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) } func (s *RoomService) UpdateSubscriptions(ctx context.Context, req *livekit.UpdateSubscriptionsRequest) (*livekit.UpdateSubscriptionsResponse, error) { @@ -386,20 +248,7 @@ func (s *RoomService) UpdateSubscriptions(ctx context.Context, req *livekit.Upda return nil, twirpAuthError(err) } - if s.psrpcConf.Enabled { - return s.participantClient.UpdateSubscriptions(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) - } - - err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_UpdateSubscriptions{ - UpdateSubscriptions: req, - }, - }) - if err != nil { - return nil, err - } - - return &livekit.UpdateSubscriptionsResponse{}, nil + return s.participantClient.UpdateSubscriptions(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) } func (s *RoomService) SendData(ctx context.Context, req *livekit.SendDataRequest) (*livekit.SendDataResponse, error) { @@ -409,20 +258,7 @@ func (s *RoomService) SendData(ctx context.Context, req *livekit.SendDataRequest return nil, twirpAuthError(err) } - if s.psrpcConf.Enabled { - return s.roomClient.SendData(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) - } - - err := s.router.WriteRoomRTC(ctx, roomName, &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_SendData{ - SendData: req, - }, - }) - if err != nil { - return nil, err - } - - return &livekit.SendDataResponse{}, nil + return s.roomClient.SendData(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) } func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.UpdateRoomMetadataRequest) (*livekit.Room, error) { @@ -451,20 +287,9 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat return nil, err } - if s.psrpcConf.Enabled { - _, err := s.roomClient.UpdateRoomMetadata(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) - if err != nil { - return nil, err - } - } else { - err = s.router.WriteRoomRTC(ctx, livekit.RoomName(req.Room), &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_UpdateRoomMetadata{ - UpdateRoomMetadata: req, - }, - }) - if err != nil { - return nil, err - } + _, err = s.roomClient.UpdateRoomMetadata(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + if err != nil { + return nil, err } err = s.confirmExecution(func() error { @@ -494,14 +319,6 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat return room, nil } -func (s *RoomService) writeParticipantMessage(ctx context.Context, room livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { - if err := EnsureAdminPermission(ctx, room); err != nil { - return twirpAuthError(err) - } - - return s.router.WriteParticipantRTC(ctx, room, identity, msg) -} - func (s *RoomService) confirmExecution(f func() error) error { expired := time.After(s.apiConf.ExecutionTimeout) var err error diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index f9de24a51..0237117f4 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -33,26 +33,6 @@ import ( ) func TestDeleteRoom(t *testing.T) { - t.Run("delete non-existent", func(t *testing.T) { - svc := newTestRoomService(config.RoomConfig{}) - grant := &auth.ClaimGrants{ - Video: &auth.VideoGrant{ - RoomCreate: true, - }, - } - ctx := service.WithGrants(context.Background(), grant) - svc.store.LoadRoomReturns(nil, nil, service.ErrRoomNotFound) - _, err := svc.DeleteRoom(ctx, &livekit.DeleteRoomRequest{ - Room: "testroom", - }) - require.Error(t, err) - if terr, ok := err.(twirp.Error); ok { - require.Equal(t, twirp.NotFound, terr.Code()) - } else { - require.Fail(t, "should be twirp error") - } - }) - t.Run("missing permissions", func(t *testing.T) { svc := newTestRoomService(config.RoomConfig{}) grant := &auth.ClaimGrants{ diff --git a/pkg/service/signal.go b/pkg/service/signal.go index a0abc5952..6c4ff820e 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -102,27 +102,17 @@ func (s *defaultSessionHandler) HandleSession( ) error { prometheus.IncrementParticipantRtcInit(1) - if rr, ok := s.router.(*routing.RedisRouter); ok { - rtcNode, err := s.router.GetNodeForRoom(ctx, roomName) - if err != nil { - return err - } + rtcNode, err := s.router.GetNodeForRoom(ctx, roomName) + if err != nil { + return err + } - if rtcNode.Id != s.currentNode.Id { - err = routing.ErrIncorrectRTCNode - logger.Errorw("called participant on incorrect node", err, - "rtcNode", rtcNode, - ) - return err - } - - pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) - pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) - - // RTC session should start on this node - if err := rr.SetParticipantRTCNode(pKey, pKeyB62, s.currentNode.Id); err != nil { - return err - } + if rtcNode.Id != s.currentNode.Id { + err = routing.ErrIncorrectRTCNode + logger.Errorw("called participant on incorrect node", err, + "rtcNode", rtcNode, + ) + return err } return s.roomManager.StartSession(ctx, roomName, pi, requestSource, responseSink) diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index a80935182..951837176 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -40,7 +40,6 @@ func init() { func TestSignal(t *testing.T) { cfg := config.SignalRelayConfig{ - Enabled: false, RetryTimeout: 30 * time.Second, MinRetryInterval: 500 * time.Millisecond, MaxRetryInterval: 5 * time.Second, diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 9c9a741e5..5b2a3aab6 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -50,7 +50,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - router := routing.CreateRouter(conf, universalClient, currentNode, signalClient) + router := routing.CreateRouter(universalClient, currentNode, signalClient) objectStore := createStore(universalClient) roomAllocator, err := NewRoomAllocator(conf, router, objectStore) if err != nil { @@ -149,7 +149,7 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi if err != nil { return nil, err } - router := routing.CreateRouter(conf, universalClient, currentNode, signalClient) + router := routing.CreateRouter(universalClient, currentNode, signalClient) return router, nil } diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 7a11b353e..0ce079c23 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -183,7 +183,6 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer { conf.RTC.TCPPort = port + 2 conf.Redis.Address = "localhost:6379" conf.Keys = map[string]string{testApiKey: testApiSecret} - conf.SignalRelay.Enabled = true currentNode, err := routing.NewLocalNode(conf) if err != nil { From 5b4848e7723f712973a3a6cd8c23497c0c19a8fa Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 16 Jan 2024 12:07:29 +0800 Subject: [PATCH 076/114] remove dd debug logs (#2387) --- .../dependencydescriptor.go | 20 +++++++++---------- pkg/sfu/videolayerselector/framechain.go | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 8347114bc..7a29e5d95 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -71,7 +71,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r ddwdt := extPkt.DependencyDescriptor if ddwdt == nil { // packet doesn't have dependency descriptor - d.logger.Debugw(fmt.Sprintf("drop packet, no DD, incoming %v, sn: %d, isKeyFrame: %v", extPkt.VideoLayer, extPkt.Packet.SequenceNumber, extPkt.KeyFrame)) + // d.logger.Debugw(fmt.Sprintf("drop packet, no DD, incoming %v, sn: %d, isKeyFrame: %v", extPkt.VideoLayer, extPkt.Packet.SequenceNumber, extPkt.KeyFrame)) return } @@ -114,15 +114,15 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } if ddwdt.StructureUpdated { - d.logger.Debugw("update dependency structure", - "structureID", dd.AttachedStructure.StructureId, - "structure", dd.AttachedStructure, - "decodeTargets", ddwdt.DecodeTargets, - "efn", extFrameNum, - "sn", extPkt.Packet.SequenceNumber, - "isKeyFrame", extPkt.KeyFrame, - "currentKeyframe", d.extKeyFrameNum, - ) + // d.logger.Debugw("update dependency structure", + // "structureID", dd.AttachedStructure.StructureId, + // "structure", dd.AttachedStructure, + // "decodeTargets", ddwdt.DecodeTargets, + // "efn", extFrameNum, + // "sn", extPkt.Packet.SequenceNumber, + // "isKeyFrame", extPkt.KeyFrame, + // "currentKeyframe", d.extKeyFrameNum, + // ) d.updateDependencyStructure(dd.AttachedStructure, ddwdt.DecodeTargets, extFrameNum) } diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 0777fb0fa..7b226ed0c 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -54,7 +54,7 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate if fd.ChainDiffs[fc.chainIdx] == 0 { if fc.broken { fc.broken = false - fc.logger.Debugw("frame chain intact", "chanIdx", fc.chainIdx, "frame", extFrameNum) + // fc.logger.Debugw("frame chain intact", "chanIdx", fc.chainIdx, "frame", extFrameNum) } fc.expectFrames = fc.expectFrames[:0] return true @@ -86,7 +86,7 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate if !intact { fc.broken = true - fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", sd, "frame", extFrameNum, "prevFrame", prevFrameInChain) + // fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", sd, "frame", extFrameNum, "prevFrame", prevFrameInChain) } return intact } @@ -100,7 +100,7 @@ func (fc *FrameChain) OnExpectFrameChanged(frameNum uint64, decision selectorDec if f == frameNum { if decision != selectorDecisionForwarded { fc.broken = true - fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", decision, "frame", frameNum) + // fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", decision, "frame", frameNum) } fc.expectFrames[i] = fc.expectFrames[len(fc.expectFrames)-1] fc.expectFrames = fc.expectFrames[:len(fc.expectFrames)-1] From f29a28611b944243f48983b538cfa39d34e299c5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 16 Jan 2024 12:15:13 +0530 Subject: [PATCH 077/114] Prevent writable race. (#2388) It is possible that onBindAndConnectedChanged gets executed in such a way that `writable` does not have the correct value in some very rare timing case (i. e. case like two executions of the function is racing and one atomic was read on first exeuction and second execution runs and sets `writable` and then first execution completes the sets `writable` to incorrect value based on stale read of first execution). Prevent it by executing under bind lock. --- pkg/sfu/downtrack.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 696b3a8c8..a2185063f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -411,6 +411,7 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.onBinding(nil) } d.bound.Store(true) + d.onBindAndConnectedChange() d.bindLock.Unlock() // Bind is called under RTPSender.mu lock, call the RTPSender.GetParameters in goroutine to avoid deadlock @@ -425,9 +426,7 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, }() d.forwarder.DetermineCodec(d.codec, d.params.Receiver.HeaderExtensions()) - d.params.Logger.Debugw("downtrack bound") - d.onBindAndConnectedChange() return codec, nil } @@ -435,8 +434,10 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, // Unbind implements the teardown logic when the track is no longer needed. This happens // because a track has been stopped. func (d *DownTrack) Unbind(_ webrtc.TrackLocalContext) error { + d.bindLock.Lock() d.bound.Store(false) d.onBindAndConnectedChange() + d.bindLock.Unlock() return nil } @@ -1577,10 +1578,12 @@ func (d *DownTrack) handleRTCP(bytes []byte) { } func (d *DownTrack) SetConnected() { + d.bindLock.Lock() if !d.connected.Swap(true) { d.onBindAndConnectedChange() } d.params.Logger.Debugw("downtrack connected") + d.bindLock.Unlock() } // SetActivePaddingOnMuteUpTrack will enable padding on the track when its uptrack is muted. From 750d2b5765aece79e2b6a7c7a97284d8b6cd0b59 Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Wed, 17 Jan 2024 10:03:47 -0500 Subject: [PATCH 078/114] Update livekit/protocol (#2390) Fix API breakage with SIP --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/sip.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 9387bf323..7c27187bd 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83 + github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 60a8c98fc..5ac51dfe7 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83 h1:iYur8jpRGdpoFH2IZAXUEu/l4Gsp5sQticAXnSeUHGA= -github.com/livekit/protocol v1.9.4-0.20240105111749-a0e8241b1a83/go.mod h1:8db9KAaD8iYZLyewdgtJBQ70A63srl4AI9hBD9JJ0ps= +github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc h1:Tq5M+BJstua3RfFHyiBXwGzsJU8LcxoDjQzjjTQo7xQ= +github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/sip.go b/pkg/service/sip.go index c69540eed..8bba2fdca 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -195,7 +195,7 @@ func (s *SIPService) updateParticipant(ctx context.Context, info *livekit.SIPPar req.Username = trunk.OutboundUsername req.Password = trunk.OutboundPassword } - if _, err := s.psrpcClient.UpdateSIPParticipant(ctx, req); err != nil { + if _, err := s.psrpcClient.UpdateSIPParticipant(ctx, "", req); err != nil { logger.Errorw("cannot update sip participant", err) } } From 899067ba0f15b9e2246d795f23bfccd354e84377 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 17 Jan 2024 20:44:05 +0530 Subject: [PATCH 079/114] Simulation scenarios to disable signal channel on resume (#2389) * Add a simulation scenario to disconnect signal channel on resume - Requesting that scenario add that participant to a map with a timeout of 5 seconds. - If a resume (reconnect = 1) happens before the timeout, the signalling channel is closed immediately on resume. - There is a clean up worker which will remove entries from the map when they timout. - The participant is also removed from the map if the disconnect on resume is invoked once. * simulate disconnect signal on resume no messages * comment * comment * Close all retries * update deps * abort resume only if simulation applied * Revert SIP change --- go.mod | 4 +- go.sum | 8 +-- pkg/rtc/room.go | 112 +++++++++++++++++++++++++++++++----- pkg/rtc/transportmanager.go | 1 - pkg/rtc/types/interfaces.go | 6 ++ 5 files changed, 109 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 7c27187bd..d1e6884a7 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.31.0 // indirect + github.com/nats-io/nats.go v1.32.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -101,7 +101,7 @@ require ( golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect google.golang.org/grpc v1.60.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5ac51dfe7..344743c09 100644 --- a/go.sum +++ b/go.sum @@ -161,8 +161,8 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= -github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= +github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0= +github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -421,8 +421,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 8412788e9..1bb1f671c 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -53,6 +53,8 @@ const ( subscriberUpdateInterval = 3 * time.Second dataForwardLoadBalanceThreshold = 20 + + simulateDisconnectSignalTimeout = 5 * time.Second ) var ( @@ -66,6 +68,11 @@ type broadcastOptions struct { immediate bool } +type disconnectSignalOnResumeNoMessages struct { + expiry time.Time + closedCount int +} + type Room struct { lock sync.RWMutex @@ -108,6 +115,10 @@ type Room struct { onParticipantChanged func(p types.LocalParticipant) onRoomUpdated func() onClose func() + + simulationLock sync.Mutex + disconnectSignalOnResumeParticipants map[livekit.ParticipantIdentity]time.Time + disconnectSignalOnResumeNoMessagesParticipants map[livekit.ParticipantIdentity]*disconnectSignalOnResumeNoMessages } type ParticipantOptions struct { @@ -132,21 +143,23 @@ func NewRoom( livekit.RoomName(room.Name), livekit.RoomID(room.Sid), ), - config: config, - audioConfig: audioConfig, - telemetry: telemetry, - egressLauncher: egressLauncher, - agentClient: agentClient, - trackManager: NewRoomTrackManager(), - serverInfo: serverInfo, - participants: make(map[livekit.ParticipantIdentity]types.LocalParticipant), - participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), - participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), - hasPublished: make(map[livekit.ParticipantIdentity]bool), - bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), - batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), - closed: make(chan struct{}), - trailer: []byte(utils.RandomSecret()), + config: config, + audioConfig: audioConfig, + telemetry: telemetry, + egressLauncher: egressLauncher, + agentClient: agentClient, + trackManager: NewRoomTrackManager(), + serverInfo: serverInfo, + participants: make(map[livekit.ParticipantIdentity]types.LocalParticipant), + participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), + participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), + hasPublished: make(map[livekit.ParticipantIdentity]bool), + bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), + batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), + closed: make(chan struct{}), + trailer: []byte(utils.RandomSecret()), + disconnectSignalOnResumeParticipants: make(map[livekit.ParticipantIdentity]time.Time), + disconnectSignalOnResumeNoMessagesParticipants: make(map[livekit.ParticipantIdentity]*disconnectSignalOnResumeNoMessages), } r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) @@ -175,6 +188,7 @@ func NewRoom( go r.audioUpdateWorker() go r.connectionQualityWorker() go r.changeUpdateWorker() + go r.simulationCleanupWorker() return r } @@ -480,6 +494,26 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing p.SetSignalSourceValid(true) + // check for simulated signal disconnect on resume before sending any signal response messages + r.simulationLock.Lock() + if state, ok := r.disconnectSignalOnResumeNoMessagesParticipants[p.Identity()]; ok { + // WARNING: this uses knowledge that service layer tries internally + simulated := false + if time.Now().Before(state.expiry) { + state.closedCount++ + p.CloseSignalConnection(types.SignallingCloseReasonDisconnectOnResumeNoMessages) + simulated = true + } + if state.closedCount == 3 { + delete(r.disconnectSignalOnResumeNoMessagesParticipants, p.Identity()) + } + if simulated { + r.simulationLock.Unlock() + return nil + } + } + r.simulationLock.Unlock() + if err := p.HandleReconnectAndSendResponse(reason, &livekit.ReconnectResponse{ IceServers: iceServers, ClientConfiguration: p.GetClientConfiguration(), @@ -495,6 +529,17 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing _ = p.SendRoomUpdate(r.ToProto()) p.ICERestart(nil) + + // check for simulated signal disconnect on resume + r.simulationLock.Lock() + if timeout, ok := r.disconnectSignalOnResumeParticipants[p.Identity()]; ok { + if time.Now().Before(timeout) { + p.CloseSignalConnection(types.SignallingCloseReasonDisconnectOnResume) + } + delete(r.disconnectSignalOnResumeParticipants, p.Identity()) + } + r.simulationLock.Unlock() + return nil } @@ -849,6 +894,18 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen r.Logger.Infow("simulating subscriber bandwidth end", "participant", participant.Identity()) } participant.SetSubscriberChannelCapacity(scenario.SubscriberBandwidth) + case *livekit.SimulateScenario_DisconnectSignalOnResume: + participant.GetLogger().Infow("simulating disconnect signal on resume") + r.simulationLock.Lock() + r.disconnectSignalOnResumeParticipants[participant.Identity()] = time.Now().Add(simulateDisconnectSignalTimeout) + r.simulationLock.Unlock() + case *livekit.SimulateScenario_DisconnectSignalOnResumeNoMessages: + participant.GetLogger().Infow("simulating disconnect signal on resume before sending any response messages") + r.simulationLock.Lock() + r.disconnectSignalOnResumeNoMessagesParticipants[participant.Identity()] = &disconnectSignalOnResumeNoMessages{ + expiry: time.Now().Add(simulateDisconnectSignalTimeout), + } + r.simulationLock.Unlock() } return nil } @@ -1338,6 +1395,31 @@ func (r *Room) connectionQualityWorker() { } } +func (r *Room) simulationCleanupWorker() { + for { + if r.IsClosed() { + return + } + + now := time.Now() + r.simulationLock.Lock() + for identity, timeout := range r.disconnectSignalOnResumeParticipants { + if now.After(timeout) { + delete(r.disconnectSignalOnResumeParticipants, identity) + } + } + + for identity, state := range r.disconnectSignalOnResumeNoMessagesParticipants { + if now.After(state.expiry) { + delete(r.disconnectSignalOnResumeNoMessagesParticipants, identity) + } + } + r.simulationLock.Unlock() + + time.Sleep(10 * time.Second) + } +} + func (r *Room) launchPublisherAgent(p types.Participant) { if p == nil || p.IsRecorder() || p.IsAgent() { return diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 489f834e7..ec3b10600 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -187,7 +187,6 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro } t.signalSourceValid.Store(true) - return t, nil } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 05f7d5a2d..152513efc 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -209,6 +209,8 @@ const ( SignallingCloseReasonFullReconnectDataChannelError SignallingCloseReasonFullReconnectNegotiateFailed SignallingCloseReasonParticipantClose + SignallingCloseReasonDisconnectOnResume + SignallingCloseReasonDisconnectOnResumeNoMessages ) func (s SignallingCloseReason) String() string { @@ -231,6 +233,10 @@ func (s SignallingCloseReason) String() string { return "FULL_RECONNECT_NEGOTIATE_FAILED" case SignallingCloseReasonParticipantClose: return "PARTICIPANT_CLOSE" + case SignallingCloseReasonDisconnectOnResume: + return "DISCONNECT_ON_RESUME" + case SignallingCloseReasonDisconnectOnResumeNoMessages: + return "DISCONNECT_ON_RESUME_NO_MESSAGES" default: return fmt.Sprintf("%d", int(s)) } From fbd488adc3c6981089c5d3e0a1db2a4fbbcb4724 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 18 Jan 2024 06:46:34 -0800 Subject: [PATCH 080/114] remove participant key helpers (#2385) * remove participant key helpers * deps --- go.mod | 2 +- go.sum | 4 +- pkg/routing/utils.go | 80 --------------------------------------- pkg/routing/utils_test.go | 64 ------------------------------- 4 files changed, 3 insertions(+), 147 deletions(-) delete mode 100644 pkg/routing/utils.go delete mode 100644 pkg/routing/utils_test.go diff --git a/go.mod b/go.mod index d1e6884a7..501094499 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc + github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 344743c09..48fed3d29 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc h1:Tq5M+BJstua3RfFHyiBXwGzsJU8LcxoDjQzjjTQo7xQ= -github.com/livekit/protocol v1.9.5-0.20240117112816-6dc023d7edcc/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= +github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8 h1:E9s9KFCuKgYWYgaKz0ZmC7K3cPr8Iij77HbnwhQ4JZw= +github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/routing/utils.go b/pkg/routing/utils.go deleted file mode 100644 index 2e11fdbe2..000000000 --- a/pkg/routing/utils.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "fmt" - "strings" - - "github.com/jxskiss/base62" - - "github.com/livekit/protocol/livekit" -) - -func ParticipantKeyLegacy(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { - return livekit.ParticipantKey(string(roomName) + "|" + string(identity)) -} - -func parseParticipantKeyLegacy(pkey livekit.ParticipantKey) (roomName livekit.RoomName, identity livekit.ParticipantIdentity, err error) { - parts := strings.Split(string(pkey), "|") - if len(parts) == 2 { - roomName = livekit.RoomName(parts[0]) - identity = livekit.ParticipantIdentity(parts[1]) - return - } - - err = fmt.Errorf("invalid participant key: %s", pkey) - return -} - -func ParticipantKey(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { - return livekit.ParticipantKey(encode(string(roomName), string(identity))) -} - -func parseParticipantKey(pkey livekit.ParticipantKey) (roomName livekit.RoomName, identity livekit.ParticipantIdentity, err error) { - parts, err := decode(string(pkey)) - if err != nil { - return - } - if len(parts) == 2 { - roomName = livekit.RoomName(parts[0]) - identity = livekit.ParticipantIdentity(parts[1]) - return - } - - err = fmt.Errorf("invalid participant key: %s", pkey) - return -} - -func encode(str ...string) string { - encoded := make([]string, 0, len(str)) - for _, s := range str { - encoded = append(encoded, base62.EncodeToString([]byte(s))) - } - return strings.Join(encoded, "|") -} - -func decode(encoded string) ([]string, error) { - split := strings.Split(encoded, "|") - decoded := make([]string, 0, len(split)) - for _, s := range split { - part, err := base62.DecodeString(s) - if err != nil { - return nil, err - } - decoded = append(decoded, string(part)) - } - return decoded, nil -} diff --git a/pkg/routing/utils_test.go b/pkg/routing/utils_test.go deleted file mode 100644 index 8ae1e9b4c..000000000 --- a/pkg/routing/utils_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/livekit/protocol/livekit" -) - -func TestUtils_ParticipantKey(t *testing.T) { - // encode/decode empty - encoded := ParticipantKey("", "") - roomName, identity, err := parseParticipantKey(encoded) - require.NoError(t, err) - require.Equal(t, livekit.RoomName(""), roomName) - require.Equal(t, livekit.ParticipantIdentity(""), identity) - - // decode invalid - _, _, err = parseParticipantKey("abcd") - require.Error(t, err) - - // encode/decode without delimiter - encoded = ParticipantKey("room1", "identity1") - roomName, identity, err = parseParticipantKey(encoded) - require.NoError(t, err) - require.Equal(t, livekit.RoomName("room1"), roomName) - require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) - - // encode/decode with delimiter in roomName - encoded = ParticipantKey("room1|alter_room1", "identity1") - roomName, identity, err = parseParticipantKey(encoded) - require.NoError(t, err) - require.Equal(t, livekit.RoomName("room1|alter_room1"), roomName) - require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) - - // encode/decode with delimiter in identity - encoded = ParticipantKey("room1", "identity1|alter-identity1") - roomName, identity, err = parseParticipantKey(encoded) - require.NoError(t, err) - require.Equal(t, livekit.RoomName("room1"), roomName) - require.Equal(t, livekit.ParticipantIdentity("identity1|alter-identity1"), identity) - - // encode/decode with delimiter in both and multiple delimiters in both - encoded = ParticipantKey("room1|alter_room1|again_room1", "identity1|alter-identity1|again-identity1") - roomName, identity, err = parseParticipantKey(encoded) - require.NoError(t, err) - require.Equal(t, livekit.RoomName("room1|alter_room1|again_room1"), roomName) - require.Equal(t, livekit.ParticipantIdentity("identity1|alter-identity1|again-identity1"), identity) -} From e255b8a51d0a82fe78892eddfb7f2502275bb2eb Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Fri, 19 Jan 2024 00:48:23 +0100 Subject: [PATCH 081/114] update readme (#2392) --- .github/banner_dark.png | Bin 131711 -> 54220 bytes .github/banner_light.png | Bin 48704 -> 23647 bytes README.md | 21 ++++++++++++--------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/banner_dark.png b/.github/banner_dark.png index d19c687ce870844cf5403bb614645024b0a408b0..20c986b9740f8bf2349c0f6b5075c546fa03b4f0 100644 GIT binary patch literal 54220 zcmd42cRZZ!+BP}`qt`@qLJ&la77U|Hw20`vMawABXM#ZzLbQ+&b@bj6y-R{fg3(29 zVHh<=8*N+Px7T`~=XuxufB!M%o^oBk`#Sq^9_QJDiE}yQbc%^_yXaGQ(GN6C7!zCC$w|mxj<&pzgpo#~?5SC2h&TLOsE+7# zqX|8sY9F*&ya#_HT%Ro_QTdUE= z^6#m_O-hs-KQ1zas`ACrRh?YbH8g(_rYSpBIa9Tj2CNn3zrv_IP*!1j2ZawR{ z1-E%Na!Zv!{(SqL$@l+qw|8v7(qbu;Rw;c;U4on#?P-pkqZsgr5&KABVBCcBsH|v< zXd;|OeKh4fsI$KQ7N;2c)A2FYxBpS4PVFk7Fmwt)qa65i-tFEm^{&M;u`KG1KVg6H zTm9~qe^@*2BoclfiV~OK&EF3gk^{@dEEIIywEL@0+v>b5oL@%qD@B>nn0F0>sw{*t zgGW>AcC(D;hHtdTy`z~w`8uR_T>9Wlbd-_^T?qr(UG8kk&UCx|OnArji-yVf?Ib)K&F)UKYG8IO`G%qUTm+eHODhT|43bboh;S z{ay@bZ#f;@f=Z+^v|aJo_Z{VWt25%HfY81q^}jXdeNGrwaNe$SU)8Fa)WH}as5Ysv zkfKN~r$p3q(#yB85qW^O%_3Sc>%Hyvd^k4gTz2&?U2=V>u4^}uVJ|P9mvn>BsG%$e z{?`t1jxd`rWm^Z%zj%r2??e0MO`_5*GGCt@rIs7B_ki>pYJ^`^L^zTPN~?vHfB4Qa zMpu{jXxkNaEts86Y0HwK+g|AU=-LM=l)I;&z9mk|-@eMGin=@BM=khn1Y+u2h@362 ztnNLL{dB6yH5uj=JwCPCqKQeyPumA4r`O3g4ikM};s0pwL;Fzepoq>M#lQG1+jjsc z)5{7Y0Jhh@D3mXbUtg`E<~oI9%X#_|;W&p;w=PBnWDhT&pSfN%;xkmNSI(XGWi?AS zOX2$OM-!8C-naxlE=IB!a_Br^i`nJLWk|o(AHR~(`%%Tc<;ZxV{oA&go|bPZo9{8r z(EWa-c`aJly|JG@)^ozyzADt>S_~X_KhhQ%fGEoU6hvU7+L$7WzbVg6x=xAxowAC2 z3k)VE3HzI`?R*5BO`Y2YWRnB|{5pBLSGQp~z zA_KY`gSFb5r9-qrIvwaG7pE&H3yUuqSmMj_PIJvzPZzEQH!UiJN(2j1zY1_+8{KD8 zk~>-vVD||=i{eug+aCKUW6k8?$T-8KFj_VjIGT00 zduv-28bct~b@=J-RMr2$5cA=zg~#n(PNN)x>uO=LvjRSO`Kfh~9?#cgn@} z(o<|quaNyJ_QOWv8y^Sg`P>j43Apv&+$_L16mg26M5l9CMXa```c*QNcly#ossX79 z7kZ)cLmI@&*Fjm*1lleyMmSY+hu)3#}I9%$K;A}Ujnr*vkg1M7E#N# z^qumf?Vo=6q-}Oe9TH@IeWBfK>9l`wZ;oYeu;EW~9&luMFa52LQpXW+EMD)%#zWC_BG|m|{CkRG z8%%<~&O{`A2>grdP9ek4eTdb5t>Gv^irw%=k#~|Tgl8YKdKICy)VYE6J=ktTnO~vfFCD+0@m3AvCj7w%*0anZfE|KH? z)7PsaJjL|@bFtwmk17C09n@1)^VknmZXb!3Ddz{8WoGtVjrb)QQ3g_qQ3`pP!*WD( zvLtJddl%|bDZ3ht*c&JE#^&lOTjHezdE8j}%vv9N%Re-6@OV(9*|N(Py<6gL_lOvd zGr(KaGbd4DHD`yQ&&SH+G8O&EruNR}cP@CRwO1mfNPI6ouck}?Ex?pfV)xCG%)BVYurwd*t6K$+qC z(;{EXGQ|REo^ZB?IWy+xQ6NVSuU9z?0&7vNuPCOwRpqi&$>l16Tya7PuW)6z>paWZ zZ0!3$XK(H&u)`Gwo>V>h-WdMu_`1M!RJFrqqH*Zzn=+8^q$CkG>!thcq_FNB$z#{^F#rc#bp^K{W;Hg}G-3#Bo#}4?&T;wL9RdJcJl$D zp7H`)D3jK>r-Ft_zta|MYbx=mN>#baf+`3N3m1JaPz~6$MJS~E9x*Wjgq9{S0SS0! z^|rjy@0##~anDVR6d+jyeIpOZzFZL9v5t!SNkWD_QdtHvLGnQ;Z(&rarA$cnc*^pQ zvd>^_sSvF9SVdE?w35<+tLi#nr_1UAPf;%X{BHPV8QpRY81YzH4o~xo0^lp!)~-tE zI`F1aLU#xb6b`Ph%`Mc^U}_F6A$Vm66a0pJuEjj#Ea5$NR9d~L{`bXPrJevdtXzUF z_K#LQaQYHH+I=0N^uBxQx@*-^dref#@DzaE$=G@ zVVryGo(#m9I#Y`oCcFx_>%zMxm5|DDy%%gA93X4{L*i?p`dxCtcT`bQW1t@u@&tnY z?E<4wX+}`m#<^pWL+NfLw_%W{>D*X)NAa*F<$7oOM|1^{{3Y`ZakVO5^o}M7Ar`KO zn|6M5qiX|OD6Y+tBn#6_W}Em-PezO_c1F9g2ckk^rlyPIy7hLR*be7tci>QS{IVvt ziacDs`Vx?V^mjEb#w&vwLElgzulZOm6pI=~FSTR-9$90`!)j5L!=D7+$pfBC3v64P zpEe~a25%QO2!U(!P$^_T&()W*7#=TXAR{b%MKa{IqvfY49(Ln~KQ~PoQ&(wzomdT2 zLmE)ECL}L;5aF@6n-Mzdb>e#qU|(=P1ZWlM`-k=9I`QC7RNQM4m>{3Ys6!M2iP~;^ zH9X)t^W-M%sFF8D>GluMnpo#uc()0KXSe2H|9}9fvNCUqkDujCVEdRo(c{iNm?XRV z;E?{N4jPQ8YOhfLUX*L}VR&c5GyKo@DDJgtL=bkA*ri7elX+WC6GMCzK%Tq)J-@f5 zvIL;r6GC`z_OrVVY6(ETagAAR-HpwTn)zIP{l@U<*gRp}`FF04j|tyUk?(IMvok%9*jG|ZScd3gyG&W`QGkgf4ZwaV^w?NqG?D&d=Cw4E zHjWqI_EcOvqP?Vmtlix&-t`{XmL2iMQGIHY1l2TEN+odV!aoIL@r`C!Lb!k_Lmtuv zj9If|oNS!&GLJS7K(EL#EmCTO&Ma?u%(t0QY{Ul}zk%mpU(gL0+>!h(?GA+L>-l8! zkF_OtMT>^Lx5u&eFna!6~h6GG~=alWKn>$ z(uGv&g)YIFkCQ1gl@#mShzHFeFTh=ZrHtA7EYO!o(jAYHF=nDLvQvaV0sH13dPdsisvJ--0G??}{-Ty*UeaY$W|)(9 zFRe;dfGLd;saMPdbm{6lOxVqU|8T^57mVh8yQgc-M%@ z0vo(B&Wtfx_>uT`1AG8$RoNC61b|*|h2p9X2Vc&#!`8Z&vC+Trv!e2m-6mu{itl=e|^r8NX14ph_ zG1>+a2l=}i)3J2N`Tmj;z-F@Him9hik&CAgAs$khDt`TXtT;KLcc8;RN%i&lC?C_c z#`Q#RRAay!pE;Ez9o>s-(k!)y&)IjxiLoYk0Kr2EAvco%DHH=XNnrFe3z#t6aqZXN zF#^+T!O%uVDCv41+JmSIW`1nHAXlai-0L=_UC4{qrNAC8Gc?#BGKhJW4BkzQ=(c`U z?;kn#6=zUw7Vz@1Yvr=DoVZ>Wl(#)4_u;udaKH(Hlnv(~@dKAT*FSskBQWzN7}*!H zWJjIdh}us7Rx8F3)yp=B8jw+cZus?YwRc#wRK8{y;{7-Hlomd>E6~h~rPL(H(f3Z_f>T{%31|I~U9Fn68UN@u}Uc;zxO< zpB)+R8GAmdA~}ZE6h88-e!y$BY;wm6c#Y&OY4C`ptZ&*;v|d);ianEE%fjv z6hg$yx*p_Xb1F>mNQ-)X@w4oS*opa7GN`ip541smYZiQf{f2n<70NJrV>qIf3XMEX zTcdeh(lrC3{RBdaBxI3dZvZxad?dZ(12wzXktPkM4;qRa(;O+cQl+|gbN#*rJ9~i& zddoHNlgkr=J7H|K(3VoB+ivLF&7i=xA4Ttk7NK240UgqF111!;u^OoVb9 zvC-NdinvBQA+51wyxzZb{{;87*sDLNqQN@ET!C&Px{Auo(_ye})o!ok^w@6ib@5Zm z=co=dS%#~xqCac=IBF6Xnz=SXKB#WjCNMsMTPoh2aW-Mcd2A9;E!A_^PP;Xi3{KdW z(8rTN#LkKLVsoRpY=BMZ*gYF(hv|u<4LQQayCv_FO#6q8EaLa=oZZ-#?i#)ggA(?& zvm$_*9SJ9eZ~n)T0LNi}IMxN~$)dq0T%G(Cy*dm*(GXqa9g`42TQRJw0l1Q+g_8 zw!}%%ak({bamcJBhrf&7zacT7T=Q3?SS#gE7W42HOp$c^KAPhdK2Ua$JWcvtUwU25 zRV6#L?`k-uT~I%{$r0lZe{cBC_pX+suJ_;<=ODAXnHv_f4nxIps|pjj04p%mnZA%o ztS-x{&04W?CvT4SC6(u;?_~(?*`QzeQ&@8jy}}d#cK-ZyKFWKq*@cmRsqG*;*Jq}@ z{eE%`yIdp)pE(SzF>2+lk8bGpU$1vzEb?IW&eCtVfc=avvaiBFY^^vHK=jLuDd|si z7+pLdoR*c%FV{C%OcwMiP8~~qC!I9)e3&0>EXy!IcRaY>)b<%HE}RiXAiMyazY{Xr z`!$}8If#q#q^&zh421s<_T4lwQd%vn5rWO%naQ2dmcKrBaMOvtle2Uu?=zhfU2D-r z5TgJ73Dw*oGOTTbBGKV@tk3TG82o9B-BmKMA-(gd__gYPGt&Jp2y<;_ztC>RVK`ux zTw||84Yri~t-xnd$vsLD!>?~o*2UeSOMj%rltKZYc#7{~m zCi}KB3Q2?OhHsokOy`=V=bJ}VF0fT~HT~$WznvnNbzOcNx7%`LX(F$o(ind7DCc*m z6&Y%_(EXuY-TPhrp~9oqrfxqSXOj?Y zxP?}qDAEozlo=p~l2ku@W8^!M+xz-$S$G!(2`4N1ssxL`wDsWvH zqIFiDjGV%)>;vq>=V6O7XBU-I@r}Kqxd-w~!Uyru9!?gLLRYH)2AeiNb*7HEkWcFn zQJgxHn$Rn)@|mW9T`nGSzCQoX<82<4cWk@7hLzfJKT9T>ve$gxd~k|4t&q(3Z!6kk zz#n;>w6j*2&|Mx)bYMZv?fw^Ie~{R{kzv;U>l2xy=QLxN)a*xFu{I2xu4mEik=?!9 zvaJ@l)86zBp1H^8(nGi7TpxihBI6eW&f>&7m!I7s448gvFRSso%MRKR*I61ZQ$11q z@i2+|_(SJ^i(~<)_)mCDmObYo6J|88lRPTmRHl1xr9?I}Gln&Z-@YY4CBaI?_v0vC z#A;TfbjU4@3RMJ0^{XgB*nNnf+CWv@b`pEsHXW?c%BKPlJgoR^@%ivQ`iUMI^{OUZ zeTg;m>xe}wZ;9XsTfbOmxT_yNjFx`tUj^Z6Q?L5qU)inF%sc??0JdsWfW0LVunF)= zPH^gDSr#Sq;qtxCPie4+TXJl33`vJZ2&i>LJty@}b1(Yx^n7~Bk^9t7tn~`U3-eS_ zX)`t}YG+QX?L4+)i|5xbhVDY@df_qnfHLVEY4$Q4)pj((No&v#XE91MHo%(HyX4|) z!$!3e;}Lw99&;e|6W~m?77W@k_T!!!6EG{|<{1#{d{!oRo!u0!q2x`;0fDKw%jy^0 zQnqfR-X$$2I^Gy^4JAHFldXClajdq_*-~|Czb5+CZmR~F8DAUnn&YzXFgu15cP2E! zAvL#Jzn&s@DPB8wdOF=9O7dHEgA;#r*i+<09<)U@I=~6P)ZQx?xD?(f$hp!6gDY(a zHP+agK;qj?xGoC6Yht~xJ>M^_JdJ72(I>}*=AJt*`D`iqZ5f~RdqhLIR|kdWZz~R+ zSyk<9bCE(V}18QACKIi@s83udMJ_cp@*?M2MbUkF&!*sie-uU(ByUb`Zhxk2>&;@rLkxszgeN<)c z8kQB7257ApKpen7dcc1KvjdnIK-)eC!gBR_%5nfELT8h2b+1G6YI{ymQ>5XRxEV%h zD>^{N@$m`HTxr1?H*Mab5?@z#m#lc~^>kr~LnI~YqG=#Iy67MvI=&$!Ca%Nil-w;M zx@W$!;{(TKT2F6xKEg_)(V|0;Wg}IB`|zWvAjrnAv`a{lqMZT^SciR3Ax z=cC8$iGg_aiuB(3cE4po1`Q{3*$)ux>4cQy&6WVk_W3oRH!steZr-kfM^`E`N^;z# z!DJqtS2yXKzwoflDd43W@(nm|*}v#e3|ZNrWw4^eo~i$oarj#*{k}Y`Jkgg510G9M z9rB^VT&EQxO~LRjQV)rBHk#8Yz-@dME!nHM*lCMU`k4%L)+o+S0cMQF6W9(L9qAHF zBjZ#-Wvo9M8nsl~+cQw3^+m&?IaXLMK3tax#fW}ApBLRj3}mMra(~KzU8<|zniHtF zG^Pt7!w!;3B-4nK0UBS4hNGy#_}i#QUV8AXzV@*Qrml?cL0E$Jp5^aob&w%LJ$=J6ae$+8X=Nc zDB)MxPJ-wtZqGEEBgKpKp&FjiwyfxD8EMsvfDfyck9rTbem!K19me34mNoWe%8Kxl z4RxWv;JpnEp<^FzPd?+L�F^B2}>i;PvZ5^?i|}eM$f30=TCyA#^JlE*(!hyz?;C z%5eR{q$`=jvc|Jz6j$Pn^OWAz96vPF=yZR1^F(KBO>r^-evuw)Q`8Gsdl#E^{Alfx zUTKoxyv=e>BXe1qq&H7fvDmvZY5bcHdKtEQW415W?Dq?fQ!U&Lvd%?!JY5KO#1!yt z5vlWFbJ*e8n|iY0VAOW|&a{2}I9iEXSZFA3-Qb5s!Y^-?ALi6n{o?;612hFvF%8dC zq2jQ-9#N{&39nc^W>&begj%LB-cO+M+6x%$-yU8(QZL#Afe3H zniz<3Ad5ui_^tE7dfi9s{8SNAI(3kVWP{mWqU1omNT1ZFoVjbxl_j_3xV_vy0@ru6 ze!XA#Mb|r0xiyEWp{?_aP5{f!3~- zm;@Iay`p|Gg!fr<`{z(k#A9SgY~R@ z@|^W1Eeu=?oQ_fh`)g9;>6xVcL8$iSn2+H1M%SRE*yGo|vwqlA7qU&a-I^YyWP`1# zqiFuIfT$hUbH4NQp7GOJ|L3DXe1nt{@7&a}^jLKGQ!~X`k^SI;tOL(2fg*!_&t&P>C$ApJ*$7d^T$e8YDN0GHQhe(p4&xw+oMgxzq<02L=~kS43X-m3`c-gh2ZTk;B2@>P6# zkiP)5NGN%XgmDf;N*?*5L?&#&jx{}Vlvs(kP&ZBj_pe6jvnw!BCnFV*`Dn;lBDVs- z?%v0X)jk%r01&zssG-!MY{#1-1p~`7sYEb~dbwU7P;=<1aIl5cV{uI+AR1-(L8KiA zAcT>u(~|&k8w$!WllZbFClRy>KVC-iHSPnT&1#bID(iZKq`rnkTI`>aYF*C=v9Ks? zglwfxDJ|;Vx((Ar(h@6`LJJ&1s`m#=D%|XWYvU70oO`o#ox`%ripVx0^SEgqo;3v zj+3es+;FGBYNcyz+~X5Tc%|*$i*z2n0#ICnj*Y4K@7Htk25snHzGqU zy?2%a;~1~nj5hmiS}Ty$W#PsJ3|0bQk=9c!EtEH*=;`!^RuF?r;Gh^<^jtgYZ!qQ+ z1|=~D?U(a)$Z7OSl=b6@C9ScP-%@*O^a63HK!p7?KxMNii{kMpj+m|#fHery-!SW* zEio0D&_nK0Olq7ar0qcV#iF?X1UsCi4v+!#D6r(idvcaE20=yuy%XJnMu)BTmepyX zyXfyZFn(+9gSWgCjE)fPw}$BUS6Fo-33T1Q(#=6T6_v`~d9oALNI@ObV&BuoigAo5 zp4t6H5`ewkkhs$&IJij@Rf>M7TxC8{!S%2!H|ua4*i2+zH*4}q;g1nhfZdJm5f|1^ z3ypGHIZ+`KH->1g7x)0Iu766}+Fh`$K$_6G1;E=#uz|wk8D)nX36*H}-8&x2g6e5Ljl86!r1B7aVQnVlVEvlua0Y1^qaak~3WTBxI}kDLs9vA6D|<*o0a zJo~R$!99QNZ)icIr2Eivy~RGjDiV(s+y-bh?Hu1!cTe;$6|6r6o;S&6Di(#iN?2JM z-7ulE1uVbOQNF`nDa)U`kC3ai?z-o}>HAJY&uCHLqk_+Z2}rz>CZSgfP@Vd8_eO;e zARyZJ#AhR3(DZr(Kf>|=X=F$9FBa;Yy5qcAmqnZ0e`L#UU^X}W2w-7<|B6tkH;r0%u$<~FHe7FRPDH$wa z%%owUPgU!bCE~Z4Ij^zOO7)oRT&GOT>iM%$NAFPlb~*OGE=xM$4hi zMMWq3@|8PO!~ofm*$$dv{_rHeqH0N!olq?4AaTss6Ta5kBc&^T0y~y{$Hpc3Up`*t zpFY0cb^D2yKBu~7ydel}bx9g}(G7 zW!>PFSjt_)E2OuSGd{cV`kJ@_O1gNf-xg&LvKLOPdel^-)I`RpgUsnwABq@K3{ICi z*7O{#u3OsWiQDR`d|{d2SXPP^ zoVA2oy;=&mzdm28d+nEhXxt36vmkcXZ<%1dC;?LRn$FGLEB30)-4x8NI#f6k{=y}e zVh}P#F9IBx2uJHxi-bX4+JlEP-3S-F5IFot%l)r`$Gs|%K5SJc-jUnB!RtH0I%gj~ z8GW8ES1!Y3nyY+1MenEFoTvSp`kUUTx&=gVlx(}5^LH}QDbxIEjk=RN3Q*ouq89Vc z8wDOq7BS0`@_Lqu&C+Imp;F52E`{Dld9}X;z|5RDTF2wc4{A^*|RWR7<;C_d-{ezF~PK+nr zLiB6u(a@xLyRF6-Hg%qj5@w6+LvGIc9v~~xcWet4wk9C|L0xte*OfV`bN8yUuwJ1l zeTni3|DVW;=g_HgF6)BAvfHwJA&&C5HcLkXulE2A0(iJsLFNL2D;y!mF;iqW+wxf~O9dVA@2<4i7I=oZp$v<0~sfM%5 zhiZcq!x0ONFQ84FC-({}NQCMKeIhu5Mm%aXr24!*#Xt5@8%X54YQS2^>a(gBL3@hU zYN>w}!tWD*3P*l{j4X)XL`Ct9Pw;%Mv{jHbqpvHRLBPFezk)AR^R0{)O%^nU);@-t z-OGvSV<7t?2hcV&a0#W8VlEla!yv?=b!!dO6XdCW)#JK_`SM;B+*4uP!A|i0`u3_u zxll3g%Ay}U!!%Ldbr^kSfeHR%$Eka5{On%-^V4L23zq(JTdVY$yUNWE0%-JgZ11$< zLxTtX@eJqBfY9~x>jTJ^Dc4Ec|B^@-7)4TPYTbwwiN>!ZULNj;V05!zQq;^TU~413~E z`JQwgN0-pPUC@HW{i(aScJ7cv$gfHdaHm>1zBrN)zhagxXh;4dxOS3bBGB6x9w7V~ zmz6r){aw4%G4FhGI?rg3O3TcH@*CjHJuiBF()yJo^gL`Wmqc0Z1m%DPj5n-9yAHKr3(tid>_4H;W;vMd(o9l8f<+7WAgZ< z%7=eBOlbqy!5jPeLu_pyL3=l6xp+$3ObnrvQHTOifB3lNGFr8raQIY#s z7xDSuna0&5kAQ?%S1C@prpy=TKxi%ECuPQcnYAZO@iyQr_L-!q1j@C2Ac8)_0;nxc zJ+8aV;MaE2*~f@EH+a^+Mu)*ctm}#zz@J$%Aj(tD+tLmpsm8D^U2@ZmAI^PS)LWXB zb@G!vG12HMbr6DS*zciC>`R#}DkLmhCfeFX482bUoTvSpb~nC2ailM0-Hhf)SIGjb zm2inMW0Jb=30L#$Q{Kvy*9NvU_EEaKynbDXLs^>wVMiyTL|>1gB!N;CikwtK&uqNN zvJ8+B5RQQU>F7Is_BQf_`(ASnHw1GB}mwQowK;HW1fBTB4gv_8fC8!FIp)ARO3*mq*QT6Nqj z{Oy=XXLih1EIR!e}4qBrJ)%71cg7<@NLKQ?}ZZA1A|CT&f z4`ZGFu^92FL|9jdB$dDXs^{~+WN0c*l^>fmvHEKNi66IgORw*8Ch>#ow9@Vl;y7!W z41Q{@hy)JQp6%niQu)*2F3(wlFgjzOW3PXOck*h=#uFbTCsbD2RN%XdgSA-Jr2?2R zHg2#T*>OE$oSjt~Fza6_3GAOKiChLNaP}mK_Zl>tbHFa|uwC|pBE#=z0hLjE#eQ6R zMQhvEvqSENhOlOVgGm+)Jok(1@FY8vK!GHn9)t!c-gL z%&x6XwqXp{hP=9;#Op1vIo)RH?eOPTO)y5kU}li2M|Zq9&J|QVy{oy$!!KX{9{v-& zy54JPHBpSzoHuzC(WD+r)_=q8W1ay!^WqH6k`?HrEDwTjkvs`x8eK*~rv^_Whg3NF8 z6~;uLnNFdyUKi0Zwz!d=HoH=VR@h}gCb)EpI5<@^fpr6vjZ+K0Ey(90eXKlqLb4rvJ0W}|e< zrgP$uJ|YX;lg}NIPKa~xCpi0NF2DB48duC@fV)pP6RBL4rQUwNqRIZPV^GG(FUS%; zA`Fd-iuLk`vc&tRrBzmud8dw1)p64%v4HwtZOU{OwG3OGCzY|BiP@v#fx=l}R9+~y z3Q=h>&wUP9+EqA<>=?@Oa1a6w=bgTCVrmiJ61XmJU*@B;)p_7m;UCNb8ZvOto?~!! z895eT*8&O+RD7iNqJGn*xAUg5ZdxgwVt9gKQ2`P8s$z~T_WIL?(7R&5n^G}*D)Kqb zX=r98Z(!RJ4AIaoTp7y|{b}9DTO#N|RoC)&ne$bVwxa4=OD<=f2YL9K=b^x#d#SDO zpJ(x4&kDZDX#5R<6;VDo#A1W~#3CfCm-tvZiJ89WN^md8ncg(A0*x4oyW22avEG(^ z{q;~en%@BT88#e#q#B=Cd0Iq2>Rk2qV(e7Km5wBM=`P9IXMMkZZ`?zj>4~og z;(X;JNi<2g6z{xJsylR(?_F=HacNH7#20-pz_Xzkj%k<8eJ2J6soTS1}RArZ8D zJlNce1J)|n5S^bsu{`zIH=7e6$3;y)b0C{wtDJt{BTj^drUgx z*<8}&_AK8jy+E(*h=8U|IEkS=Weva{!UdR2^g z(DYE4u`>m{`2aF05904Q)UyI~Hcx(D-Nha#>F@dBaW2E*4R zn+j@+`96fN??`l8RYp3))>uL9H~-H4y<*I*=R|eBdb*r?vBwXT{6H_o81YDJIu6lHUAQp@|1i7p{0+3-{ zIIZ=osXV_rS2)eR0Nd6 zuW&@X^-;%~_eJrMJR$BVn+omFqW+!K)yC(bocn?tv#1G>o0E4FmjAV@SkTR5s0#5l z`V=NP-zEZV+XOW4hb(h@Kfl{4^Y_6^>nJI#YmB;j0a6HcbrK%U-l`QQPc+hwT)89| zN$_?|80QdJb|Db|SfG9V(aib7bzpNinHfqKq|+kJi%oEcw-EIL@E*Y<#4K$3bojZx zM*#MT{JM)TEd!>R3@hDMy`Q{(@85!%Z0Ue}gSD$CGvs{8jd=QG7iDbe4!;7@rT}YsT?_IRBmbd6F;r*zIHVC#s zMPU0NIwP=i%IDouTyiKhJdO-<7F%PSR`i=ajdOwX-BM*J}lfHw0+J^Src}bu?`LvzZZXS)l8Eq=|{AL}k^#MSk}?0-7rCRf#<& z7@ml)FX+K)9Us%ul94L2NTQw&x37LCi_TaS4|)T_r?a5vWk85WFVX;QMS$~qFu@L% zQu0-T5FaIqmLSc^!F|^`h`neVgafUZ)oT(-Dhob-0YnC}$x$42Om}!_tN$&P0yY z3`b06ceIVS->Xks!bI(3PwPgnnq6SuCgHg^^uU}eYujt_At637~7V6ipqnAk(NlVEq z-{mkRb3iqDorxn9ea0(>rQQ3_#n25^a>P?_Cl;S*ALzdFCBydiXbDnsfng@WDo7ws zjuLE7`rUc3&yoD_J?oH0fxnqin=Tq5*}7AB3{F@w9DoKUqe323tF9R&1`5`%iLPi` ze7%Q!g-a44+=GWnlVY3gs@6owsfgc0#?%QEIV9VvjT(j7DgAA2os8pzdQz&MxQyneqnAN zcE7xM^ib&i`?IX0BMV)wl35DhBnAq2nlGp|TMp4XN94c`0R_W z;q!t2OlF4NFabY-+1$zp*LwMsbBSDyqcNxfz4y1L$%Ozf5sO6&jAjM~Go!e{8?^W1 zqw@t-(?s$-z9(=46g8_$dCR+U_A;ot;E3#YD+Yp2x9Ui$JM*-GWC}zER5BNT> zbB1GJdFmFw%462YydFk$R`K?lC-SXF&ufK2Q)@D(ia)_P=`<~l<(`b~L(;>4a{;`p zbwlPg!x0(VhFHtHry-F3zB%#Z?Eh07r_3_~v1NcYF%tZ5+7)XeXohD zMvGCp=dq62c|5ca_>y(i26~f_#ru|n$|;D@No|`uo#d-G^=10LnFe{V@5(*Ws>0WC z$9?gEUN(HWiUbp)B#-pl%b;3kod z-&9zGhtID(!UUt9w@v%@zX6B^Uh^0Z5Wcu{d!4Hdo4?DgE4BRu8t>A)p3#?xxrX8} z6#F6fHjAw1tb5??c>=9jZK_yQ>_&=tuKBv5cwO=RgknnKvbq5>W(qT5YV5KN>$q04 zVt#yeul{Cg-si*%8Pv&D5w0LYHV!wO?@v4k;^$uCH`8(W`LD>5{lhZ|L=fx>vPAw% ze)9j3Ute@L^P}0d_t^V!}XMiYkMNE~@-&T>g|#zOYVAD5Yw z6WkuvYUHR;6J;K)XzFC^`1&GUl09=^J1nsKteqy9<5pL5Nmb!Rf6`b2<)SN}`_*n2 zzkN=GMPW8lvhnT=V8S22-hUf|s60Bo-s+P8o6Hr~qaE7M;hN4L8>A?!4?2v^b+#5n zd>j-RSjw$8uyt>l`$au7lBVnbf<0<&1d{&djcinWreihne(at1$>g3BN-YIs^&hAO zKG{mUQDT2w{%JaU4?**HoqR;40)tNo_W$p!c36rLxDS;j*e#_Ai;uv6z2-rM`J{?? zU%TGQ^THK)30hE~{|S8<4Yrsb-FSXS2-BN=^v;rC$LiW38Kd}OZW{Og{nZhlJ~_3% zdvveg-@DnH=n9@uOqpLL#v8rdZ>+z%K52IN_jWg|z=tT0cT+4i)WHI<52R}+orF?A z^I#Q4bStCeAt$et*kWH%Pab*7Ojw1h~!_)675XsVG5Ssd>_6{1B$!nvk; z_ixPK1vh6sQGjF=5zQ1#shlQAZDd!@x{rG)D&<2?ZxD~WmVO^h|J8}Qpl3#zGItuD zw^PEda1ezl=_+xws7=GaW7DpL8&^95X)GI0(tFa`YPk`WIrqUmV%6i0&i}>OZWKI{ zryFj$B1K8FSD)SmP1Svv+_b!repQx^pN#A#jcn=_0z`b0qq`#Am9T&J(8sp`XZEI_zQ;#F)a;abRx1U`k?!IF}e>xxFD-(p(g8i)|! zpAR}L$Nl&jZEH@_psz&B2fnh^j37hddtBrx1zZ0!S*R{ zZxvud%ws8I7XII?i|e>EC`%GH|b>V{p7Y86;dZmt4FB|Y47>Ue80`>3;>3>YX%%xDo5c;uy_;u3i$ew- zfH^ux{yj%P$weTwX3@v#{JNR^+OVqlw_Y8^w0W6fq{-Q*{OUXr5K{2Un?br(=qn0m zT5xEws6IPPUaQxq!eXsSfqnjy+t?2ht}Nb>L_fP+<*TEEsSSXQw3iH67!({ZrYp^J zN(HE$L$8-Eg!ub*qws;deB?L1cwJRg1QxnAqasxA>x1^ed%Gm#Uoh#)g==3Riu_Q) zf%dV{vx$##o%JKkm(h~fMtr~=mCXRO;KSwP?4GKzpQP9`+m)eBSzYh;&zH!dxGa#$ z&Z9>bBQtM!*E;jOOU!3m(C#GI5bNG?prj_|eY?BGzeC1_6)?`fd*KYJ;JmG5H2Z?G zoLuADhj)gPg@V#~8<(#-0178*$_5J;(m(~4@&!O#!03lVh;<<2lVD6^;DjDt&P*yL zrI3#`>5x0mGvg)yebH((3BEtAzt5}GiczpRMxhet!`xkp3iF))y~*4xORG z#%FOQaUFAga?2jwM5}6I0yHAu7Dr5h6nXzJbXQW%_P&}=?k&uA1;IklM+@_=8|%n#vh$qCzw6(-X-MZsS%=r|7tYT+MCZ4fRX4N zy&}%>pBjC038WjWP0b`Q&K*510Mov19cPvvJ)M32fo(vAWwt=}7eb{&1yp66+lZL| z{74Vy_hs`5}_n4;2 z36DK=9d%5PrvR2C{|HGF`TE;wTUVn@A8O6vgbV}pIlZ!N=Bpus4AW3?_OTVY|4j`; zk2@w@aQ7z=|MRj(TyM<=|N3LKZ><(ZoKoJAATVDLbHClT<-C%?^mdtd%*9^Pt-6^R zj23rY{~yXzf`N~nfl*^NG7I7Jhk-L}gXA$n6-E~pRI1}C!Kl&zg{Qsz_CdI||F$=9 z;RaF{3PeeUKa{`5dz}4*azVR z)Aho_-@EmKUD%N`4CVbwOTex1jF#!|Yoa~%O$geq?~{%-Xhf(S9x8@v5|9d=RoN>~ zxbyPXPu-BTR8KBsvsHOr_b;B30=4)#MZI!-A+HZ>V(F5P;O|1t1hK0V_W}1x`EO8T zEwnY@GvoI+Zu@(qcGtJ>QlNX4 zn7#NC2sz_X|J35wTvUs#UmQ%XVI@O2>p8rGQSej8hvo>Y8DndV75mI9Fl%*T=tv_! zAATh#EE%~@7gVR#Fsu5fcy`<))bsNs=ktA}gD+^U<2uYc;nt;k<7h9O4c=~BXq!Be z7A?9k3tIoq0#o`f@S8bA#iB#sAjm6rWD8HB2AHe5gF~!-ByqrUziAF-Qz&{>_T&z1 zmhJ%55lV=^mHnU}@s{nI?h#1smMT{6RZiUJ|KIdz4OdRWLjN@3oDFHWe{qm{bu``b zP`NcX>%PhmoZmoFF;Mv;W*@71M{CN#bi*noB1Nf^c927Dvn{3HnZh(6Ww&`dv%%wc z9_9Eol-cse*4c-ah>-9OM2SHV|=e-TdeFk#0xA=%zy((*zzj&Tk`pY`W&);vj$Pb$O-4~CHiq-(s7G8qU<2P>{&;wD zTA@97D|9foYejb*m?Scg;!V-tPvHZ+<4A3b90ZN-bhPLuCqQcIzp!`sFXgHOs4j|3 zY6{72f?p0-R_FmchSsP#$s`s@O4r6YVE7+#X0cZfIDc1lxgsy*K)EIn5&4Dohsfq% zZva%c*1npDjzeEORlNEyV-uqNfp{Xa>K~tz{UV?WZbr3^esWNOOn&N+?aM3qFaxVz z390arj=m_216`&J&m9x~M;8-tEE& z&WVfKwJ0DMxrmAV$E(APG6;R@Jw$6SKH|TiNAi<@5=DNV?q@|%ZNE#>PP;VImFjnX z)3W*TUn>KGBzKT@3nlP^I6QP~&XtRaD0_~e*kF80= z=CP;M+PQXq+pMUT>0M4dU-zNyX z?0+8k{u5Z`Q3f41J<2bEwEI3i^!=JXWv`#X!~ACb?Kd}2y}hUxn3z8?`vtg8N-!3I z3h|&rc(Pqff#77^zAuma%-r^!{u(Gpyx$z5Oslg)!+$4?$cDLoPHexRy`!Ys{H7#Z ziJ+aj7P__B7QVj2{Lkm2WtKIae#C6x2GcH7A6Z#N(~7Ae*YI9+ zz~Fk4RfI5x=t5@lDUOv9xdOw&^$2?@KL9c0TIbxLx1pph>TCE^PlxG*4Ht- z$JC9ux>cXA%C4T-+J$ujJAM1iiUBE5(5T(%@gBb{H|&(fJTX3z(MzlW|7iiM40;_L z$NjZ275>?n=AzWUDXW<&f1lu$rkQpQPTZ$Xf%X2@4qy*u5-A&o58I|El0%E_oywXYHa>-RFeHc{PPH&EtWcD3gZ>s#G{ zWe6m&nA>}Q?-PFh&k)%M$h?|(=y2kpg_>09sexFl!p+$=#aIW^0GVVcU7XO`gF1Xo zuzvzpnPV%<_{uV#rh@usih;5_CTjC}d5@bcNco%YckOd^)qFqbm~WG6b9%H(v|H5& zOzo(l>eA5UBIMXWFPTd-X15>eO6*M5qTAft|97Yw%&C7~%1-MOyGrBd+&{t-lsaMY zd^H_>On3qh2(OYtRF`EZkmGyDj%n_uI=Z{}gN55}CF= zzrVCoPzQjxYHpvLhN{#IkZhhoSL6l%52Z^y2MX+>fW^QYUzz+`LHSR}lwu;6rvF5G zGQxuNF;DhQRXq|$60)B!!Q0M`1ZSpo7h(z^O&sY0>LD@iT_aw49(Sa|e@(p$TEAfISu?N)Td%DGgAK^zre+LkJ|~CXDf>C?jHpb}J#bEI#x4cgRYq{9fv>w7!gD0dsTh zRAj|GGu8UpwU%=Tq{(AU`SMQ{!3mxgzL#tx_n^i@!#=9cX;qwL8>m-<@|0&!pZsG? zTiAE)pjz{(dV8tRxbsbI5=9LhPVw!**N6&wTA`U#(x$`V@PxxBtX+?}rZvK%xkT$~ zQk|1nL_^>b`pFXcl^sr|Ck0iMWVl-IZF0X$qhfEHLGuCSEyD>i#D)i;NyajPm{3&f z7&90re(wiSiy*p@(SJzt&t`cjQa49s=wjc;nZcki;E0;2jty7BCPL?G-%t!s3^Ao9 zJ3D!9=?;_kU!rAGTdia>P-_ch3-^4Pe_*?Q*4TQu1a9IQ>@xBbe^kM{qWCrpG|$ZV zMLiL}-XKx6iLB5b8+g^`IcYp+RhlDSJETNs-*jhw&HAC;Qbmvv{BM;MJl@d{#;PmW z-IQPqAD2lF7gJN9)QC7041eEu>EL%X$LHS8CKsu0l+B6W>&JtS3~vE;ug@Ise1{-Sto~=UetHZEVxZ_jzfE46)j;DVL!b7mfv z2et;IiR4cc=R#_eyb~%=xgtv&)kwa=iAd4PcE9SdAHrTl(oC}M9_kH(EPYXb;5L zS0GxdB?H{~Gb9U>jXNsrZK9g)MW(%)FK}*hkF(vIt4#|Xa9SjebrDL#n6kyVlzaLW zB&@197EzNnj-2-TIaKIPDy+x7HF+B~6#SmNLn_U58j=6$fA@LE?2&Vn>%yXm%Ow#I ze!0(>3z@l44CFxGs=!Tv00|KbR=Lz*zv7#ghb>&Q2V99u)5i{vNmUcNpIic^|HS}J zjReUe%~d_CRp(#vCr>OY*ZtsrLOKoVDquRfLJyr9=>^<f&;#C_fb}_R)94`Ldb;lMJcH;L(TQocnTn7UuaWAj<2R}R0MLty z8O8$iv_ILV@Rki~u3bB<>a=+Zr_;h!7=XFn6}db8^i8jgk7&5(4(q!8!(8rosM~1W z$VvY!<|N!nNyr{J>m9n~d;eoq^S&ng>g{}mwj=lfNQDf`kDl4!Naz|?@lj|J41BSRjkJYb?$(_{*?~XrZVc9U_D8{`0tLUPuKks=|&!~Nx zKqhU>#&g}J&fm{-xH#I*a+d16+_`e6F*4y z3CKp>N^MN4O4Ra~`@!rq+wp+5*0b$=h8PTzXxaQK^M(8HwElZs?;EEj`{1}9Yw{h} zCoso&NtF*dO1)GZ)B%EB%l5%3gGmQe8H8loocO06d3CqgwNj?J;8QzKQ2Uh2F_h%~ z(pmC}-nM(4ja)w&-|Ak$&~bd3qnP>VsBCYB+)nvyW@9)pGC%P|#)hf*4Pv|UxPl~1 zf_mOvt~VVfcRali(B-jP`LXxe*XP}`Ur7AULSl$I%hE(y6#dn8IOjwV6#pdB{O|8% z2V`fI!8sr7Z!8?*mpiUx$EC)9pe3*C0pSOIWLv*^uSykwI^1j2?e%8^+=GW%9j>K8 zyz217EQGPTA~#c|%dbL)VL-CtvF~-myFqZnIbDl3p{f+QpT?uoQ7NCx5jA%S!q+(K zlSunAB#25HwZYmR`6^ZDMIn5$1_En^x_5lVrD-4^5+2 z0avJ#@7LU6Z;!M1Gu}-~>mtlbLrChA#9QGFqe=y9sjW!D6{84c8Vx2$a@XX?l=voH zF1hj=8uS`~Yxv#2bzVIACr2Ryi^iK|N?v;ws6md{d{DdzQLv4a3)5G9?z!8nter9# zbBXliOsJLfImYMb{K9+5m)iz@z!i9Cc|QSywZ$oCP0#y|P6sHM0iT>Co4?M)xg?e0 zxk+Xfa~^v(0**a-*nwm16&+qrv66h_x$|ig`GzpjR_oUjO`E9ZS_SXq=b`kE1+w|3 zafiV|r{{Oc7>Oy+vod;W6#S@O)}USMNMz6*p>l2Pgd~9{`ad`!F%hKQ2NHCEv&J*& zzu=`Erc+>#vu_xL4w+dn2Pes7N$(YEvi#CszcX3w+@28sCkkpE z;$k+j%~zLJ*$OIPZ^j_S47TEg~yPnlf4t2B1B4 zHHYvRRacWc)d1#@0Y6#hm&Dy_%i*G$nES`jlwc|sli1OJAOxttclMMSMYRbZscR{F z@qq2u{ZLrXw$lFjt&My z`OW%XrDcg><)9wWu8c=q|Z)L5yj8t9*Xg;^9giEa0_q3 zA^3eo-VGJz@c`!{Dg>w{>`Dr@`MhEJ_8?(yX2<5jCaUh-Jot4-7?lUb`BvSKQd-28 zpoCmk^NGU2GelEPdv*i{8MEjHZe>`h;g1O-ST>*0uGS*)#1R52}jQD82G96UPF zOEOq}XZE+eNk`i9q#u}r2thjXlD9?MfwAUnl>l3FoerrHB!)b?fpq*sN9;Y3kOb!v z61s1cX{>FHzdyf)I*gtoBMJ4#&FXp_jb~*phTF{ww_1c7EXx~5HTFdyPcSdY3n~i^ zN@|11Hm;7ezCGGFH8lBtN>g?;n@jpf-{F?aJj#P-w4)7sUO`e%Q?e1H;wwI66sV`i zgpKU)eLT}NjoCOooesUoS4ZdNO`TJ~ub^7*f@xdumy%#sGVLcvg=SWm6F7PM2VwzX ze5ZHeIwFLZyTPBQg&hHeC0v$&hnH2Mh@7`e6=oAXtp4}>#Dgfig zLxUeYyqpg4&yO0CmxTg8<&DBt#z_@_e$_vt3JZlX*P!;w2+C5$z& zta||zO-QhMW-iUAfN>24kn_5;t7xOJ4pOXhn)1#Vng1A%q3mTOqvi-0%R&$M3L($r zo#K(X&WH$UFj|zd8>lmO+CRNPhO_rk@WWXfcxbCDSWxMAo|JhTpOyEhWw8B>IZWQ5 zy>c8R_`0H~WmD_-ojjB(uVGr%w&pl|JV-B2uQ4|Byy38gWb6ss=-~1PYmv%`w26-E z(JxQ?bA?Hex7RK|h4_oAKYRjyCQ7yF>kIR0SNcfjUfTIHV$&r6{JXK`1%;Q?*WXHB zkOF2&I`qdIOh3pu%gFgQ9CX$Mfm%W7oByhUis~-`+ezGD8o80L(-1Cf+?X^I&|RrM zF`8@io6>&HYx`K(+i^>398CUkRys0O;jDZdMK1Df2BbKhbe9D2@pML7+7QaaG4lf) zO-GFa`)j_gKD@<*&O9M$qG@$4RW!M8mh$mC7iZT{2zbs)d;Y$uw*mQIRc#(^8yio? z6#e<#Xeb&|LqM?H+E!Vt0%yvixiqq0_(&C2U;cZ$RO4G+{CnaD)FEW@D@dgzv^i&< z=J+og{MezM)4)l^q|hwA+Vp8!V^^b;=5KAb3AoUXMB@!MZaDA8rmcz1hw1b3J8t6z z>qC7qAv?D$5QA2jZPv~Mm^AH|R!95+Rqf_UWqE~eiW2?hwijd=S!L(<9VJc>oi~Kx zAgU>ltFL<8V_CWd{Kt~}Gh#y%caFT;chGjNL;GIvN8kQPPqnu>?0-{xaW(7yrLLxH zV_mO2^#kk^mN-_qD_UzO;p8eq-)7H|C(7cdCo z>BuZOE5FAQzky-zjKFg8Mbn`<$#MmV?9fjl2w@F9$ij1BE&1tPRc9>2R5pZAF7r8qR^_8K-ras*mxE(a-XBAHj&vCqvB8wIB0V z1ec=tthO67@7eUc7!2=n%K3bpiw&Ci9q4WQvL*C>#;n-jOr%ZRSqWP~uL14Df{)(W z5u70r#8BH>j7Z=bXM4rZ#$?*?PtsuJc-bf5W4kB&^^f^=%hjv5m-s*Ene<>5cPYA*aI1Ez&ZijfL?xqmJcy5%Y{=hSRtR$T2y4_;1t%_r3Qj_~mdsZskPcgb zu3VuZ&dugwsQR8Ba(lWPaRx-e+HOC=dLL#NwpoB4ERM9xn$^k*At4Ui-|}F+p1`e0xW$wN)o@*Gt8> z5;ptcx_Pg8=oPV<&695jdXqsB7y5M*!nu9t-w5K$*A^mY52ConiO6lOQ57B9%DZcV zr~6(~ey7sA?B3ey5Y{di8mdm%+nXi zB{gbdt;X!kcH)7QN8Lty*0VxH*}cOJr*A2B=K>bp49w?ic%Z0y5ND;(0mZKM? zVqf$%z;{Nzj+ptfAuJ^fK$>l@i5)pi961(YC^x*|QXNY3{;Jl}!72e|Vfsfc5y7;# zJtdMbxO$qS&@rnh1-w>s;YQ`e!y#>tPY%ACc&(wQh7lCERV(D_lWt$Z5$Ug=_tA^< z@um+Wwa0*>$D8cE#oOT;KMntl&=sqP-bZkP*Q5N86bucZ<-rJAE9maHr~s%vxYUZI ziS>=tLakIjE?BcesFWMe@>71@VKld@dtHEa{1bohee|{^5^3*X3&WVxI9qbH`$WA* ziHnS*>Oy$WZq@GH2FTKCd|SiYI>Mt2S;B4@`*Ynhux@T&y@g z-N2xtapVQ{4=mx8-rP;G@L|zEjpIUpnuY^TMhUHKr*mE%<&UJ<58WN}T9YeZ0ivM@2hUXYH2-O{CC;gJg79oy@6DXVeZ7Eul3w6fKU zsJjO#KOatxM*Qe~dje<2lSHkYVnQDApUiHR%>$(rrnrZ`ISJiAv_$*mLX zYdvj_^rM+%39yDEv#sS|T~2drwa)=IX!n@!;V+Y=n*h6z}Y8Co?HV)ANb&Xq3xJ zaDZk-J(6$0mQhby|2betU7uvp-E?f0f$N&5U}HGa$)%+4dvk)Zsyei{FJ*Jdr^+J!;Mi>H9;hjj*qAm?10rnqL^4v2?v$Osg09l` z9VVpAVjP+u(+Y0E+jkAmtO=N&oP%Dgi+fa*6W^-5HIb^^hH)JCW*Y8+UYG=*CFd$Y zpHFx&@5HkeOuTqAl9phxS4c_l=UfXWLyEm^q6ZCLN1TtwnqT^IxbhlG{riP`BAxP3iK!#(QZzmPtvzOEpNx5ERtnakU-#8X<-0T z0QKZJ{4C#Qgma3OltW@p9N=Q%QMtrYGj-Mx_Jqdp=OYV3=&(og>)(!U67_hWf$8@Y z*j0$n{ON&Bw2{TS|?ckQ;y zH^SFmBdRDZ#>7c&saulnHP z#vbyl4tLrK{qtRtc9)!V-1YWCiNZ})My`qrn;0T5G2CxO>H2R)lvyzMyd5F$*!{q+ zVImUX*)H`9V{WQiUy$vI@(@D4w0_l~p2m;)lDE`hOl2-IUDejxY}MA{I1WIB@7cR( zBFduS#=POltK-Ad%GW9P+X#K*xd{d`5mqgqPOnuPx2zo+e^{%pf(PKkW>MI}=@(CE zg+;)~bJr7tnm#z1?|h8x83&8 zp3jc_CDc+@uGO{0>m27Tgqwb002+VcQt^Tbu9cKn9-T@X7Q@p;DH~EnYQc@2Uox=9 z^z#YdS{eGw>my^*kxV_4WWd5ssIwk& zUKTR9_#c&j<)}7o5Jg?C{|1B+KQB8yvQ+q_Q&H;(!TT&eKQJbWU8st}8;{MGx$-L* zF4kAM;)_;-UPvxPYUo3B5bx9^d64{Mp6}+bhE=4DN9rZYdB*ScQ189}5yV2d(OR)T zur#yLel?5}5h1~Wd8=BPH%=Z~RJ}&{Fn+OtBO^AetG&a>QOaM+`Gi#dfv)%ZOcZ>g z`3l1hEa?$qTKz$|A6>e$*Z)m@@9y`*KG3fI zbTf2)Ml2sw7{C40h#YS$8DMWieA}M;va%u~$W?-NT7b1(W1?FVHukt9d&KMpMA4Q2 z#AceD2TeZ+-E1NlU`X!E1PlF^dYw}1e{DF;2>wA;+;{-<;AC*}0FFHO^KAET|M=zH zDKkKz_Z*8YX^+u4(>+Ip5kcu>05mds_XW#bq5KvPaXKTwaZGNLPQkqnP z^gp*uQ<6iWoMHKj6V3iVe0!zj*oUv^hvWKbmlDYE_!b_OkJ&xdv*uY(QXXh`E}fkk zQ17{%W-Y>7QZ1#8&Xb00tM+{!E#A*Os!;aiL2O^ZQlPlcqT?oHL1iN6FMre*g6c0c zktgFDN5hA%SJAnOYNiZCx&l9Ahg-}eJ59s50bXt*Q)g3RN!%73pPR!rsG@V>QEkzp zw<|0_nqS-peoX$Y^;~Hq(m+|Po$rvv^6e`g%ejiX|@e!Wt?Ma)v4upV7Qy>N3q>wRAJZf`r zc`&skFHEXz4Z7}E(A1mcf~wY|e1kID5Hz=*B{fh*Zr25XT_CJp&-#ON@p>;mhouJOcP{U=+Y*C4 z5yeymtaGv5LW8@!WMG7nOtaTR1nXWuB+GW zL#>w-{$sMMGvnSyR;-0JU#(*)mx*q&;H&$7nrzZ-X74X!wSZz(SA4T;pVg-+)|Z10 z90$uIO5bc3Q@KVGj=lvNMV~Su=3bO}DFwElJuGz#FWC!QYOTw?%2h_pIyNWD3x3JP zWz#DV^3 z0L4AQT>tW)e0hhIx(d_yTI_L0wM*WHoY0p9sGkWrW+SIz+|9ixF|W6)Q7?91iw)&5 z17O`l0~kW?O9oBqCU^8CxlI#C2Da`;pUnqAhVbBihJ&N#Tal%@=cwbIo*#6v+IGP{Bej9{y( zA$I3Qa2VeBMWxomE`9skWNL3fDt*_HLs>FGP=6@Nf4)w;U)=0FUvbv7&58XQds@Er z2zHMg2PE&Rf2jcTdW4=$ z0O9i!$&Dj20cnD zqw5|IqBUc0CeYPB?x(SE5cyrj?(2L>=r+&Pc+9m$=D1vUYWwa&D-Z5%e}>cF3_K-S9oyq(@66!w6@Xj5pVUP5l%iJYwy)%VvXs z!HM2B+u)iKTkB=8iU=fwS!F2syWZuMnyLrIdarE70t!{r`hn(ss-nJc?`1;}t_fcU z0s9rga3W7@kHY_O_8OIsnhi95cHQR2{3tkLe!K=0D#JVwzQZT;TnA;)b35?q?c`k1 zh&f9oS}}RgyBLeM_9H~H{t^i2wM9Y`*>IRUe<{D~{PscQY&go)tqEo&>e$39P$7c}E#;c($4Z`>xVc#t{Ra zwWIsiJQ&qjGt}_7lNntdAGP_JxJK*Zg*)7*S(g9^hnccbkSNH%#*dpd9lCnp0GEOd z#rPra1AREzsdLFzeC0#YP>O}tUk}b;cBQzO<;5DLF(zMTre?T*A^c2{B}kw5mE|7P zf9Sh)@Rn_nwA={A$5!xL(E=@Yd6#x8c6m zk}o;_ZeL_iW0O}el`C{h>(92Yyg;0pRMO;0E-JkYd*%mAIZoaWf-FB!j9^MWGWBM6 zbstj=yh%~e$-b<>qMTeCF4aQwe1L=mrG$fbTUYeR|ITckk6CbCmRodV?>btt-+jD? zpqW$;a+g`WUGBvP!EsE-S%FU_!HN&&QlAp5aD5BCuI=w;*gdWi@T(V0Sru(bO|4z8 z3eU}~E(T8;y@VopqIF@aD0<^SwoZPkq)qF!TjKDWmMn&;gfpCXe(&-u>2PTZ1^Sc6 zG8G~DN3Kr}rR>7x9@8HU_c+XqT+Yt7-l=V!ouzF=zUjzO9G{nmJ{-0&l& z6xj0ItXHVxp|e#vAuL2O7uXQM6Va{Sk>I@^iMhTp<~wy@i*JK&$!I>0~Ux{8Dl z_aIGnr)9TnvuL@=kGnLK5;q@em-9gwS@L|W+?a9FJ$YT~HK+>iC%6nM1Zt-FK>{rg zgCy1BEQ0U3h049@ili7HLi$MdUeMT$Oe1qho;MZ3mea3%B%!kyRovm0kG%BU4Xp2- zt?ww!QkOyzI(GZ(B`_=1gz#0Bp4QG=-dv#7ba_|}I^AqWy{_d>o3W~5(H^)WsCLZj zjAMO7(Y<7OIra+6_Klu=CVtM*Nl>XdtP`_H8%}dXnoBljljudE@Wv>he7Wr~qF}bb zuTqGFqRAgy!n>9+fWWLKZn2zLt32xuf=2vSpCG8eH2%9Z_U z?6(VF&W06?r?p@CRW1Tc$t+mU9&-qx^X^b339*jk(k{HXPxMk!yC{E``yhc;#C#!` zF1+}hdTps6+7QuIoc6=#DMNgfzDHy0{$xDN8LsMCW--k#*RhZ2|7FBx{omI1X)?!3Oexn{lJ z)-q|(Y@{+VOuWnDU&+8^HSDS1@F04swZ7UZwwF2lAP#P|HXih2Y}wl;+-2@gjKaM> zI_nm4v?O1Zj_A{>5oQSb3hl%cHb)Do+3P8|yS5DS39c;EGP|~qV|bhh_A|4fPO_h> zSxl!X13b#sT zf{*+$Dx&^OKDp20lOma7`2^-i(!%$MmRp(AHtyAA!JmZwJyEbgI{@MEFcdkubbar# zy*5KS9J;m8!u|_&O?$8!0ZZL^Ake_l&YyH3X`YXetJ?h-iK@`*A(X?Ln3yW6;$x~| z38B|n58}A?;H#3aG4h_b=XxWG$g=7vcV0*5x2M8K^lnBkU}F2PoQ5c)xx*=PyNb}~ zF-i50|E&F%yhj-X!S+QyeuHRxONn)ad_qC7lPPR7ksrGh5SP(?bDiMiu+kx+8ywt8 zCTrqj5d!r>RZ&FM~X`$y~qcy zpz(0DP-`~G=2vbeo&VG$JpGl4r&V65VG-|NP|W=oD3kbEmgAL(E7>-qFbvXE#ugBcMbR(c{NO^_#K zJft(EH^b=SrI3z;Pc5aWj5L257nqlh9lJIf>=f57JkItKdFDxvHYL4yUlH?NmTg|6 zRy8^;iS4D-KFgPlXKw&A`zdpAGZeeTu3+}*s<`EGip9yRPkAk$9ZLPgE@NxsAC2h+ zZO$0Iy|y;Z-<*h$w(Zb~X^Nokxb@|#xBg>l`#)$gpfMxU4X7LS+1dNSBnBV3E&MhpK&ZToA5YY-UKfFxar`-8B6;dErjpAn|F9Bp5a16l zqYX4*YEsCd$SVkM=HWfB?Xi} zw^OrzPFux#>1jny%=8Mu?7X-v!5l(6GDlT`Hou;f3UYG~Khx>3VMUw1HB_G-eG7i( zL&f`%0l)LyC{kOaQU>X!cTz^M%{;pOtI6-Q?cPkJda9Y<&$wjCw68kiyd82El32!j zK3uJL<>f@`VjKt(mL1%!hf|zy)-C<7Il-VIUO?6X+`Y=+v7(UI9`Ap@&s1&`BrBm#P?WeUP?5%&@OE*doi+l zPfgQhNf8+tiBHSUC3J4sNr8w@Jl9SwSW*26AovX~bak)t7b&F3V;62KlB7!?mFUr< z)JtDBhRDz>>L&J@f1e2xFgc`?M;%j9qm#UK|2?0g9%GXr_M)495aqxUi$8dfmA9eO3SwLfO3X?!0!x5>Wn|=Jjqnyu@2AX}~v}|bu zn3qT-c$%U@@gYgPVXJ(BHc*D?07dpv^3bJQp+rW9VzsEEJF5<}qf&aUeG;$T37E)_ zO{PcRDrfIm-;jf;Sw%h}!vQ|qYwCws9ZzX74P>mqy$Lme&(Q0R>9bYLYm~l z;?sk)yWZ^A`LI5?ChaLUkf}BE1ey;UXY??7pHGTe!Y=%w_;5UT(_0R8F4SR6L^>Vf zZ7?TE(SoC7G-kB27^6myhV$4Q$uj-9LaiN0F_Co;^q$yFsoZ3@*Gz*|xFar~ZcI8y zi#||loY_Dx1Fa+&hy`G5HN0$J3+OwQ5Rd#c^?~L?h~Cn<^)lCMF3?(4(y!nP-@2kk z&&8b8(vz8Thj}CUunp%6-s6HEwLlWDld!Kc2i=lM47q$>V3iQ6Cb>ZNai=C;>N-T@Qp$+uRoTfH6b&?t;H-%A6j&odL$8 z-_WDGSaAyX%9%JBuma$0_``BsU!`l99G+8tYE<@9#!72)gc;@Kl3L!<_pMwI#no?h zD<#!)fq4fl_m~YZx}%jW%wHJ7!8S+I_dq6G6!`4loKO~8-eann_{6T1talxhT%=eN z0vA_|Do8m6Q0QG5;wbVVxIg9NZPph`W14>^GA=}}*&9r$(MGhM48`PDTtvNJtTx&c~PeoZedxtppE z*;&ccQmIPOW_b}dc<@M9fdfrzHm{JP9dydFI(%J!E-u&OYKG8mVr5uRA?pw}|5?+je$)^O5k^Mv1(ENq92X(C97VnmIsd{g)#8dz3hr=xm%d+6m z02?>&r%{c9OGhtVCmmYerF&HRC0Ar3l6Bl>Y0A?16m&;JYgWTQkX*r!quutQ zMe|o!nR+gomyOPl3D+LO>JmFb?mgbYbfDMaM zDs~z$eq{cwZm#S=5B*y?RsoN9q@r%k2fV0Ube9 zD{BrLa_zz}v%qVKW|h%9pTnpAKf2yJstLaRAKw@$E!_$VsHD<4BozTEgGNNU+~^vh zfHWq}q)S4iTNFfLj2t~+!Uo7j4@P}=KhJ%CKcDCMeShb0_=|Hk-gRBqE3PYeVHg#m zU))qvA&K5O@I7B6p)UkHtg#dYDY{#?l)Mc`VedcRD%cCpLO(&*4ObhYJ+^aqX4!V; z1F1~z<}>wIoUDRysdv$LhG=NU`SCIdd6oFhh}p`k8^g|_^1=5x93Q*y|UY6I>4 z5f9-kaw%@xD8~cjB9QufjNR!A&%9WGbqg$rX8@*JVg@CPQSKVzgV z_l_^QZlux8>O;5?hUES9z5|nuPb$;0pGuJQ6hV{0qm@I3;z#;BR{cVsYgQ?;zMD-6 zN&U{ix35?x9qXBZ%Q>wGMv9he{KNF2Q6S@e#34=+h?~H6p+dQx`M({*`&IfomI=`< zm*BHxAgCZlbt98#_;9w(mgC2u>j+WDOTBcRV7kGCO9k3ijxe9<=aS-vH~mi>88ta) zS1E8pb%54TX{cq~+4l1Y_am2zF7}CnN(-XWeD3;35py`A#=h#*?yS*vDZugYgwS+8 z6?~Go!>n^0AmnxouSBEoGpdzNiCi}kSbk_<@z(cd_ojbs%=TF0Gxx+>LSPzb@8k#R z=>3&j73Hyv+88uPZDqU)=QSJb>5Wg30=fmjj{KU_0Q`abPh$Jk?C2Fj_1SIwp;tsq zkBhqyRLbN&xqvh$zC4W$MdV@wzj`h208Sm(%&j0w9O}b_g}+{&ElFndKQ9llV?^D3 zq<(b%)@9p73z1|LJw=1nu4ny3?(E{6W+9h#DKFd|_ z`_{j5M2m5CP+Mup2`;H&vlc6R7&|iQ{JoPo0#@CKH@EK5o#lRZ-~i*)M?AkUM?Y~` z*Wn6o@_&xi^W%HlnMZ@G{ZKrtSB~euLsJW!JJ24B5*V5E`91CsS$?Ld4Yo{Qw0Aur zA~2Cf2M=%4x|D^zp@5v6ZT~c(5|ZwXxOw>KDA9uX-1KukZQch>wm(%B%3GbQ!-nV& z^vwXmzVp;$85@n_GMV)XnvY8^`wMR4uFPpEWUt%?`uJUw@;|ivJ z7{WX1XxKJ3k*e3})Ft3}IfhH_f$6iY%tdSMuAaqPD{c9gdSw|_|FpG-9+1(nZJ7d|ybV6-o zag9P{vr+2w{I?{Xww(_;J%={WfU91uH^UjxS2*eJ-uk>1Mjl`-oVsf-O?&!A>SGW+4LKYGCX6a9qPxEN93^G?k0xdTXnw73csi=rw z!6|6(Qz5yCiFAPj@gO@=((Wxy^q~*?+c&vsA4_;kTG1HGQ{&pss!Ia*Q^UJ$v3f*O z?mMxb>2-Yw-?~?g22~3T8vH5;i`VL=G(_q{6aDBY`vBiERYeuinRID^J zQs}LQR{3THM0GJt^`4K3{&0(O&Zp2esP_jNv#5*zz|G<8>deToG`n2dQLp*=H>;r4 z>Xc31x7iB7qGO7X2vL~hdrX?>klU&OjlMdupqsN{os_qt04d&;X8Ixa3L(nK8d22M zb`RaQ@a$P*@0G*1kN_v%kwK1FhbweCPH`omgDl5cOGHt(%1dL8M<3_mzG+uEP(+Ey7RG@L8F2#dYP zF@Z=F2I)xO6=FX4I=-*eai((UdPLuXI?Y>`W_h#|u~OSk?N44#njXOY&M^MH0j2@u zQD@V0mQKNoRcy7=LO0Uvm5XP*P%V)qXZEaH5fwX@SUljv2a9SwAyUR}}$<(_lfDY7&8%I*0J(< z%LdBadeHVa1!-ly$<#eGo3|H;P-M%U-JgNDF^T=J{9eX~ksC%X#1-j^Ty#gAl#H^O zbK06Le#JqAz{4PDt~)@Y8W5#JR+lnQ3frtSQN7UZXVtijXHFhG1qu;c=q{jY!Pc zANpaAsP;JYuLuN_V#fUwfi%JSUKODMMKg5Y>tF08QTt8Luby5;1pM*r>1Q700{#wU zdY5iQ^C1|iG^YDPxhEuj1AAPCcTA=`wYyU2NV=Egh9AoXlH~kxjp8Z}JfU#()ttzN z?Kt+$WXD_7Cb333Es@hA71rM(3J%(q_JgeD#^Wok{KgyFG>%>PA~kVlG|yp2-pI@5 z`{SGM1U@r)@-!Oue}_k8WJjFcQa&y|w>YgCyGV4X_m9La#=$Uqx(W=M(A z1$=P&zV7WK3gCQ|_Cq&3DZm3zsFWaxDgce9zt833$Q_SpKHzh&{Oo$}J7FN{<}U3g zmv@{z*_AWkfBtkIRk(@dmaruHPfRgdN0AGgE~Y3O|4S}0T~;@;wLGq#@N{>!3q+G~ z@b%8zo%%nGCxwpKAWIY?7vn_V_A_p_E4*OmUFP9tO~V;zpXdI4MD9jQXoq{!2WFg* z{9;({m$>nR-qP$}jd3W2}(cdMGB)hD5i2!cvgam6r2D&+&1)E3qFLOkiY6RGxE zw2GUgT|n4dRN!F?LW}M&q*LtsO=I^wB%WfA>hSC2TfY@Miq@9-EXf4H(pFFrw~YOV zNb$f#8T;eo-GGF|0|$@6)%+i8A}*mX@d zg`hFj8X+2W&r=SCz|i|7CTws-wOLck5VSmT}93t^=vs04suZ6{lC?|GJf>4`*) zkOgDA`|Y@Gh_=c+GRJtDPPb$CKzz8(Wy;Xbze@9tEn9^4kh;oT8v@cwifSz8RgB$u z%-8GZ9{OOiqQ5=D~2@#RD|u>o1S60v%LC|%ppKge)(DO%-6&o+2}!9Ij1R*j6OlP|N7LN zRrtM7_;JdTw8HtFEv=P)H`zxCq&Q=*mA?f=EidI5F7V z=vn&7w<3LWVlSm_qSB-9?AYv-4*@C2iBfIU_OR^0tBr6Qt2F)6E``PY&7Mi9Mdtyr z-wUz32^O;=lg0pr0{42r%L3N3s%;}saANFECg50Z1mfg~G45+OV?bip`)$A`w~vlF zcilc`(Bobmxg1}r zdOE#G{Sf! zbB^s*uWpMMIgZ&P-4_N?+KUWccX+z&OUpvIAEyfA8P-)z8QT)7cAMq8xHTb^8jbVc z4n-2p){owiCk0+rcu?b7w=y=7Pajcs-~lC&e}9NEvFPM)mxxqQk_-o5+aaAM^utJ@ z^MPUE3n-f-BWKQUiQJ5m?s@JPT#6bE_iP03Qr-I-?{#41_3a6M=`hEPn9vRr^hI28 zZyaUFnsEFg<}Vv>ek|h?FdP%$C4$L{m~}Y19gdu2EZS-;r;j>deUuyADKG8fL!eH9NF|Q+;G4x5?H#t-7@B)}wW}V3ad^kF)v6 z1b4iX%p{QQQh8RDtPNhL#uj?p1mJ1YFOV(pw7z=!j*YqebB`KNnosXGx8C*KjD1e+ zNi{2*g!&NoyHuRFU5@s;VbFKNHhMW;<@oh2wnm2h>s>c(O8YDCO>|aLy`YWnt#e`u z(`ZZ?i)*b}&YyS!>e3DA zdamFL-O32rKrr!Dff#uho`FhfTWR5%sKty#%Fg=z!F+SMzZ1Rc*j0U!6 z0MT_d%dx&P0*ImN&A-CMM+)J;!$mC|7fcN|ssCiP+S@8!^t77}@mMM_M$e!73=dQM zr5(~RHwIAOAWb73KqCcd-c&jKO8E-wSqaZV08TA)bXg7ia5H=ey1rnpJ0K%c_v^Hskco`c5Bb$W4;aBj(D(pn{K>weBplTda<2&bEYoND zNoFT4*0yr5fzZx~OkNEqH`DD8z{2uk_qdUXw?kC2s$)VJ-iPQYDm0+|$?H^A5@mbc z-=@bFqH7ydI7HeXmEp5jrE8)-j6)?Yy%%ZsBgqT@{;8x=jkCA8(kf?~SH@Fd6fI3gZB2E*Sf`SS;t07gy#S@Ky8yS_=AH*} z7_rBF9~tx=4Z%bQF@{ldaAD_&(D08BI_Ao!f4=56&EA+HX@&D zEps2sy9jvnmue;GFb7a<%0n%-f%;PB+kTeQm)Zr)Gm_EZS7X%M6d|c;#B2*O=25Rq zFbTvQ^)_)~PF&6?3C;_zkMJD?7aN{bXehFY^g&vOegB&XTO?Tx6@= z`JMYU3^<>866a7%f3QLb4?lB0Rey|&R#WF=@#h+@_^}ol&I6OS&P*7%y=g+dP}x5D zgZYDWxSESPLP;HU%PGnit zm`$2qYB1)Q3f)$StH0xCe?YFvJur(7TW|5@S*S#Z4u9^2I4Pc+Y)quom``}B@Gf5o zd@|j-YKLLg4LsGVoAui**|%=snsxLC#YDiKgn^IJp=yv*)@uHZBa?LD~ZdiULjfjlkz<8Jq&wmBlsc7=5a5@V9_L=C^RMuvy8n$)#8H-^2e8CZ5$faPEOqcwpIoU^xAc(z>jWC zziNDwz7fsaAgj4 zNf6(|Lbl#klhv1ELJogE%_vdyR?5m9yzhe@<$mfaaFQ`rf8I8&T-fFGC&N0udhZ&T zc`1iWT*l=<&=_9~NZ<;zHE@C{W)jf)eUY>O5i~fNM|Ec02>99?mXe-WESJHy! zfY*p(nb2A;+}LKw-9(0r57AI6TwII`R9dRPsYj0{QPc!SOh()hE=YMPnuCR&TE1Pj zV@GZi`lm^TR`;ScYQ;Za0-FRt18=KL6wMZ{JvLpuqPB6V^t>u1c`R=qx0mML%*NR{ zj7qe3GHB51WF!rFR6OhmP`hWCYG+AllbRs5WZi84aQGiBfR?C!l#}^i0n$nNIsfq+ z$u-oKr@JR2LdBySZ!jqllZa37$2G4|OX}f5L!saMMf(d-bIi|A$bP1-q8^(?8$mTH zjekx{Ck3gh0K2ZG605=y{DAfDzAu7#19L!kYMkyLQlsra2k?YzWtxg4#RULf$)8RE zjacD6;$RxuW!~oZOmJ?Rz8G5RR2t@evQdn+FM8F{CSO47F(JW$PIJ~Q*$b9J#R%Zx z6p;}N6GZt~>W{PoGr7ft(;1`=8OZX2SxRUUy%QEE*{huTSkm|#t4$7ha`spXD_sLn zMYt(QXz`oOMAZ{kt&V;tV8&8N!*l6`GEDd!=rnIpP|R#DcYom^^w%_}nzS6yqB~AD zAcm|XU{+W#+f{9kd&nQ7o63CwV-aW>6S2nlf|L_5SzSOypE4ty@JmC76IdOt)}GQy z^X?)^#NjedMTKdR$sTn*#x)F!L74_nMfS7Y=_g)5DGPN88^q=lsF8e=H)9S}Inne< zjHDQX?6bTZeI$onFQm^OJwo}tXls#8bsLrq^}B!&R)bP1sp}rfIpiuTl3kr~t-{ZS z-$+qp-Mj2wHih9WOSM=76=^L2lyBVH($cVs=W`3y%$wfPmBH$N8ir6wW1lSNmB^v! z-JMH{S~*&5oo4S(cmVFVL#=`CxL;GHce+G}?v-O{Be>I<%-7yHJ=7e_`MTT3dZIL; zF*|xTPOyBsMj#`3qg~><=}>n{ zd9u?4yHd;4;1|m0VG=1nHjqOwCY+ORR4T4x!II*s(rd3uC&9oJH+;PDF28RLhi5)- z1Wqrl63u0&*5i_Pz4`>Ai|Gym*x7WYcwDZ?ekjch?$0oX*#YAHh8p@5oNq5ikT0xi z7&Z8c!CZ53s7_bhF7T4bXLkz6YgD*Tw=vUHa1WJ4k)xkY9?A3KSvAlShG|i5F(LXg zlz-fp@Eooc!N!t==B6zmCzZ?@nb)2WM^OJJ4Pp=07?G0R3 z=P@#gY0Yb#C(3jeFlDaqUcJlquzc~A0c2U;nB!sj%KEO~Po+|m?5+(5UZKM3>08x2 zU7x?FZ*gCDqD>X)_G6zq$vhVXse#PKG@+PIqH+NXBU;3xgB~l~TdekDosg5V6NT1EI7{VEV z8As9%Vfbbvoj`JXm-opm*e~g>W`v=sj#`e-7ZdT%ugYX&F2Q0)pOimn$57$IyyjDG zW42}Bz4wn3{MOQq_GKasSv8B6vaoz#mYlSWu@oKAzN*9xMqC?@WaP+ijFVdcnokfQxts`!QBD(-)2}qojzyb{mJGl3F9gr z4IGB>^&8RSlr*8VLLcRTi2~egat)|RmfK6bZcP1aFyrzGOtuYpICZMA3RZoS*Ly9L z9pwPg>$A3ko2nS{_WkMyNo>#e5ZYNTzEj|i(7m9J*CKP$-b-}2Q~h!R=?rj?(Yf1f zLL%DQ19}6KM8+Q_VF+rUffjduN)~Tzz?c}f2^nIf+udTYPiHj6-Vn~r-jw2dCu zqm9#VGlQkyE(=b&u<696%Tsx}hr#&TOZC;p=1XQTNGN)X5Al!a=sY>f>V$Q+juwqW zsw?9zuhSzHfdaMJ04T*qTseCnhouRKq(^+)H(Q-r2+6=6-0{;7m+!TFK;wPFh*J5Z zxlIA}NNB^VTSB_qg>;<0DYJf*2MRQdkae}4*`aiERB&yl=KK;NU=mp#(7L@GHxKAM zRKwHYr8_lAN`4HSDb3S@pD9`V`g6Q;=UlRuJF_~4DhS<%RwqkGX7#!tq+jHH@{lL8 zgn5wdTxwA2yLzT`6oEsS1fklyb4i_g=!(q@{ATPOwjH_dD?i4KXj-6e<*Q969aB>i z`&qQZY>so=V>h$BFn|5X+LUOiKEB~~R^81;_`X#hiQx20t#OfJDnnyj!$ZOj@;9H6 z5Dyf6h^UpK1zI`W?zMR0wigozShy=|#V9>cawo^NYw z;)e}!?C|3^3kf%yOQcW<^rNxKHd!x$(I?SjIGQ?TtI)JvM;*OG2iLu zMe$-a@7k;B3XUAI!i`R`hgk}>UwVQ^O+5c-@7Let8!SD|$|;mI^XYWqEA^7PgLXb1 z{Sr`u9(5Y98az2vPO;dI-kpdLMFLM@Nm83W@? zEwXt>Iz)fO9}Zi@UW_Qn*OP zW$BqKDFntYb_0(lRGQTu47cnlN(* z=InMMnJ@G-Xwn}eH`oHbbA#7w+3?;p^8F-c_VEq8RNFs9&CNkh1T`>x73)k9X!q?y zc#(rmoYB7G{AakGN%F8lwzN(9@fWlwYGB7YEPrS0=qQ4NXnS#x3bC?e4){mZ##ssE zf>(-@j&H>mfZ{m-=qm?WSj2e9mmCq4OgF1w6^}2A?o*a$=Amz$_$pksLCTH6hMQ4& z{HazlN(Em$7N`xqOgI(kK_`_4jTBA`&xRF&OLc6j~KW$Z!W+)499@>j&Pz;sJ^9@ymrlM-}ZI>HD zW1W2M>eqMyLNEl89r-8Q*qHw6rgrr03T@`L0g)$+i%#Mex!-CmQ=?q1y9^7B&qpg{ zxrtm`QBa5wl-tLnRD}>|Fm@c?V8Akc7|Ypk4{Yy&uP}R2+lPLjz!FJ==S6NHs6+vB z_^TeE4!T;I9<+3|K%7Z1y~5PMma=?G(r1v23r2OANU3r6Ft1DhJwfwJg2zU_JvSVy z)hbkWtnJJzW4pG*(CRNjk(+dT7IaM2iBzbgZ;cF-1cmdFR=7Qa}#W}9$ z(8H(T(oe{r20jmsHh0|H=G|7p6Aour`9Y&859+;7)c3eXzx#C!ej2;i+faMSTuyzO z9@(HQSD+V@OIGx+o!b=J^&I%<$}4E8G|cyh=-vJxyy%_^Rqs$$czbiv4*v0@S38OO zJp>T$HVNx|f|AC!bgpEbb!d*m1Q^%%ynYYriV@Z6LhUm!bz$e}?|?$Rf%H^R#iBN? zaI-R~u9+>X>#YmwYeAn=8+x2bymvANsSQ<&`up3eO~>cxX#xp~LgW`{h;fDd;DDu= zfB*g9_%avZerYt|&eFU?T}@&VLLBr%ZWP2=QUn#|wOq<6?T#vR7razlN`^FUXv+#7 zq(}TpZjr%P;f~CJ;HG58`Ar_AzfvCM;O=Tt5iYWJT&V z)vt0zpX0+! zORoSzuG4>0@$pPOc*{HB9NBNRK)gDimfru=zNoi_lL_}-D_r4z=DUR)fvA)k^|m^T zT-I$l8*I!|&Z6U^i9q8<9OR$?7VZcS@i`TBkYdIpo)Pcg3;VZGh8^B^xkJVIA(31S zVxq!c$hmr+TdobAXsOc+e=ewV8|#r1WG*<%k(w>{QSKwBQYqpp1tm5R&1)<*Dx6m@ zCIaIp{2`*<8AP#a39+X?@eFN$pxl+%9Q|4Kg(q7s#8daP}wyJH2Qi@ zc`P&l-rC*s<%8!NYLB1oD`OB7_6%!=!HvrlQ1h-}bouZ0I$di@oFKhsK=SV}>@9#m zfDd4vdEBWihyUQuB91NUK*&&|EDP0P*uAH@z~Ow5sKT{B`@6F(wCmL?=D#apvRpGi z`#VPzv`cFEvFWV%>AZ-UDjl3`n7^a#UcD|%NU>`L&Z8errCg|0=93A~8N2qW>qGYE z&gV2>ypsm~wa2E@RxKt1Uw8L^mllWF9!Z`w0|XKuKnViqS8Cwl7kls2#ECJxO4i@c zk6$pb7j>C5;4uCsyaa0aaSW-*VJ&}h5TL~Ko1^PJT~2-MFE?02+0d*{E1`skSl}-$ zs_~aq>EB;g3_eDf$SiGjiE0(fIWBEB*;&NLcB?5}zkqx8t!9e51oZm}{l)s0)i0~m z3+#J(kC``BqjuIV&leVbB{zbaaBxo-w~jM>z-^)(yzq5YZa4p~O9~;(PADOi9+yaA z$W)+*3gn7r% zH8DQ}{bZBWd3vbM_5L}0fNC{@DAMI-r!iPUuwPIJ*5KjQz#ull?-%{Eea*!MVzH^XaRwu%gOYh^YY)R4^b?v-<*~2Ah1pv5h?;n{RXCps5{iDh=&KhVxf=l3aEGY+?3E7H*1Qvd2*Naq8cid=b=nKF<(IzAIjVLMCS&idP-1pIi zcPl;5dqbq#k~bZYDj7=jU$I%i{+NzcDX$)-0ivwQFACg_P=OlztG`e0znH`3K;9pf z>iPVLdZGa$DTxzU8!g{D)>{dDwcysLk^bY#4@dj`H~SllzFSvt zxTk$~>H-i%}jireO#kp z8yN+l;w7p{*5WfAqdPD(pH6LZ{~Y{_bm5OAs-_YOV2ns2$(^Ad2AOIOzej)R@G;<2 z@Py{de;tuDR`tN5Ca$R6I|bNjnfEl}!L?6rXK9wHF3o|GJi*JL=PrdGDD{ky;tgc} z!{E3U&5_ePX&u+Iau_RopbHGYsT6pm+=*IYFm%&ulRkoZcQ^Z+Yz>fyHTRz;5Z^To znI{VzN85YNQS;@#84P}3npENB7kAnJ`$vvHU3N80H_s!gbNf5*TS^Qql<-jD$fS|q zg@emvnPm#ho{R+3{*L<3W2ZXKj}TBd0VRbV*IE?|{ROp?f%jtX{A4>;aZjkRNSW(| z9+Uyj!rK3-A(70(h;|p*^3nA7VtXbIMVsMeMJjqx){07xBU$+w1jm$9Vu!`mIk8_| zSaJtrE^-TQ`Kbx6uBn0i1yejhHohI0eQeJ&;vk?Pz*jElF2P0X6VEluX2*rvL|=Tf z2E*BtW-=k^W?b95GzPa6vhGj)aU61-R*=n&IXd+Bl?l3-%!-=*lQ(K3qX|IP`g7~^ zsER>1$t(g%HS8xQhItn-PbnF$)+~AwRF+IrW1YK;3uz8$(=Z0R7LQ+@U(CZSc zCI#csP2y$&6=zecV0waXSp0+QUzLMVJmb7H_yjwTa&sdPzLx z7}5rkuB~NaX#{dyTjM&n0uP3t5p$rLo&uC)?D}_q#~H}Vrf2_c-`kSIV}Jg$l<$Dk z(7J{IV@&MkLG^1S-cL|MZ;{wQeN%0Dck%{kl%<0}hdVyiFETj`P7zxdxT>|gKvP2hax!JJ}9p0SU9hn!#va5V_FfVV^TY6`yH zhB3mr=9*u@1(;U>@5n}7TC=p7Wa8%(pxSbv&H+ns@6nb_XMMn(8rTOluo&#RKQ1VV z1Dk>vYb)Awf^nue!XDC7UZ6~SokAoHsn>i5aD(X`N;rY+YL_^;sHYr#V^d{(2K(PF zGhNcZk|$i;VHm9lV+{Lt4Z$q}BNt?N_k zc31kei7uGfKG=64KeMLxtuVs2@|^WySRq6Y(l!&sCs(nt7dWWvwsd(uGKU$X$zwOx z5F~-XwJ;(CO!w5lagG|zE*Wii{Y>&^wq_GSD&r0uDQDXOk9nHA~F!tj&u{ zZ5Hd-w|cZm?%7~a{_s7mCIBhKFiJ`AU`?%^2ZLxVKZqP84^?XD;W-3wGq?^@Dt0L;@jydY^f* zYu!kxd`aZUqzgk^H^|Nta~mq}^I_R-`?j3`+kHm&EB9!I_u+t*VQo{2;bzP|LYa-XhQ1ruA{k!cGJBVBo;N_f*1A(9y2|@9F#hCydSBmR0#X*s^It*se5Bl$OE((MKqy>@nRp_~5Jkns;c{cTrERW!V5f~-X5Yj3QV2nh0B&Ay;8rfhTYRJ`@;Is-04s3 z+RO#1bmJr^){m=DfifeQF9h}+brIwF-sevj5r=*T9?hd+?I20?~$yiCy)G3Ee6QT#w z_4)+MdjWwm4v&YQUv`#r&=M^_`h8+OkHr{}E&4sBNxybTnU(?5@Z7Wp+7^#> z-UFx1gLDu;!(FCtdJHCd75LkPPeCOL6dZ}!D<3C zZ5VynE#G-ZC!tOp0Q+&13%=p|NeBbNxypT@!j++PKPmLxm0#`kNHCZMoPNE4V{C1D zoXAurEnO2Jfw>dIFWd>ZHmK{95xafaC=1@uVo2nuA$7z(v#1EO$xp!mca+4wP&N1S z10e_0egi+0fvx@{Yo@iDLF~Yg_p6RZG;s5T(6<5M?0OOX>3J-cDDyc#DI8zECchfJ zm&8X9JN-b6uf*xa?nx0)!s z(Nh%UQH{S6_HA9Cde`^CUrDgLmfE6|p_?CTg;84m{&E)9u--K`mD{% zc3(gIad`?^>@<6-L1_-ocjqnTGz8RMkLK5V)K3ln5ZW@byPW1NW$cVvo;}d=v9t{7 zvdXz{#p`cuHd|G0d@L<7Tlsyqs)v~pH}k+LzEQho^|bHN{oc;3df4w7nt8RcMDBX2^S$KAByoKEYFQc{R)T(x_p*r( za`L0Ve4V?Wn22`N1qdX!`Z7J0hoA8z69A?qKF!{TwdWba>FX+zPIJ?VH$9a!0V?W( zDw-a%_l9fq)mEcqx0dNcRIhNVeBERL?rf!#Z2i+wUgOFM!si4^_+OG%slq}`FG6c3E>h2|{pA0ywO(k#~ zAbiWhu&zP(OM@&AEX8j(Vspxa2MI5|SQy2juPpWwqA){I0jVe^^#$Bb|ABg-tqIV; zhgBU!At<0pZ&cSIZ&bU3)(_uNA6a`P6SM9`dC&L0Cc8I78^?XPIUfYsdnqj3%4xb9 z6Hyi#SjISIV3^onKn=PMQb8E`Rwb*Bmaz@i4^pgW&eq5mY;HQVc1eHHpn_Q)#$l5? zeqqW&s$@^9hjTwOeR<6&p^?kue4)AKQN~^AJ%+VYhF@HB005i99UU!;L|?TZc1zVK zhY>p&g>`YUui3D{N$O*c?Kqg(g|^js@|aQ1Gpz$kf@6xu?OT(Ps<&hE;5@r+KdcWN zdwwr~LOgDj%OtK6^{kH34OHd-^|KGsMolbGdxqEeA#wts`Em+Swrne0O=}y$$K>k z=~i!2cAo_gGUMi0$uEy0W|Ztf%hQS6OkFSB54*y0S@I=yiqfdg%^**INdS^IC^L#dn>elhx8}#>3W*pk~ zlvj&Lc5x{^vU4{k=Y&!AcJ4X{>BSZNViBwY5{Ji^|L4B8QQN?(-AAyXwF454odui8 zF=!Mt08Gu1m6iDDp(gy%#wqb-%rOOay4G8fi1m_jCeM@+G!0nmrmBe`UIrN_Z)1Co z^!Zmo8!!39o*@){^74{t=sk+??--Xn0Bhd1?@TUnEYJ*w%VFxp9c~9tauej?n2&

Hg;TO&NF4RLU2iobSk#;aHe;Hu(hz$PeycJEd|M9~ENHn03@s&3wNxaRBKP82M~!at(fN7~0Ed@gv+sYmF` zdH#>BwtuhL!T(ybC^E5n$A36xuS$Gb;&MlP@yEV%#|4_v^k<#{8iD%wlZ{5gjSQ}} zpuA7?4_|adC}IM?2A4s0LmiHce}me5l1a96ggb!t2gn_`Ef44L&*H{gyH7rghIMxS z+ST5S0Fi3s&RwGM(i2^>Z~i~WvhxTKZYEx*f7gW(TRpQbVL&u#K%{zedQ=5ycB{qC zo}^&UU~aMT8q;_yIs1j zCfeiK;cZoH2Nl%wO`qv@@5KM>O_y?8B&%h9r|(S{flTfb)|1A`g4$sJ-{04NsxPWh zS*#sv8qGJdQHVv?Nw9oLo-Tkqq*Sk`e40>_@KQ@rNG1GZ=!3@Jjt9fWBWX2o=j79c z{|81?PkjOme?2pMXDKX;Olmzd%>=8l=5SH%wLQ9IkSZob?zQ*T|3~;a(|VH_)n_%@ zd&Tu!FCjh#!`8^Gh|V*BZHH4Ohum>+iS1nGUhBWI9R2^wa=NK;!7Mvxb4y{ZDtm)c zC`c6K;iopSjA6>3*>=`OEo6L3SvJEYi_?z%M_BjZ=`?$W^n8>N`(t7*0eWc@vzz)` z;>*2&;m6{iN43X~(KcNb)8hCcAe*ML>xj5B! z?@^&O2pt?9a}N9OcZ#1p#wLkFP-3mQQ(#t%60&f2=vifoB zDsA&rZ)6h|(Nwc34~Uej-G1II z8uMJ!HUC(R9ueMIe)?`Y)hiao_7}7V{+}n|-`~d=zDQ#wq|~H0G|#$Q0JgjCf3a~v z<1bxujFmIIBmd-H@p$s$$@4I>_0d-4a_E!_9J6;i8199fX^IG#p;Zs7o19`I|9Nyk zc+QTttRbmCsdbmz6DFv5Ww&xgLFCs+V3o29X!a1za~I{`#x@&nAy>1P3((P zkztqTsc_d0a~a2t8TZQ}lBCf0 z>iuq5w|s)_aM#U=DKS2KXX^JFH2~7rF82Rk8ahge33h&F}bhg zag0jq`G~*H<2cpe)ktJ;+Z9b}st_Ta939F>9S)HEk>x&mSQjusK46VMgUe1fZje0o z^NH>aa(|kh?XAZ_xm;v4>@BAVJN(9B-|cR10@-0zFaL*Oq7uC`mxwm`B zn4IlM9vnk;{8ti%<2dq@Zs)p@>4x_PcX`AQMSnetc00;LKujT@*kkvnQ$j=XY2nf* ziI@H-lKqRm>*oDoY}%kdJY>At>C0s8BXBWD*g4c=Q*DZ@_Qc6Jymt)i2rtv^=ScGo z7^eLJOf|3-ii0uD+O=Kj9Tw}wP8!H|LrVjTb(7yMVZFm|O#R8B(vK;XQvf$G5tW1h z2TxA2+{g8EF^j&3Z|y|DK0?SG2bxo{o``eVTzdGQY|+#@2XF2h8f_!W#k*T-z)6Nh z0Fj2=NA<~Hee<9Ui-H;|d>sW{7$0T@^y|%iA<7S!S+XMcN7)!a1$!0ZFENeQz0#ar(om{BY2@Ih1^4j zM4E-$@mw^Zuxe^`q0F54$qQU>hTQ8CNf=~F)pI}O=WWbP%LD8&^nL~OJ)=+&9t#p0 zAwku{E+XCRE}uZIbkM@huS8(p-(*yJY1?x+OJ*P7HB_gof<21xEKZ4^9$IaUS_H?B z(Lh^gP?afE00Tn<{v7fl632rN!^f3haZ34y^Ur#D76WWeAY5R3viYP+?Us-Mr7h)E zd-1=y)_-4S0s~y+bqCq(Zk@l$It_2p_l2(CX0s#ZFQJ8M)kU=#6&DdNo>S%hWNAqTo+`LnpvXsPLeEkC$nTMT zgbp~1au0F?4w;wNh@(}5u|;OvQrhM^fzzFfYSFJhE(owj+=@2NXN=;MK0Rk?(>|eN%5>-m~sT?ntiF z`Tw8Q-dw}jS*JC;FL+V2+CE*EyuENPI52UnS4V_t~-W#Rv8;WX}Pbgo( z`b5=&Ut+x;P6Wd52X2qA;T5!LLUhW9r0^sK4+dOiaU*fk-9WOR+o~bRP`;?ksal+> zdztbkBLHX6gyn^f{UIypSdrpP*y4^^Mep(=7#Az`P`K3~jIe36T)_mkZ-4*&8x6r| z5%db4==)oU`9BZ$pn}Jl>0X!tR@AS8b-Hl|6B?#n`wN^PE(PZE9|L^vSSTzr41S*7 zU>$9g1BzvbdU8C&M|J+B2VwXqNOn$fqgb8<*5J601N=VkCd<9deyW|NY!0Q3c1})} z)cJ{pj7LLMRC}s{IM+@+qvgB!aHRja#jo%kW|i`$jclmopt+=)L#W(S_B|YTdz>sh z*ruz2i=QbEsRmqr1O^ugX%=#x1_cS|VtA0{jS)$q+gU2pm=i##=he92mZryJa>B3( z=FZ+H1fwMo5Xzbs_Ub4 z=B-|PB=2z5XN#yl4a4iL%qD^7*DX4Da!b(xjcq~92a372GbOREG&?MoX2BM4Zho+X z^!vMVZMTi0L}rKX4^1d;1pPDRM~+k18KBwPiot%ux6z z7kKh$-LVKm@fyC~*99`^5(yt4I?a|A^P2GKU2MG`$ocz$&VLDV{=}MzsLnU$O=0+T zNoB%3S2~j~-;eeK!;%wJo}-l%hZi-TG5NmKOT_YQ%a=PDyO=I4WP7nq>dlFwtOHjI z^A?mXo%%c4?9OtXp1$1;cQ{{ZpM0J6OZ~jL-+bT=Twg42Jh0hpIS14x*qroq<_`y@ z7fNbBjZo$tE4E7=d+KtjcJWJwk|hy~m3OTTytQN+@7Z|;YnDE)j{ITyeP^k>!3VYb z#^Oe2E;Z;x%)b}kcD|?ZbA+$OcGvSu9q&Bazdb1x7>tcIf+)dQV_`bsu7O_U^mmsZ z-tAddY+I42@NI*q()G<-RXWstzk9!#z2<07PQ>{vljUn>oPSe%qHm7(+n=ZAi5I&8GS zmiyXWJFjbdUx(EsF64lixEi|yf0$0-e(62Au}Z3Ed4JSy$2)9?zj>4udN=?0EgT_r z=Wp;M^^c*q;TES|e;;H@`}gUeaX!*(f6|<>lPl=6w17eebN3XSpWa6?}cD zAm+fP^O1887#%Yyj=%Z$snmYmBE8b*-T&gc_Nq7NDNWFCLNOuVk5{C5x zZI*|10Wo(z9kjkJe)D~LZ*Ee3rLNA6W4R}12w3cAzwf^Bd)gaN$+F@}w<>H$Hu&UB z*{cq0E7A|_l6G0n`oweb)y?L$eTBk%*Rb}yIm2we#s9VO{-Z2L(U#2ze%k){*SWWx z-A=$kJn`goZ|M4WaQq!GPkpspdyx&VjQ+7Mh8=6m?(^SBe{1}HXT-a=k4k|fdstf@ ziyDE;DfXt`p6m2c>CDvJfE)hZvb#&Nr@#MSEuRFe0FvSj+L)NqA;!t{4P z?p(gnRD0xm@ILiV`A@6mi{v?gU7OkHjkynsj9-`j&U{~aE9OA$qm8Hc3qNZ9)P9eh z>BaX4Y8^;RS3vdc8EWOLc)Tdavo(%Ep^a z&vO?3v}5?W=2-dt=So1I-nYi?5JY_rO89@A5qy-Q-n;vIBf~G-9h~Sba`@^IVE7B3 zuxCN=k!LYBY^k8k`tKYUfE5vT!+|STB3AJKt9S6+FDU6K#>oH#p00i_>zopr03xR% A!2kdN literal 131711 zcmeEuXH-+$_pKK}K~#!V3rGn_4MnAeB7$^K5FxaHG?gZugeC&gdv5_zK&3=LIsrj? z4WJb1HGu#D5=sb=m-~C;eS6;f`+t06yfIEj#>j`W&)IwJwdR_0?nFJ-*SK=&`lWN{ z&Rx;cR5Ltx?jq{kxxWas7bw5j|Ejg6e9^tu1bd!4#|HTO^Ou$(|IWE{x6f&*J$UMu zyNNW91z7qg9?s**ii)$OfK9wjv14%w8ZCHCB3m2s6xSD37M zuU=*3S92D;+VRf%T9mc8X6nps^*a}%Vk`<6HC=fD4bq1VUoe(6Ccn&#OTRqJ^Vxz! z#jW%V4(Do0pDC8U!kv~K2DWcHInHyu`Rl*CLQ3_R zG3GKCscrx3VI&v>UfI4;nSYb=KX^I`Ou%JE%#GK~v|Z;eSpRp|g}j8nr~=RHQ?+q? zKX>`Ry1p0SB~=RCzH9vt=77UX=>jd`iY(p#U=Biuy3d8Q{3RG-efht98Eb14k>fT> zW`UnE_OJi$WzJo`EYE$O`uIXF^;^mRdO;tc|Nr9r|Hb)#dgoY=adHGvkL+`}jy7z- zxT6L$eYuyRm}B`L4H)=el)VsAo=pkzi@f6ee@u<9%(7$c|om~l3ZpW0D&|zhV1AXa!UfrO=!TzwpJaMmb zIA=e`q5?dams5e}_}}-F__zDLMslj?1~>9nDhA!RN&2y$>Ltpx9%S{kz|nKfX#$@_ zNh#LY>+%1V`Ye<$+}H5%VgHc6)ENCG0dm{5_;D-TRx4m&;_G02m&-T^K&bTUCO2{#fo?5D9r5-whnSlvvoL)fIi=IY~(`Ri}z)ps5G0XyH9z_ zL&N{9!WFK6OHcJ&m+u0wjXlm~f~mC%@dxV06MJsgWM7$~t{^HU+#b9(iLDv_5C`Y} z)s4hDO$y}QP=ih20JL4!Uw8lY!E=UHZn@XBO(op89e@APQB@8ev95P= z5>}AGT={I=9yZSKQ&POMOV7XM`|oQ~ zj~`*|`wJ>MO>zv^9eOA45}#^aA7|TGReR2UvrMPBu6CyY@4jg^RN*WBtNhJZy5%6x zE0yt{Pt}Y0Mw)^U4BcXKRa@J5e5{yoa@e@5Z_*HJ(h0;KI+pcCPl=J}|c zGM+}Zohz62uMF3g!)6BU7iDtnTjC`*^?eY&S6$Lg-LTDN%dqh|sEEH+fVw90~zwrE4$k^C&Ohfp;SLXw-7pcMg zM+0VKbY?tXURz61L`b2ZoyQl&n!0Ts*SWuKUUjm5nH8^wMqb7F+E`QaB$YVzAM+&2 z@BsS$rwCtU_R!ZkLRtU4=zTUC8*^}7oa6ksg$gQC)AvA!+go(7bU1%7b!l+0{C@SX z83~jk1oH!rTTfWe{ zeuLGQK?#yGDZ<3Q`CxS=v5}c@%EgT6Y-^Z!zYvefsEfZPxpiT2YJ8_lwrb z&pTPGzVpEXX#P50z+fYtm?G_UCY$V_{!?o?DahvOod98x7lRlj9b}`Z+tP27LyOd= z^n0FX!g(2H%y~hW*;WP~j?n)C-sSvIA&zdnR|!txLzU7NK4umZtmAtSfLrKv=-aaj z$`#E{)YqdYNEClaS+Itggwta7o`1ao-o6A3vHIK`Sf9X+l zDmTzxEy)n(w{3(z!3zEC*W7QvbRAYPXT)<-BhXilotTME=?q<%n)0X2i`A$3{n*wV z<)$B#K2EBNkdG577|Z#5Itucy9luKne(UA8=l|H$B!Pg;81V){W|Iu({L2Y%mx+N7 z3tD?v)Rm@EDNo(^yq6~p26u1tV+1MtBUE)tV9Y{nTHZ9roo9FqF{0Z4UbywcNTTbLBV3#~V!k>~?&IbI+y<1j+UYL0Wm1nr&Xr)$_}s-(csfDt}iz{ehx?))o4J zO}*TFZfz+4ZBLUxBl^`K-e@n|!K!}+w&{OmAZEiuk!q;GZ%3uzOS z{I`1lb&5(6@zcv}Q>+X9GQ{f1k{0fj(5yJm^Ps@O=3IURUtTU}Hk>4a9JbG4_B~J* zv8-s+>?<#_@D6vjnD_HLiF;V!Z&x87k9XeHjH?W@yN^B$akI+x@5jIU2oSY1{x(RI#pEQbHDMRkR7m2{IcN4X%&hIW7(=qiJ~?$3Pc|I zXgk{Wr^ZTUb<;CrQiqP9Cw{|)wg}elxEQ%H`uWRaph$K!O=GLIt6{1MQ|vwsENfXe zW^=7-1RH~|(Zw9=$nVyzc6$)z52tKe(3zp>o*{Ckx*Kd198)BwFeivZ6!RXFYh4 z4#SR%8rlcSQY&>EyFq&b^~gir{Tf$D!7xkH;j^vtg_8^q`8}nlFyMqiHZ-rN{mrmz z_lEVC8CyP5sm)c=r5w*(ax`OvY$_Tpe3wcQNB4gqo7zKKESPl1i`xaSIH!=K9-!*^E}A2r;LQBfI^!09Tx1O~tMD;%|_i zWDF6ESts!XqM+m+Qup8cJ@xfJb00Tm$<1F>l&{b@vB8V3$DaeV_|5cg!9LQRN3k_Y*8y(eZ?fM$b2$COH#hz*( zG)g;Wq)b<6Qp0f~E7_T5j6V~Worw~DL1z!bMHeK=0z8~6uOvG+ zB&*!UMxsvZfB6Jfl=J>NEb*1nX;GFxyDzUyN>O;_pN+NaIZJ28G$2t;Es$$}!|57I zqvmycS@qAABb;isE}+T0fIY@j?ETe}!?H4MCgTRG*&gwXy@7BzJyMSaPzy95{#fxn2~3K(Lf}wl^ad%kO#6He&Jw z1PWq}Qi8gEL3kGOduPs)r7jueMuSrDX|c%O@Dc+3YQ1?T8d#VZsN;|u6PYrRu^F7V zdb59Ld0ZMbc-fjbm5O;8G8d~hoPtjFhNq$ruR?^A#ON8qXadWUPxJ0gsJ)n$I`Zc2 zKTi9do?ay$BEFgFsW-eIgG38oo5_}rfY^(QSJE_YM%an#T&TR6Fc65xCCkBQ-q}5k zKnqc!f&oa{GSrGVa0l!$ku(d6BJ?q1w5ipX)|?#Ql|`ymTaW_0 zvU_T07%JjhV$~8OyS|BP+qqn)vg9vwo(PEhYUkeA;r}f1@;&?fS;6{x-^NI7#TNG| zl7Q5U{{2NII0Rg%^YJUz{o+sRHL_c3TEJjQg@pgTN*>L{ zAoiES&S5buOMxs8?ifl6p$4JpEEsvUmTXTQdV8B0@#!E}EeG%d3M*Dlb0@je<17mQ|#C$^WVcFIvIa3tG{AeVB{mJ&2zd{*xCA&Rb=hG%JWSfa=2w>A= z@%lFBygIVJw8sHtDACJ?q|t#sSCf(j0uTfJIq^B&?$%knJl*)=bVVP zdb&JRWvlH-2)s7XUFuZ&0_8e?p#gB zte4m3Kp#t6v6Q!ol%|v-zR2hfO1&csoIaZ3M?qUjfveQ8DwEggn!faegJXHeG?qH; zF2^3vTT;a^?idb3&M&3QDbDsJsQVF}7nA!tl{z&G#VEMoFrY99E$H^NxFT;cKi5S1 z-Xk!60Dw|Q39dp9X6TBFUi&+e@lRJTm|&@x3Mw`(E4j~`e)MQLls(O4rc&fwyV*2J z7R}c2h+Ui?^VIsHl4&c05xbFsfr3hC8uLL04*A2$wpG`=n=a;a8j|wrRPtlv)0)Ry zC-8haaPFaPaFvqiZi{Z-nBRkk&9r<_#h+A)jdQpmqhIG%pvS*?6Eq5ZfzP50Y%kZ! z2SZ|1Z~=^eBi#Qx9lUTXjY?Muo{Cjb_AgoSgo83r;BvDsqFs(cpYQ6WIGM};NNbbac;9_IHkDC7KR zwfBdXRi|4{g<`1H%607<>ooGTgzZk@-IuX#e}^CwCXa#l&-A`U@P$T3Om#BVDSbj& ze*?3g*_X%yE$t%n{*o-kUT=_Gpi3Gb+j`37S#ERfI&62eY5z4$m6)`-5sWD&jL@6t zt>py?nJc}Gs$R(_#K?p{ol2azTj$&8$8oB6%tngps-SM%Ea5>pX%d{>U55pk*31g% zouE@_W9mld)=yQTmWw=A0lf5j$u|3ZGGUtm0qI$3qA_OrYYacBScD9Kc+Ch4jhVo@ z943rSZS??O6L+1(jWP*XoDRZm*4Ml*ziG{gBgj*=jx*8lpoMBykknkle!e_jhre0} zj+MN}4iXipEAzaoNlSR0YYRP_n!4!i(vysqAHWM`ZErg{^%$6LS$peWZPm5BrWe9h zb6ax|n{{to0SS&*1Bx$lWw^VMTV`s`0XRbE)e0`okJQXG1%J^02i@q6iVXEJST4l% z;6-hb#~n+!nSBWEW4KwcO;oGY;zp?JCBoC(z{jggf&IE}Z&gf?(@wMmJ8*EXaC~1K zkSmZtb?QI|rSvb49rLk!TP=!fmfhJbn7!cEpx^ifuiYNz!G=YRDOQ3j1a1!fkiGyEKg!hhtQVWgl(gYhDLg(j2=rRo)2j zb&@$b?9$({+y}OCcy)e{@8svi1o$nU7yYB)5}?Y7DHSEeUM6gL6I6ggD1evG#|{q< zg=Fn7x|Qj66gU>-mX-W+c`*d=n*YY>;;xL9`{9E6%MfvrBRiwm%EXi5vN>T$s< z&GgC!#dTcyYPx2VlOQUOdDbMAo!%>H+^k?;?x^h9L4!^~>Ae3X!dC;op7K;s@ZX3% zleOE2Upiak$3sA2il=Iq>Dp-MvWc+nXAxHJ-;9{5$WF=*wzIPMd?Z{if1=&%tXOL} ztJ;^Dv#0=U-$O^lITUn@iM21wXU_GqV%qDuoE9s}R5C=pvK+yC0aBoE)Qu8(fr^%u z)k|7mj8^Cf4PQP=xak(SxfQ4+bgZU#3~fN(N@Aiz@^s+SK zMSkVfr*UCwLBs5rGh`0-U24a;2i9))yL9#M1a9RJrP&uD;%}b~3uc|yb0`pb|ALlK zNVGrr$d;A&iwr{O)?Z~pt*It|AO`+I$P?>RIv*;BSbYdyUzH|*O{T32B4LVc%lty3^2c}YN_|!f{uZV8U zWW!V^m=&0FVnp19l&{6MHF6%gzEzh(5v|)JjQ-e&9z}ARB2?yu_)x}(nGCCB1J#Z! zq4bCAyjkm&->l+?rNgtc_F)oDIU-d3w1kV@5Di4-&H37od3K2OB(xG$po|9Gk)E*M z>luY#P0@7K8b01@msC)LVfQBAtN1FrI_d#tN@)q57*MuBf_{sb5UM1IYF3(lcHk%5 z)GG&hN^tpaXU1rM)DAN*5D%r4j&~|#UTHj~`^Y%uzH+sEZ%gWtJ^P2df~bL2Q1ZQ{ZOIi0PHu9-WxnoU-62 zHPalo>ELw1biH9eJ?w*FOrpl=pHCotYBA13>8}j4!8(Gyjo#V{z>pT&p>+2PSr3*4Eq~SVkhcL~SR3W?%3a`O-DSAjv8I;u z5+M=meTiU0zMXBrY^X|haH3L#6!y==`-cQS7Gt~wXngq5K%@K70g)6-WPJyIHp7eo z$3XPw)bLs#U%_Q%eryJcmz61y41RYe{vR(uvu5G6=YF4U+e1GKhpE&4P#~qv>EoAS zyQixm*bo!48ARw`IIwC)@#uVyNW0h)&*j;2_%Rh!H#|+beYF(TtfT=VLM1_ZcyM%z z+7?B3-u$37^)>nj9rt6KKOzS{`+_W~ZC*_3j=7ey^2-$=lul}nQrhUSNP`D%p{Q@+ zCpp8}%?J(v3=(<8jl~rERx|gcovN30w<6d~YgO3)BARLyFeFW~+nDhWrBGSIwE8{E zH%%FO1vx)29<>-}E$nyf^YiWH%W!gm-(sy{2uV30+QQ95+fYUv<=aO`*wSVWY}`n? z)ly44A`ix7QO!Oq)Vbh33A>%EF)FCw#%14YmQ=*$r;l;TYjzH`>=Ga;Kn>IUEiQ zSXj;G&pPdOF-3SD@Nitc+T77lDy*ShpwL%G0YF<$0;oaeZOR04o;SZA^LNVX_@|Wh zO=!gJZelnW9U(=gZFi|p#CAN*dz}SCH z(=WV3Zfju)ni->8BSE<>bZpC>=w-FD=z2;6)qJpJ7i4c=*AI1p6gcX*9|i~wsj@RU zJG|!}#wm0^&17s42A_gGMNA6(H3^Sa?BX~guxi?(6)i!QS&}P*QJ)>=^wnmwM5q%9V7$asR=B$y#hSBl2>1c>0Pkipae8FQdy1%~E z32UwVFt>Z+nOyt|bd1WfBBXjGDnca~&qU`XloJPXxGZmH?Q}6@=%SQT;vad$?NbJ z|7t_3&oGv=Z23FpQ1rdDvN|U6E{j|#i5YG0n@wf8lBToN>z+gMtMW_yt&xQ_;1rRRVF|p;5){ClV zte;EB0x?$us6WQGHdqqD)TL{WYqJ94bFay9_oDX9C|$5Aou0H^estAv%AyDg@Ui?& z0|T!_0y^VWz(I-c(|1*(%(T^8v#Uq@a>J|`F2#|-0Z3H;kBn%ob$fCAqb`lK4J&ux zYPXRK^V2S0Q0q!SPxJ45i!gBCJHp^Kj5_{cS1YpyxMva}TH$^xSKeYZ z7Gi7^r{M(!9Z{{Gv#e$KWhSrUyFihlK>Nm<2)6I)f6u#`INC*oc(nUnTnHg;lFz@~ zag`~&g@V-i8|{|08L%rs_3=VF1v+7zy>2lyFx8JaQkGIX++3}FWjl$v>=ZU-c{6#% zLDQV-r5K~}YuS@_kq^H2Uxhy22pJ>z1MR@6CkGs)h191qS4pdU)hzlhbV`y zCDT|wgX?HMdzCbE?fo5+09r`yfOnF<2U-xyHx|)7|H<%_!mU`*KThVsueK#w_*+r#eDuuKDXeoP{r6#+ z902eWU~8qF-^5n*jWhYZn3TlAk;)PRU(pg)GS?fJE)l>sL?7*XzD%&43f-Xy1-zZ> zy{P#|@b)P`91a+{#w-EQFHMscA44~3(x#`ZcME?2ujFbviskk>{{U>3arMg!-!hj~ z+E<_b2)~OE8xzzfM4zyS2$Q2Htx7)qTwVyC`LzO_GTesUR9Dd%e!1u_*HAPm+Vr(x zG}oiYtM}@-7>AmVcGvEw4pmmBRo-{2MEnSNOU^j(>TgejxZNE!F|)+Pz-w10vb?R< z16VDlS2zv?*hl;rj&;yf5kdV)7P^(n$kN{)M;7M4h#oinw?RGw69Z@Ha7ENKE>xev zOYQ7Y%!wGR0RLAs!{a{+y*s^M){bHRg6-uF2u9L!1%ptZ}VG zgmg1xuHhr#zVGE)iotKkA@3h7an@s1sGL-cp7!^y)kY(>Vp6#b-`ejXQQ!XN@o=bK zR!dOQHb=(H@+a|Otlj&%d%?oPIkASgt$z3JTxXBD2{!cB?v{6jnKE!td#`Et8s&5NZBgy33kAWGF&fDfiFiNU&EWT?N56a%yLNLlw#NU=sEX(5Gl`_3|$( zN&iNlsKcvWB}JR~;wqGk2b1=;Y6{g~+ssQy?0v!0Qn8=*0b^`(cbiOk$us_Ot&Rs! zxsW+mPG9YMw>SI(RMf{`IAMoweShYG&Rw{#OpnY}J7imzWA?V>>?yaaV0X}}6$#5p zT2oK<4U|ImN=;fV|EvNbL{Yxub>9nSp_Nf1OSvL|v3~f6shYI)A$3-o?nD``Z6Y>1 zf0g>Uk-yQr;PCCd&jM>QHKN%0 zs@pJ7nvboxsAI;=4*HOHR>8c^IW;j)VM3z-;jbSMV2I3uf}@Mrq(-p^RWi2aRn1Ss zLtffLtuh0x%nD?9M#rXgE)j4-?k&mRwF@8@p}DQf^X3n4&mlgjaMyAKkqWL2w?4ce z&;9*~@#-J-UogaO?#1K6iR5tj=;zN>J=}HtkPHbolfsFN^}&ApHVw?T3^&e(9c!Ol zA(}ZU-Or5U%8!YTp@SKJP>VVnXOq|6u+R6WB{)Y}DH9H2sA1Jq4VK`Rt54>T0rPLl zcASzzltdNTzo=RE#;c$oGI7WLIWs%39cvewFs@I#q>(GctfyI*?AgyfWnUho#~(}8 zw?oUN9m?bO9WD{RRHb1|;r=^T`^`ZhV=8TUL@e=lstS;A^9!v?h6>q=CS)$i;h>vu zAcwo#09?Rep0ezs=0Rq}1jsI(m-=JB{f&17uYUaLFXHbv7C7zE`vUDb*Ot9k+>o9P zA?T8woA(sn)pXHPtf+gf;ZV592sV;{y)!0~8<9)Q1`J~_5sE*!gv@<=iZiowGYnvs z3Ur##DVVP>4x!W}Cho*Pzxn$hjIlbbRu>^SK5={2JgzvQLGj64_RU8HCaGw<0$=gr z9Md~vuLX|x`*eYj3bMoCCjOfWcc2Fql7y*a=^-4CHah|@L{h_!$ovRP`q$L*+^$D& z=>NFxPIlIlRhmHH_%!QQ4mUP&VN09fuu2@d*RO&A-RCX`4P#AOvq-;R6-A{3V4|Z< z>FeI4X>=S(;1c19jV#S^T+oiC4y{|dXa%6BZnn~xab6_)3iH+kPz0HAOBio)b|s;_ zQDr#mq5^BXt!okmw+bqOe*97Y?q*ipU43DnG#6_zPfct=j*JXJKLJpRVbG};#REY| zd>}q04GkE6*xPvxdn>ep23FnOXq9ATl1X8=aeG@dR7lCFZ9y7KQnGLqk{P9ay?+?W zEN8-qF#1+PLma=Les6K^o7%iu5*$)YZc3wu?KeEW_oesXb+FLl(8W7SYJYsT@7ZdN z3L?mLpl(z8&aCF9UgSO<;T5MvJQ7g*k&Jxz827B0q=Uz0J>f^)$f3ttdVM`PlhR?Wu3!xI z+S>Z5KHZC6r38wjf&ac*HC8Dz5d}?_1rX?AnI&NxgHQA?8?Vss?^XKZt#({+8llRv zCpX$4xo5s@`8g+J3k-2*%L{ts?zgiyjDT$|NX4sM+d;HJFLtEc4D`;dcId}~xlWFb zaMa2u^0hYNP>gEjS?DP@L(u#V@SP-T7cU(2i9FUOgXF#%@2VuqvQCo0U2OH;!O!jD z4uTsgJ|F`i^<;c7jtw(75biM&_Ee()znI7n@kb!_)gh^+LG`q;OF(l!dN1IEnj@W@ z1&RI)dpl^FtE7=L+`g+4TiL*sBsP!}U28{iq0GgrxfOV;C#jeu`@A^8GKomudQ{sS z-|1be31q`KbdemP+ z)>gNp24`YU!{jM+xXQ|T-anQb=di~bU+V7sG-qAkUxgY!$sO|0bL_4ah{$1CkBe8o z_ZfOh-3VbL@0*3~NNVG28?f4i6VBkpC*ikCsT-3!rafE?bZpJ#rwtoTt(=m$`fSOu z_-`PY!w0SUW}ggVQKXcNqHT7C>68V*?$!;lYy=FKn?>~XZM_=gy-W6US#}#Z%dmFG zw}ZE5O$u~Qj+g4*`YX)K+xxIm%9!lDRR0(|Ex|z;v0}UwNq%PVtqCpmZ^0YKx}iQ2 zcyQRwGuo7lwqJGdEJQrevM?@|Np+bkUU0SxFiSyqN?m}$0sT#kYg;jd=LdahuU(&J zn`GiX$IflJe?{KXe$IqZk^+Z$iI(Wu7&9C?lfzdAh?>}uT@SdyVR9@am5veQVm6>ixgo9-w? z1PdJD<1ZhsAAsMhkk1xMo^le){2EA2H`3P3d%18j&y?9O5e~M4YqqCp4h1OHH~#3& z3VSD7NL+)DZK|w3~<%=ZY-)U=l zJ%o4~Sm)^0tOHKwMoV(QPi5Lip`q53^YiDDtkSF3CZ{VHUGR4n`zGh&J3I{UPFGE( z_r1@Zb5}F8(qKkbsTerGu@m~-!`nm7tVX~rcs$|xo4G;gP7LLk73@mV1n*$VaNrE- zffFR?aAz32e5gl`C~qahM#I8tvnZO?yuu0bTjbpK#d+?N zk2QJol1EdjA#+WVh$I^;%UrkhJ1m$SGfsYz6|zYI(s?;#?ldRppJ1^;PKbBpen8Y* zJP09}tleK{)kKA($UkjkV|Bs;Klrfye!wg%zGZzYd{(q?o8N1UG8cHHaX~bnZswcH zA?Lggn`0T|i;3LTBy@X9$5=|;joZUH*Sk09Wxr_Qk0<0wpspmy-i*fm=_Ly_Z;p5M z&-Q-jMQ!t;mdiY=z5G?-2rKCZ#waN+n1Fioi#KQwwt43*10y_=w zeFh(eL-|9{PY5?FbE2!3WRY$OQZ%q*dZOri_-y|@H(iG$%w?M6C#&uwwqvDoN=hV4 zL3%~Vv-^DFG7xHZx^Wn(hDe2+;m~F~Z6m;=r!aO`j2SoVCysL;`u2Jz4e-8sIAXz{ z<9j(|)<}|}$lK6DLAy*)`ZL}ZBRVTi`eNnpTMZNx{){essVpDY*NbbvL<0rv!&nrFO2Px0WRUTmHZ=7b$NS5R15lU4hGDH9tkZ+GMBg~wF-=wB{zwBur#bXlLP%x>2r3HVxTci* zC%ugFyTy$Q8v=DN=4Qz=($@m3#=l&&pVG+yp!|P`oo3t{+kiog2P*zo_YIRz4$pPj)Q}@Hg3us`v-ZP>w!9(5hVMFv(s0S z-(Nj=12TY<*&W=Kck^@3K6KP>ZoQ<-XW2aTfq+T-1&8;q8(`&1k9^-L+sAC>c#3?O zXTt+q+6iaI}a1dYp`3CC-4Wuz2$bFGwvo_&T?47=bUYD$- zKHmAX?=B>*MC`5(3bAr}{|H{zd#$Hw0(g9C|GJWF(Mg=M#a+5EWy}>_*s!~I4MTVK zDuy7x)Ep^um{~%NA2PJ>y{Z;~E3@$Vol1CJjjj7+QC#Tv)X_ z=rS{PDnD>lh972Fz~1>cqIs!BDMM}?^nd{g;tmt0VPc`9rnWpBVk3J{E|ajm>tV@% zK9Au}fJ(@pK=)r9_34+N_NuuV708GTpSH+CR@6Fy3pxRdbG5Q{)W=yJBx?{zT=IUe z`;PN|Jq$Wc46dr{jbK`go#yD5?Qmw3VxlEclM zTa|ZBrlrH0oeyQw$@hQHCFq^OKvpBWzo+%0y`yjmL8HGB$ho{vZcRg83HeRz0ZK== z#+Px25Oc$`3~gEhH|MG9YkZe%AF~Vq1`atZ=_X4oTM)x|`rI}=ONSBLyW-@&`Szf{ zRdVcag_}32f1JU)E$&$H3r=uD=O3TWQK*wRS?;d?yWQzOVy2Q!loh0a{nB9ao<$2S zB;M^JJ1PHj%Lm>lZ_Ty+AIl~iZf0)}rI+>=?00rBY0-ObW{Z{(p}K4(WPArz#}M<)Y>+8piM&mdeg%0E_T7oTRgce513DF5&@wx ziADfJHM-}`PGcT!Z`RiY^q_N9eFhBLSvALIiL31H!|Curtlq01>ZPQoe~B* zKPFnlx(WrA{I~{)v6Z{2z4(BC^oM4jr=`5&Rk1IJxm1d&A4Q_zo{%XYIzry#f_@)+ z(w^D-6{Tk+@$Q0fXg%_E4s&qenQ}K^z%e*@KXYey+&yPH z{v%A5O^YYUZAX~xF#afHgu9xnb-vZseuO;A%Z6#KJSksL&LZouwziyzVWB2XxY#)i zT}VqF8wRTnVyX@kz}2ik-CsPw;R7a)#5#8NQX@ z$u%naLm3vHD`!5?o$zDQ)ROzo&QT6VNnkp8qQ~roL$sHTU|?|Y`wANm1mAfiUZ(CJ zPji3$Ooh}xXvmjH2M&7sc*`qsygkfI&IKFFvOpJY-GKRK!+Kr4OE#`x&vgbdl3gE2 zT5U3t6>#u>yZ|g|!7keHLk}95-(`(z5$uup0o*=*tb8?PrZS0f>JGTNcb6_-WO+}? z@AZkdjX!}g7wjA_sBs;ZT${ouT1Dj-5^X&PX~OmpqcuwllG z>7Yy)dD6+9WONndZ6eG=w_S}NDK_QinC-}DMX86n<}m~lEm;(XGQWf~@8Jh?j3_<|bg7whiX``dKXOKT1=O$>F3KOh zZA5WwHIee|=YH5d$Q(#(4p->!6+@p|RtS1*J)(5>VGM$Q?Cj5-pMZz#(-OLLULL9y zDC^r7YNfB0BTIx+bX)Gov~oc7heCsblW3XdaBtCQ?y|&rt@~MFLX!ZB+1!KwSanN- zCB00V2IkKFYZNejOVMVQN^#~eh4>I#ujb+CxE^eyczb4B~ay4c~uxPFNW{B&hTTIG^e=RDM48mq!CX{U2E zq5UdnDs!JH(bPgAyDCc#&9rN&l|3zkP-i5g!x?!A##nSYsS#zD)M zJE?*jUCr3Vf}w=U@pb;R+kF&2(virnn;sh< z+o-9+`BP3wHxenkWe0e(d>Jv?FKgud1{Fq$wit#lZ~{` zrm0SW8mAlfu|clK5OaigEeS+Xw`h(%_GROSDo_Lv%$aQKW_&vzTz zqdDH4cvf@?{1Gs5B@63;sH_kvS*(%BzE(;)vGcNt%|yS&{hq)^;91Jim;rw*l* zx@vUUxNion=awFur4ON$jBS;SmX36_2w!tal}3dTS!nllA0xFK@SM#7%(M$Bi1&5e z5bfRCh}r5>s-tcsC_~cReC>bs-*X5KVA_mx6ZxwmK_nTYo!M>wd-c#PH)8a}=c`Oo z{o`Hhy)IZBcm5greI~?yD`Hxa?jXW96mq}gAV2?9rVD%XqO#Z*)YW*nc)X_dTfca((QmB-VV(| z{r_6_7st^@n&ZK}(_!p^l5f=Wx*45A zrMv`#a^PSf_3`6gPrM|3nxqT;XiS*eu=t_yRAncn)MJ{AR~s@^l%J4<_kM%Y$B5@k zV#k+PIYsdBKuXC1d#aARAvPtwD49KP5yg|%^I`{W(%>>@Ma#d_3O$_y8uYZ*W z_E3!Drl$;4FJ%uqBKyT@iWhH2ISN!-hW#4W6uBm0J3TO7>mSh__=D2qe)tG2jBV01 zOsY`0M|~X6hG{BRQPy@bSRu9vub@o}u~Pf;o*+Qf-3YFoO-GBotf=VbZSgLnKH4BFO8abLaFfGK zN>9sRO$fykarxTFq97t|Lghm6ErM7$%f0MIDT7^|wa2j*fv3GRU&l8$P z^N(0^MjD!8fsvYIq3*6GU4?RS*F_3?p75v0$%)HfI4!Di0cbL--W%A7S#>mY-Zt1) zV>Xv6(Yv}~|9*P-$<$hm#58CMXQsA6IdJ#0CGuupdJd1TR){#D%aY1c%5Jbb9K7rr z<$B5Mv)_V99e}@jlyX*JU)J08iqa@@YW6uur+}17Zk_aZI2CKy`YQIYnUP2Cz71|^ z$+o&$ZC_558G|!p$2767He8)|0ylOv%;Kg8%7#)fy=RNUrT&S2%{(ae=5$t!1M%Q- z!LaK3?`e9PmWKu0y4#;*pOUXL2Y%~Qn*hujkDqG86KtUs;SF=~@%||YBgDQ)3|abU zeM*^@qQGL?5MpEq)-Cwe!J*Nz@UDhKut-f_D~RmaW}r)U-H}0)dH3slmzACE`>;o= zovoGBjWyObj$AVvigmSk6Gw7rhDyfKey3Y!<%!s@(Aii@pJnV zk2O&J=P#!Hu`>TO_gxZ8mYKr_uo$-)yq&7IM^l&Dk$(Aw zP?PYUk@iiU&uC2;LKT+Lm$BkNIWNr~;6piyms;yXPbd=RMAUf-70wD#jX}PaCNDS! zQCW5YL2GvW+R+NyvC3NeAMVo}M?3{?<2#ovpyKWlO-^z1tBV<*6Gx||)|efF!Ax2d zluHv-Z%}2*lNByLjavOy#NsAG73j+hk|*Bp%(V-r{D(#AHK5V%cNR>eqyMwaWnk3+ zAc)hL^55<(y92 z&jfaT&@4ol6g085@+0hhSxD?_HO}6OI#&ICmk5u=^AW0!@25{u6jN`5Wy@?&yp90P@c;ZGpy4QZRxDFAX>2eb#yX-g+Y+r%wEgrg86p=pzP{Ce%^pEC=tFM|%+7b*l zlXrpst^GO_3dbGymrr-L2*Vxl5t2`AqhW z#;@7z!$WhW4c2qZB_EbCTtFWB2UXknHPwccsk8}2uD_`>(|@MUT2)L87mud{*kDJwsOX!i8#)3OS)3bRi_AC2kd&qgG#X9taD5C=9dzmR^ zE=*ZxtPC?$ioNPm16)nXbXYbFJ>_WHf*2p_ht*V2vRo>Kni(xY{x8PPJRa)z>;EOY zN?DUFd-jaV&Lpy%vSc5Xts-JZnql}%3nBZyjUm~WkT47t*#?YZhxBeM(Z*MD|QNR83T;{!`81bH6u66*cs6hO6t4kRXX3 zGqn&CeA5(DxJ?339SGYxhKBpW_~Sc|JUos^s4?Dbw4Xs)jLjc^O$*Q+eg+gr-yc$P zYF3-o4tS-}NATS+8Nk94*`q#)BgIK6j}T3=F%V|jQScRWX4>d?aT_e*t45I{pvcg} zva4Q?N2J9w!FeOl0kpF=GkGEMJ7zDLOlL;sRCx%Yi``K1EKk3RzM zmv-NLFR;UAGcN1dzvAelf zQdWL#5pJnc%+jG&#nc)=4;Z1C(Mbz}XITPP^A?TX?AV zoj0y{Y9+6^+xd1zOexXh^pe3b^Ysm&NblcX+0I*Hn~AEgT2W z(oh$+a6t@Bh-NEO4r8|@p8dHFg$ZuFFvAOjJWSgZ1Swt8Uqix->+ z$XVHebhi;b=cvWmJ4>nWKR&rT46PEzndG;9-2;tD#a*(093eL-YFPyy=b~0=sgW%h zRh_oMt@A*x>FBmwU^4J2!CS93ErGUS&r`x^=q;|WMijr64G(J0qhz@~*w(E?MV6#r zl~yXjYd!aRxyB6R=0LuIK3+z-R!C*TvJn1oYTA}L0Z2U*`~0QEk;5N^e`#@+qTeu(2x|g84uOc zV;H#kOZD+C$LFLiwKVQOK#B8xkN}Z@z!%hPo6^);X?j%UkM@qafkF$hwp%Mcsutc~ z43y|{#G3n_M zlz0H)>s-@(w|~qpXbYVa4+E9LT8)T_GRY0O{o}(UbUDwN3vQkIcWus7)E~?^H>zIZ z0rZJTWeyZs?4Yr$rnC|4?DoK88%#V$t(=Sc=9|fl@6vTn;wHOI1d86{w4-(&Rw@$@ z>Gw6`hwT3e0YB{KFzFogc+b$;qN$bh(Q1PpwV0UP%cGaMf)BHhfIp#IDDy=mhOX~!BdC7U*dw}0r$pl^;h>sh2~B_ zG|i1Ult40|gAu<#`+#2?tjbGZP5=SnZn}p@KK{)o2{-8Yc`CQ7W3(P5;xRt^-Ip8= zN>TTtlFS3$aB<5qBfcZkM{6R)?cgF%eaRsQ`m#i6O=}}QC|!whm6@jM{>3jT$$K{? zrv2yI5oK8S6%dlgPOX+JojdQYZSG-_ufNfl3{ELEI&|!G??q;ERPhB}l_mi=jtHUb z_m7=Mm@&Ml${)7!bR08ZT~3ruUM#``b&H!$JSUJIGP z@n;TU;?wPXE-tTnxya)IZrBWo36Jpj-6GJQbey+qaRstJV{;-2)s!Vd1QaX4CwS!W zQ(O!pY1T%&dMoNK5>sXEy)9R&Mh`Sf-n1TwRx=?RKU45dNg_%r-Nw>@A}s_NhsX|* zsi+?t{?ZNod6OkTyYS}?hJT}K2GacB6I`$W)JB_4Oa7{$M6AsV+tzz>Szo7m$`yyL zkF6k!;v(ZPRHgQX6^HqR7!Ml>`u$oA9X>xXvQ3Hy{^-U<56HU7Fh&GU(l}=ujudQaD6auLTyr|D4W%1ZgN(!0xL(;i){nr*x}M6EEm4iIbr;$t()(XhJ%+?pR2ZChw752Q=or)#1V!5uKw3x5nzvFk*aM>)_PY<1JP88y z&7MLf39_Bpbek2e14uFVW0pR!uP%)cr^2)blwFGivRxB0-uqOyNjch{V|^sZ9;0{9 ze4Lxg&?wPvQ`N5+RYldObCRdKARy5uA$O-jSjTgz|9c*=@*%3$?A@Q;Q(fI=GG4jLyYZ>O_e0EUD*e z&)g}31t5OgVm#59EEJ|!*V%{$Tn5d<;STu)B`kA_(RFFQO*iX>juj+GHoJRF7UrmH zDs#}J#N*4VY|@1CqlVp%lSy0(v?U-*e#Ut{ zBp3gz7*WGn`%ZHT11kuUM++KJ)C$R-+gNBKd~)qGsVj2;)P2VC7fGhv3fD)oq|Xb( zeq9G@ZiuH%&$%+?gbGKpryiO8u@&|u`)5`^>=(pH{-Mu)t*3D`R|5OhNm+TU12;_P z(0)~fG{1p++n_Kcm`qhr*b_`Sgm%J)lpWY;s%%t7dM>;1V*+~2xP6X!2 zQ;|9z2czPMggw)#Sx5?d*}(-mpxpKGujMY%eIBF{dJ4{i3)*`UT#rdikn`+*~YB9$0GAs&Cm` zc6Wz!m6*ayy!K=`wSd&J`Szl@yID#1qqxuN&p31DHa)jIlHI#`1^TXTPb%ZYd(G~TBiEfg!+oF`1K*e0@>qxe|M!a;d3qm`h zb@wg?+&mMVrx!qgXJtXT?D|oeBEvxGf*#kM@Utl)BwNSM@jX3`FQKo}Uy+C>melWv zD<5c%-V(LbX8?N7Iki*&(lZoPMTa*6y2r0M*-XXd7(hqXr%oO{FKE+#TF;q-hlKy1m&7+s&8xo;)XUXN0M1WA=OI|r%+Kq?5REPBYluX1t`AC0f>vnyAP z0aPufh`OIMg&$%9?ixO8eDIm8r)1Sw60Z;fgv?mwY(OxzM^AuS_&W+!4i^09Ob@|tKip8`O7aKI1~Em4t0~HW8Rl% zaSQH@o}CYnD?byCIk$K>^$-`e^4Q^Q*aJs{UkBFt^|DBNY?vJt=?tLEc27I6mRNgF zN=%vM2pNFHbdo6x%k6v{mL5h3c_PGyZ~xY_;6&>+TibhbGH*<#fyU&>r^7GCxubIu8$VFC@)7MSqr!{#6%t zcuHbUj}+mR9Yaha$&K9-2M2gO7BE}uI0UFGPR=2m{9*gu6Jh(8Fl^L8tb|2jg$}$2XL` zd}68?a7s3L2K81EBr;pk?Ko5n0F>C@^>aE~abx#uVDl{M<`f<}O^E-aaA6TP|4XJt zTb;4kBWL>Ihs40c{;`>qsdB~ECoDCFY_w9!dr%du)gxNd(*%=}{Jh2$xn~$1_2!&B zQDRp8;CXt7uDzB0fA<1t`^d0qX+iDi@WU6biT4t&d?%pgnBN4JgbazwhFfe$2?tjH zXdKm-i?kqrn5&VK7O(kv_C3NPGsEtOI$ZL(sF;>aLGV3h_Xb*^1>D2fN509D2-`hH zKHxJ6wwYu51-~8$dN}O+I&=Ic%E1|4A_N{|D1G zz}AZOZGrj@^W0rHCYh_fsybq2erO(9XgQCR-Ne}NB8NcnC5HFooFEW5Vz0J1$X^XH z1$4kVewc~PHe-gHT&OU~I64Q==0u=mL0O!($D%NB0yw(+fO#cbC$`RtKEjukUS z*0ut0F2#`$uj^OZTKi_}(9{n$N<$Ap?PQ=^rxvbcf%t_~KEB!wg zXAL8QEvE{yr4CfhU2~SFmfW3xskZ1y6QA4{B5Z2|Fqnk;hAb$Mmm{ z8|AEVP}h>>->+?iUIM63r_o2{w=)A3lL>Xhmq>e0zlL%CB|i`okM2pP+NFK5maxKr zA-gAcQ(7}F5oJ4R0y*~LHd(sx8ZGH+MSPjhx|TAsI!r6yt#o`D1(hS2RJt=~O)}CW zrR*p|6lm4e_wu?IQod~b2FNb^_HIv?W|Ls{PN^!=gz-+vfNzs8iI!UZJ?HYP_p35| zE04QfDkD&bMKYwYhJ#ytzF}7b5;;?*jf#9N^9%BD?*i)6YP@$WAYEIfraMn{5;is0 z*w@C$&qJaPp{ot}%bzuQ+M5h2(vo+Q_0}Wxh)ZnQi4?*sO(F-ypOp5#1EdX4a=uIT13owT zn{oQOq4>d5etfNsPS^~`puKB*ygJXaZ2gqOm>u;NBQj3YCT)87Nb-ICo&AY=;@V+= z{At6k?YZ^%{z-L(lEj8l9VcOv=K@B644`NBN^5=J@`OAQMvoNRY~%|M7PG{%gBbSB zzXb*fqM?t!IQG`C{KwYcppU1$qJcteh0_k(v)Z@FbO7FAND>&evCf>Pyqvs`~FE2lA;~3UhQ(66+fBPx#oEFUSbWzm}YfhQ&W>MMN^pK zo=NG1RZLN$*x+G+I;pRgzM*Ov`zP@6Q?X_aenZXDm66I%HfQ zV?rw|m+6~LjW&O=_sjHkg{lt(BL%9E)w9&azxCRV15!8Wi=N@7@vG7Zs0G%iR+3cR zevjnLSiMmc(f9Lm^&E=)qjAMI{7Pn&dD*!e)SBt2q<8Dk)LLy1V-uQJnzmpT?8`h&6=Q#d&$qD%O@zrD=;Tj(pj3fQDEtC4(oiOFVV%7+s-r5n!KuHxp~bpMKsm(tk~&2Z_5) z$kJ{q%FIKZa&0&lhm3t}I6E}VI=0eZ;Mt0xYpMLnl;v|&?gz@RuKZ}syz29CqVzEX zP(NRaZ)_}l=H_$odaWHJk6AVtIvp73`;WvxLVGNAXiYs7VuRh4Wug5J3`*OZQS2tL z!I6K|)HZ*O^?q4ZlS;OWcjOaR5*OZs-tco#%?Mi&cFR2}|5W%j%DZL<`fV2(&+kOn zAGnOG`a);Z=cPzyXs;s&$y+iXSGGA8S#mIZ^>&+!sPNtUABF0hWY7S~5>Hi~IGBdB z?3UF4>)|$WAi;o~(rTJCR`v&IXl$1O%_x^P_HTCz+G)5viQtwKXwVdjH_3FZk zEmpOVl9x<3;T{_V4T4rQZdG|}Pf<=~3p zYl)Wrtq4xP4=m^@tfRMKIXPiLUdW-df19j$3SZxKvL|BpdbA?n6BgRFh^=_g@{!YbS$3C2NKB#Nb%0yB{Y-w+BS-8%`*aMt zhl{EYppEPv^ULc1>;aWKQ$c1A@P4;~ULHVm)(_Ox#;#U2BIvXla&Pc+pv2}7<)*b_ zS!k$@XiCb6lR-5Ad&^f*!jN;AfOu%_&=70jHmM9EMauTT#YiY|^*@*N^1oivKL{bF zf>{C}=&4&g`)$m$+ozLLdY^P|g?>Yd^<}CmN)gFD7Rq#5@=43awDK5vXH1G#MFwJR zISfdE+5LzQZz?<9j)+tCcKtoYYI3dus*?8F=|dDhU)*6PN4xu~rR6B4yoqwl^=%0% z$S7RPP*OSSQfil0rlWM;-j95;aJN6 zxGVs;gAD(Z_=NsF@%fTrzgLo(wk*>pD2jjJ{t56mcZ&m(axjX$DC6UlWQw^FW-=RB zYUrbzOYQ;t1PeWB0ymd$P5>y{6^eBmEc5kRoEB8X1XuD?Lnzy8ud$To7dLj-{CbG$ zCq#SUdl(4Lh`cKf`cnJ&RuvO6Z6rD{d)hY1ZykzNd>g`BqyI-)D;1>r7p=yxhkR6p zgT^9`wMml?{>HRt3`nh$fcYNT^A?=9%8e&_JxS2N0x? ztUpzAwBBObHJ-7t$43@Ae#HYfnL9?itRVbgsyY%YHHW}u9ss{6a3MrAiFHq{`h!T- z<(+PM^*)IWu}=!}xx`s7{|Nf& z^ivHEe1z!~5HA7ppPqX6tFk1Pw(sgnK#~T$+oi22)?3mE2jqw5zn;?lTlwLYpLakc z;L2x*fSO|)nr($?yDw($IqQ=}O>~>!>$}M^&m5Ad06%lLF)?8&%ZL_g{G2ypAoLsJ zS{yV$=~g7D-mhZUT@N|188kA@4k!^X7ZVZmNV0Bn9n|INz`My%jPv2tR~%zf5(Op3 zhP+o=6g zJrDqB0%tGwe=y9gl_nkP?T?8gU*anczdhaR*R;tCkn6+GA<{7GWk$4x2Wm~+nwm)N z9vc8_KESx}8=32Q()bEvvz;ZD;zZDgFYih0?*Fo%QzRZl|B*ysvQf8vkAJdJT&i9J z2Bk6=bou4K7ykfQ!ppM6k{!(e!U%4Z@(CP5ks>*zRpVbYCv}vgygEJ>S?1SB^=0r< zzlq|np`jzYxch1=5^na%)U-^0IiiXC>_H^WyS2WtyhAxfwL+8>NztSe?2L9j?r}t8 zw44EGJ`j7yQ4kitpCXA4SV|`x1Q?Up-;D{Vv^}ftl?C473QV4qy~j#}CZ(ptFU`B! zJB35VNg1&eGh^Cz#8%ebZ-}i!H~=LX=j{6CQax_`4Asz{#9ieBfqeB0edQXPttiCQ zUS0D2`7`I&A4BUD=qBw?%mtPaMv@b&qFv`Z$9s7-=Hy@e%pUi*SNb@hqY>y`7o&nY z3~<_=B(L$8*QIS#h}ky~*Y|1KL=Niq9HP0IgGv#$n$X$TJ?v$Lqyh4l|CBwDVU%&t zuFXO~MGqW*cx_y+8wo7eZF;yHcUk*>Wb_Nap-|@fnjnq|jpsSK`!c*Hn0OAn70>=d zc@F#&L=)c+P3Pn!+2e+h3*i`9QU0!uUCza2&Wh@paEb8123UyNs%rEmtSMivqVe42 z1Dv%SNyQ-7=8+01@oGhUj-%={enEGKPG;KkD2lZbqwhB`v0V2y8{-U2kRWtw?{fY@ zBj$sTLDb!p@SmJZe{8(_?>6pF(B4fd8>%AWU5!^yw~8uJV=W|%n+_uS9@|lY(HnhZ z3`p7eG0zCtv_-R9;NqB9jba~Jgs5VayF5FYJrOQmX;|O&6=!w_`q(+1EIj>MckxgA z%cKddYO}FiwL+OZ32QP6haTak8&xZwXHRbfpV+LxovCON8usG%(`R;1 zt$)Pr&MEaAn1o<8*LhXg==#!nNukV;t1#oCo3ZL{AOT?JeF9J`ZAE0~Vy~sSxMp{{imz5^Q)WG5rr0fWm(`}>(m+T3)YOY&qxUN?N=?yQuu9(U z^XINC8$dHw^v2IC7tZbsKI;U6-k!gAj$D-`0;4(Z&edzk%9~)Jv`P$&x`30>Jv6U+r1BgxZ8Ybd1oWD zBN;Gwv~&o}sz~A(>^C`>*^sT#Qj=V)ezHsUHor`~RdD_#Q+2}^d`rr3knLMy(pB8s z;A4^J$bEB`%a)qG?UXMsf&b>!ageVD8?qttdsk|E!k0U;FLoZR7j3IcDvE|Dkk7Nk zJ@LMQzdZ{BdT1`jxOUWU_1I^}^yZQ^oHXC|zJBY}TAg`Z9qDNBIxvCrZ0DnLyo_O3pk45yM2LRzn;=+r1^bTVS||W-^FR zq#x31QbqHrV;2UACJt9U)D1rSnrgfQ3`oi?EZ2i1-T6DOzFaj%un0{|HQ8k8=PPyU>az+CB2t2d~AjgYCHJ8a|8=?p0mHMO~5lQ#;(@)wv2AZq2w?w;*{biXa$&5 z(-EFUvc;w5yU228=e7u0MVp@ z-PG7En}AYT7AR1AMu4d~_hUqsHN&r_yq3u3W`?t*xhx&(wtedOb90sd>&+D|P9odl zv>@0uGlj&&M8!6JRiiw5BwCXBnJRKzCecUaVOObQM^@@pIO8U{s4^Pkv6T2S>0CJuFOT)a4v-q4`a6GD#6;2R zda?AUxd6NQf94xg>GZ2A=Yx#g{VYB*zv~pj@g96KZKxIA^kTinqSiJZ2pQVh_Kip^ zcQDtm<5AH0V?KqrCqUW*JZwX{q$#rr!B|CtQvFTB3??9 z652i7kzu*}{59ZPAYyjbnCRidXK8jv4Lh0Bg!HZOz`hMx|F!PZf#{Uhlltv>b5}P?JK$AlTWG-ckv#Q;y zQG~-{phwF^uQ)x#hVD>Pe(AkcO)}LlXy&NL`q4xAbfRa(!lWZwNI{0=B57Ce-nIsZ z7w;Gumn|!C@IC30O@l04IBO-m{ZWrhxhb4tsBA-=1^7mrsceB*sr|PFC59s+j*mM9 zS!t#10K)RQNQbAj)0dTia)LqR1(o*Oy%`93g8hnEd?QsofA>zWi7z|Nj)_i7NQ|#` zZcjaZ+J@sQVqzgPg;jVVD8(gdW4Lrl&BHCGR+SRiml-3g)vbEJaK0`ed?i zMM(B^XML{}rNrt~Uy_m+daYO6hRg%I@zQbmoyCgDHHd5JO_8HhGOi+TGY4Ed4s9c0 zh-ha2>c)mfp8rMJJ5zJ|abr;6x8OpFCffj1y*JPaJ_6G;;V^QBSr!Ta!+s2cb`_4wJw$%0m)B=b~ql6SvY~4KQRa6*DY}N9;$=~5$Ygi zOKb&5Q-AGSq>;H8LSYoQp!zz}bZ&Ruf{q~2@@_Oqsdr=VTw0OIph-R|xp37SJI;X$ zI0G=7tJyqvX8T2T)IvGzv?!=q95XzPO?yi#4W*#<(P_(x$H~r)x`pKx>hE zQSa9ziyFI4Qeu1j4RRS+*k6Gpv886Yw(Wj4ZO04_{c1Y((X{$ia+hk5OZZh5|6m)x zTfI)nrhu!i>SFR>q2U+xX=9>ZS78>?BKE)frX#Cr^y!Ds_morqP=FJHCvPi+(m6$s z3rNM`klDAQrxBR=lZbRjMxt-FTQpN`yKTe!Y^|*BZ%sI|#H?%m|_7qDj_|nSv zSg`HsISKlgBCsZ`>j4g9bd%+!XKRotY&586iw)Y`E2;`y;C9_gsylXMu3|fD-+NIF@??2b zt-0yx)dN`5Ox9MbHYar@GXQrMKH-anX=i2&ijzEO^f*)*`&%2^APp~l7TV!t!WuDL zi*j&|D)H-FB>3#-6BkaLx_9FAKPRu8VEmuI>2Stcvbo1642}&)H}Q_g`3|3boM94j zyFll9_4>EAcwW;~l5nZ@QQaCZ_0Doq!qYKgSzxg7rfO47x!B2+aV7~m{1YoIlbcY; z1dS;;IpY~hjT3F2^tsPPK*)zjTz=$WfZW-3RGOv3jcytovr2x%732_Htex0-^>KEO z>qY(B@f$kJSDzgOOWJGL(9|1Em#^N{N^=j;ewteCFi_1Rc#7{~MeVp&n(agLZ_IFH z%zW<`a66+%NTGywwiNDG2R&{8ljZYzdN)fE>ej#s4{8Lj`>mFYBEPHO;DqLi<8a`V zaQKcEg%XUOGWp1;u0L_WlqMOCTJ})HRLz%qJ-D4rr&ZVZ!n~t*4-2k!7?;It4R*dH zMy&+ZO7)DGaH~?p$~1(zsCOuu!qwYu&dX01)tq^=ws?amziDrk!y{kx0$A)U$*w}@ zkX{|t_J)Jnc(35BfcRjR>I|b<6PQ6g*zSv> z?7v%`|5~?|6KL&eTTWK^kWT4m9N;DE@r5qRc?vq$HMwZ%Lv zw3@wl9*KRJ z@(*f&Dbr2bNu^tF^djg`8y^)Rw2*+TE!HxoRifBNSFri&)*kGe_R$yH5&6Zmw?ZrI zH46|i(&FA7wc8w2-8lI-MX%^kU5cbdd)W{l1h@Le=jxspV<`jvo>Oa2&pD^{wqdWo zlO(-F%vVg}L^LRebcdNJ$Q><;5&R&#;4cD4bbaN1d{8 zQ!fFkqq02dQAj^CJpJK~3dre#dZHq0@6boOEa6k>a1l=GV7Rz3c58L~(6^8G9QAcb zq5dVNzpjHT@T4&ZR-JYA&&R?syiB6AtQv-nB;jEX_k|T>S6|1)m@CL{KmWTIpovpF zH}!*5rI$T6;i`wlYo9oIAbQGrVspacz1>}4c9X+Mk|h1S^J{ao zBesJ3GAV09u0}--tzFpU)b!Aq6L_`ya-^(zj!cSrKY^e!edeENoy#O6nazH06Xim- zTUJ8I>H{y45^<7#5JQszXLRaHUKgdVz4|+Yx|2&TpyiTioRqld9(=`4M6M5kHbq32 zXanc_OeTjgZnjnTSuYa`w+0$qo^N+bOEnxVY0_Cvgf}O#z;PZw`!g@V zqs4+U&jAEG`sU1kd2apJ&H#RLh2!*GEhBCdITGP{uYj8>-ZR-neoYW%*!MoDj6UI5 zTQXE#0amof@>ZQyuVq9|jMaVd|30tVlkF1!w()je4c-k@=B|^>dF7hH=i4J?oeX}r z#QTckvpQiE|9uHwDg_vq0ZGToPhTzb4EG zHjmQSGIy`4eG`!lj?tj(F>Bp@)(fLUm0v<@TX|w331FQHagywS;Jkv!)o`#RX%X{< z4W54Ca{N!O|CP;hT_UN4V@8FCxfvdhT_YW=n%NYSguCCB4B>eOt)ejP7pXVUO|s@# ze&{JJhcwaiGC3PQ7XW!hcyFX0hb<`W*0O)YazKJWbm5ivOD1Wip8e|66D~$wR2JIH z77>z!{DgF&J}9ueY0X!IZ1t$84UM+}Pz^~r-~wzoDDJ^0IXW%z{&(-yH#`XN=)jBY zIjR&tVF^;ge1ixJ3ax-UBJ?7bZqGQMnpAY!+iIE~d#d(6;(-CSy5oCh%z5fgovA(> z?T6&;_l*mXZ_bpL6fsU}*iAYoxxT$?B#o$_&y_aJIF}}_ue{({`?#j&IXaa!$DAg~0hojmoIO?}Vx!tL|aArUUB(~SC;KI`xw*d<#^CGV=g{cNimJrbx)l}nTM z!q4D`gNsa;ukY^L$83p4JqTRPl=XP18)};GU-X*x-B&#LS2kcP(WWLcq=0YCv=&`- zum}{JaG9%IYI<8zOwRIJq4VRrvRm%~y&dOiV+m5ydmQKBqZXbR0`H2^X^?O43mKoi z8jAl2NBGvgFqUJ)l;I`PkT~+gGJRB}X>J8k*q`scoA!wPm|S<}R#;ZHd?fw;>kH`M zVm&&o?7%~-+gQaSHn_U*Y+Gdc6DDMVDmqwHykjxFJ-A7|R*-}8$6R9n(-D_=6Pr>Q z)esGq3jb=tT~G%9<-58c(fmN|!QuC-bXhw|M^}hDAv9T%^?02M7XuEdFT-MdUEyPL znQ8vUgOQ}ROj9wI&bia8F{b%<-m3;qxUVNVV#$N_`!g5N+I~^}KvbWl&%7U7lC7(X8;_@h_Y${%4Hz z-)GGUM&KtG_!z4m&z5h6Z3F)tF!c)f>!J1_l5khrhf91G{8nG83VXXpp9GD%7)BI! z!35FT+_Iz$3-pxGW3-TBpEUzA_VwkoVOA?0%kc&5Kb5kEYH7C~s9hd-2x6GdM--th ziCghF9H3&-Jj;D`eA=zvg_3VZ^HO_gbf`;(tw4|D1N0gk^)$Lu2jNy*F?PL2DUF9Z z6Lif)nnd9bES*{j4xIEzJ6+)GvR>H$GhFn~@+?jeR)DU$SePxc`oo?n%x zsK4|%b9TC&7c?lI%3C6#ivsg0{#aP+^aB2g{HGm;-o42tw=;!Ks*0r1Yhk^on!&r# zEqt{w5j50myB=*!lg7)yZLrnl%wGeGrTS2Wr?;8XhMT}racZ7AsI%YOB9DZNEh@SX z#=Z<}a(?W^ANxLGrv1<%iN*rupFc12#HoL|IX8KzM#UsEL%2W71rF*---~ath7w9Q zITH04YXavPR2j;0WRQaK8saH0ZXA5E(P9iS-+bkh(>Qh>7`9-64*mq}iD?-)Kv_+q zTB18Sxdx}w#U;N9l|SxtVpA(FPESw_Y)BdRoUhSVy{S5HP%^s_Y8-g%n6EVax8N{8#=WAR7ReqfOi+U3-O}GWHlhg2u3= z8_tG5SA6JqGH`1D>%4tG=E3WRXQf1EcLL2EHPqDBXF<=zQ}NR}VF34igAV_wtXI%) zH}XZR2%fj{zJaHuStcRbD>ifM3rxox+n#j*FTe8*)O=Rc$%m5RxpP5n^85%G< zyPq74T?U32QlF=Rr`_pL3SilanM1ep19(?POGSbXrOqXv z;X94E7I0aAc?WLY%N1E2xLVoAP2KLwg~*k(`FWW9LW0l=!qxd!*mi~*$(Z)`V~yp4Fk4kQMs_8w9`_;vcYAv z!^rlVoLRQF+fNQF>HU`jPyJ6PGMD3POclpuknAq-avM0D0%A)bL}nFW+A`yCD@&Eb$=jJ z=dQwtcsxp%u*z=zee=a*myN{0)5-&)8y)^A?5ADFX#Vq0df)AgS|x3LUBRJ*?%cjs z)KwE|-7d*NU48kL{4V-bIXJu7H1Kq|;L`KYHlGi^&8h`{{pRR0Bt@{(!(Y#Ms5jdS zalG$zMRzvND%0RC-$r{|c=;BN^QNtp$3j`uno-X4_v@7~GlAdrI=o9(=QSyA92m){ zH_1C#7{)(^`PrrWd-+PYQz+Z5KQ!^C6$Qyus||SPriMvQ{RIOD1 zUSW6lI4DbIy3oGzqAjPB)Yl4PbHMMhmW`oS(0%JXFvR|K`EZw$Kj?Lp9wqfsoQBTB znO>P*Yd!lb?j@+&EM=4#cO&RLr$qWU&G$1B3O0RD)`#k1bj}v%dn6cL4=msmh|H1k zXemvSHe4IfV<>(sm0uL4;Y8@hhpo-Menn|^w_z3hv4`S4l{l{7^b4}s^kEc5SuA5JJ}KaOw<9F=)OLRzPP}^mGq14lyYO`3 z&MNFJG)5APrJMX2HXuG8M=($Pe)Afg+T~=ll+Jw77^SziY1`-Z+LllA&i38()r0Bd zrkbk2X;PJnrRO8zE#9W*o-+qg|1#}x&*Mt0d2U9Dv899XwF|VcCr#>R`sugA3||W=5gj^yb?8-Tc^pINB5{`K;>7)Ld z`+Mi8p*pFOqnAjLM{aa?LIqA|fJ45VQL%sY>9`5rIi?Or%pf3~=cwOePSUCUD-Q*u z(|ROHD2>i0Dj(cX2ellS~T9F2G=_9fCgv~4=7 z>4d*ezx~yqZ7d@a($K?=lhnX-(0mx^p>1Q z*q@YOwRcHPS&+MqlPX)3F4nejb-FRO6k#vc|EaLUzPq3l1X1R>Rn8~vhQ#cinP0|> zAMbB!7O9dG7C9LrX}N2Me+>yXG;N$5KLmO@iQoe3hFD0T|rWy>9>>GGfWvUu? z_Z_1IEPeQRVZJ_&g<)d%C0$l;-}WA1h2>ONrl;AtPLr{-Xjc`UjdsB-lWH>i-bkU7 zVn0-BaethGDg z@_saUV^8JnpzTd8uMQ*g+`md9ZYEQ4Az+)mSp**U@Y-Xol1B#9#n!m@#WGiAdn7gm z=(IS)q1w>R1WJBPMsc@MuH{vkNY;a_TbUjzi(>V+iJ)6XsZzqis}rt4J%+q@$h%cm zSyIb}%H=dWuw`DDE3U{hTk{8o9qs+==Ck-SKvY8go-*mjM)M9C!X1oHf9m(DKyG^) zcxQ@`B85-U`<-C?|BrGUa7!|#1<8;i&c9m<34$otz-53AC|!uGD7U!tN;=7oFrV9s zV~LJX)wwzneNX+^n-yVQo}%(2Mr~Q?ws3|XIgzeh1WDdTvLZ0Gsx~Q$oJHE~gC-hE zbbpS@6>)xg_29lQiVgGJ)TrUc?QM;<5B*;2LF5JlzqfyRu%**^TvTasBs4^h-6l23 z23}lYSt|A=^fsoQnFg{-au*xjnmP{Jv3vjHuD$>d(8zqwEoJ_TvFo9NM7}HTtH0#V zSZ-j{2IgJULeNFuaw+0V&V_z6EWtcamO$$WKW{h5pGp27ORUDA&hn%AD5JTi-z#Ki z0}{RROZVTeuwR+3ioL)4Csrr3f0jC0GqN{BXOvk9 zM**1#)?Kz)7nj_}&G%K|4JTUXpm#6^Ccb#$DR)<-Jlc>q=26qE6uT9c|0p)g^(DAn zQT>-&Z4ljIiwdgMxEi%x;#|$JbvZ_!9=YN2&%bnh;28AGd20049tdbbR;9=KnqDvBifT^wje~&?d1R15G*uu^M)DVlB0csS|0oJ7z8{1#qN-&#SdHRY@bnoD>atV z;p?u93#rQHKpB03p_V_c9PjJ1{CsNjs>sTOBh@ix(sZcab|O8wrn40ZRnoMcSBr^^ z6BuGU9i?O>*IWJjw~KK;*&@2A^C0m^i<55himlOgAgaoJ>HLf!Msj}I5(PZg`0-x$ zu&$54!CUFSy{VH{SYE|mEtVoV)R_*rU?nu~stxyiek-EtkYJo%z&(Rtru{00amS@L zt3oc5w<1c#QHy#AWR7?6j~mzHCk6q~^f$zBk1!?9q{Um3My@(6zz)4)`2U_rgp>`R z+#VK-(R?5=*>rrk?y!7tb|rP=6<9I&^tq0c|KeI&sQk}0l7Vx+RLW9<+8Y=wX_Bu4 z*28ilUk_RN!_fxIpGdhW=4T=r%c-w@kRH6c@=A^Gh5y=s9OU*{U!h~WuOCen2UK;j z{Gn37ISZw+#9Y34HiCz0qqSc-e3M+#Dq6nm0Wik>4mcl zmCuewM?&>O6mm6luB1?KE7`E((6-tjPH`ZbXo&}``~Oh()p1SlZU09EB@`48R7z<{ z5s(g*Zlour2#A!l3-vc9ldG?ILxX0C7Pt_l!%qp`0Y9PViv@cv`Q=0W&;Rw!Wxn6Ww=MW9 z>!n;Ot$^obf40A-ig+ia$NjD{Zm|W%yk)2FS-%jA(ff{fhi%G({wTDXyD?H3H?O%` zs~V|I>q@w2dP0{ZQN|zX!)y0dunBmb{DKBx?;Ei3*URH6_S?R-j0D>tfTLm$+I@@@ zi-uUv>pqwA258f^85iDqv+|i`Jm83vgm+G%| zF1`X_Ny%+qq6Y5oIHZSBL44o9o+tU%8 z=wcPKq{S%8483cL$*}ic!^>U_oAPVUV4w5q$!i(*@yixego@|~)pIh|o1`_||MP5^ z%NZ!ZMU~G`{urqeiAhn#lXuMo)2GURZO><~Y3#Qzdv*~HAK5YshYW_=f7bGO%by`( z7K06Kt=;>4p0iexzlTc|;cx{3+Ny1oNBh%@X?~fCh-1}{56AL132_XU7hGlO%4)wz zrqBY_2TV3Iq=M5oMVW8oAfYU9SgW#O(Z`)%-cj{$B<9B8G9hi=-vM7{^xef(lOEy> zdfbD>+I@uQ=1uy3{Lr3|EK>wZu4(yTpg)lUaX`$i(oj%Yx*vXjEIb8Cr_cU2%N_GPv~tnIfq=B}2)2tPkCGX_ZY*_dRK#bW*ji$;yQlu*zwQ{+YlvW^Mb38-by`F-84hJnB}6sH4*Zd zuU*gitrE4Nun?fvbPj(?>x-({W)meR)b4B&s4Q^Hx%iR$Q3Xp4%C~OqB;#->58vKr z7tJmQwNR;BZhG9XxOy{g#Dmf&L1%HDm4c2=JXAj1MF(i}u9>m!jm0Wb?HN ze^Xd+G!5X~gA@xw{5^XbynQYAQS~ZPqJ63GM)=*}?^3ZpV{Y(q59c{fbT+5aT}N|u zF+=+Eg{V1zHW@?w)F8F@*0PsnT`UPI6!_eGh8!{ThBj5XX({^T`~EJUVgYN@L1K-4 z1&8~~xzyrEO+@>WXH`tp3jWjE2kxWOnBM#7-Jq&Pa5XvKEto@{_fnkjaPUkrvfRdt zkzzk}`Y_R0W`DNCrh+#QSBr62YfBvQ*V^cIJjvQVs6S9x8=T$i3OFQX0QJSn2RHhVU&|)*`xF=X z`gVMxYiHA*ikXb>fLMIrDyUXYNJ)3xbIk~&$FhX;L1rzvk$7`Och{}5DhCJI*x5j} zOGI`~Xs=Qm1D0?Tx~fGKA_u!H0M&_&o%5t%x9y4NZ&0_ZagQ0WO1t0a;M-flg8ro*Cs$k*NcBG;MpVKvvz(QUV=2<6}n?I0c# z#m4>Nvhg`5!*R#t&r{J;7UiQ(W4DBei8Lj z@JITOTuUosbC3^m9B=hy>=PYxI=@LZ?FknRTL?u+QF>Dnv&J5rQVxcu-{|Y&L)sPf zUexfC{Ip8t98abIN(@*lt8yPKBtsQzVA}I##n&1|HSLC}=mzwDJ6F^nPLDl6wBvgz zIAM8tppmxOTQ(-r@IRs%FpTiddSP^Q@r5_iM{@7geEL(8sgy$6+8OZopwoANmyVq? zvN!9o0c0c`!Y)OIWc76cT}E}Fl#T~oLH>0+(+9<&1)!VEjdW;Y`@~$}CUM+&P1AK4 zy*Z-vC2ZnZ1Mx8<*6d_RasoxR`zr)94Yt?+BxO$8BVPXsxF0P_}#O=rx#<*1MWkPcR3y9n1r^PSHyETYCCXQ0!S zli%v1frqNXo4wxT1c9JY%Y>}51CqswxYiA5|KrR!02a_&4ZQp zzitp**=k$D=sB;`_{Qi+8h*oTdg+txqAD$L+T^k{nA*qC>1x?XlT@An$o99rdsa;Rd;^-41r*@^ zQMF(Bv$Cqpc>vbX{|KbDi1@k|b^ZB@Uatx1;0>{fgS~B~Kr;dVprrwxA|Vz$4c=({ zAJJLfk4ebRQD&c4Jl!uPx~6iP5*Y#mV23eVI-nY(;VR*c#9j*;ZmrLphy_X&z4zyN zRCN;B)mC zG^a6RC7ifE+5{aYf>_GQAom|+WN97ayt*WjL-65{$U3}hzFqw)e|0VBBZbJDmI?y2<$`5cE25%GQZgeByJ9u zPBXOcLZL#1NF31M&sPp~au7z2S3L^k`Ies5l(B0l6>`(j5M9ma*|W5;twtjghlb8R zt1-eK=|uNhUFsT(h5BkNp+Vo6l8$}q9>taitgdHzPE$2@f>bj{1yC`@xzx^`XG}7~ zbG!zxU}fo1@%@HMdCxsN`DhC7VixwMSHE zViIwlLe_`%MWgS$%u&XsTC|vOpFM-h(_u`>M1_9l;LDDA849s(6&aJE!JQ#s_5pX_ zI0*ZlxZiB=tM@AY#xo@#ly6KVIoaE$wj@i=Cjlf&Fg-mES+M+Qyqei2pH)#hxlgiG z9M(BNN}I3mQMxLwSFR~}_bXuN+N}uRAEE#d=a#y+hi+4>}E4Ol98>U-)p&e@;kA2Tgo1RfXn&$dE~g6HedCeJwGf3?MRR3cH3)@`MJPh zR3@Uv8g7$Wnb}KTm+Js=_OD#_1|8HA(g*zAHj1oY6bzpe`KsKP|WL|*dm;UspxH%$~wB&-zb|cLeeVbXw z6bl=Ir6BYARp^lK-YK1gfVGCv{h^+Wa%*B-(m3urAuIK}JY>zJEU3ii0Dt20dHk7` zuaR?B>Lo0EF>MP7;w6X06c+X~L{X8~$k@mlo}IgVN@{Fte)UHlq^@AuYh*U1FhBV# zK$4)QkcInUiayv^$cVUHyJk)YQq3Zer9aKqJMdG0vVS-Z+S7b8cyblvFL4dvv{WWy zH37O#Vd3J$au<8M8Hwx=DW90Byw+zRCqy`{BO=eg#!EyKbZ`$C%V70!Ozl};$&a#eidXZ$dcBAiC?~TAS0SnpnwQd zL8@nl?7)rC_(>ZOjojhaL=-1vj9eBSG|e*jf;gEvJsVm$QNp_GjDkf9WtN11gDUSE#=H+IN=8Kj4c zB!i?vl8|6p^g$!tGt8IDFCV%nHr!9P*WjZAr9+dTwR>dp9csh#PM|^~6IrTVZkCc^ z>ni0FY`db{=ke+FHmRiTTxuwQx=Xx1Ed#iHovZ;!P^!UZphpejdc^dxsEA581846z z$Khld^v+u`bH9|44BG0U-0k6d+Z+MsMu@~wMwud?Q+L;65YB=-E%{B(-F5D>0H3s2 z!Asp`PtT-A=;cj(z3DY_#7oZKOwuYLJTpS6jLLTOJbg|=8ZRcxbC@My762bKs)qe2 zjOeiqLQj4Vqj4 z8ML(ItDPPm-%8fN&ef+njLBCPJ)R~Zcz=9!AlaV>@jxWGgIRYB+6LiS6?gobKI~D% za32B4Q5M`LotC|xCw9hE&Ej6*GX)YZ=y^c6`4GP>d z|0eLrz6lUFoD0bJ?|YRABcF>CfFFT?`N6AL0c3JLu4i?Jir6|1DfP}Nd}7ghw?iE? zIexj4IU(zD_~Irtak!Uh6nyYf#U3K~Fq7jtdRupuGF_jwl`^YjQtein0i)oy%Mxe-*Fv653e)a-c+U4Sv`1Cd~Ju_hnpF zaFtT;$Bar^qF#vCd8qX(XZc@&sLTQ&R4?v_=w_ab?ld|ct%oQXM+NV8n&EwNeeU~Z zQTj+Y9dEamYk$k4N~ z9&cIF+uZ7wKL|7zT%X`D+s|I3VAv8e#nA47zQ)&Q!;uZkql1c2w{R8Sv`K)KqvtGr zo(Bq)}}Y%=jf=tuVe6^zjn~iN`HCzHbvRy z2^k6%vyh7F2^e*bEftn~5%g^v-XQYo8YUmh9RQH)0BT3D%awDn2-RB*tbq<{#jUwK^w zkk=tRuvicL+(C<9BR~yOJPy@&s?jRDE1S%Iz^CL>IihF?N9Z~G7aotGG`v$kV1)8{ z>Q;T120l+wfG6;Yj~dN)WlVUXr7Ys`#O)x+On40wO#fG)_kTDz*$+E}f&&U~gJgHAtW!g7nG-<>_y;OlFp2l!M# zi@)7Ne|DG9UK;P3h1$*Q*y*p&N}zgV4Hzn7ep<|JOgr4I=1P*n2pZOCYW0CtdyucM zfC5oQQJbY~W(RctwiFL4_FqM~h-TCzF|TvjeWh+dPp@})pfWW@01iPY1GbWi0<3EY zKu%VF#+SxbBuBXWT@%2eLI}GSqdi>H-eh+Xa?li# zI?Yiyz`+YWuF42rG?Rv&|wHxHDZi%Hc4!Tmn@ndNiWi6`#m@R_i|p4ND> zZ?czqyh1GRX_zc-!kdro#782(>WY|>EgWkfu*y+0|N zUBYZ?((3v^_0Sm4?1v_LsLM~Iv&Ei7XCF?sbB{$^O^{jX!UV-cXVPSO*BKr>lOgla zBO_}_c$jIXkttVp1>k!vd(|q_Edt=^F76*nNeVq*+$WVzXXNir(u}mwDHSKE(>GeA z^GDY$hEB}w$0Z%5Y~qZr)UNvOpwPX<*_fva+mmi}3Zs&JgKk5$Kf7y-w1QjS{xuAK zf$ppTSwor$w#q@0A##Wb)>tzxx6o-67yYDTqI%mWy6)8%^uz~hfbk@GBW0Le7Ow0^ zzVBazGM^#$xIL>128MxY6ZPzR6huJnmLqO3@+ytYq7JXUF_lQO9A(GLhLL&|sd)D? zxJ^nx;h4kbly$U~r~DmgDF`AGj{%!ttzk@#JCnd$z+rXq_s_}1dUTWQW^|4G{R}#H zr#JY|m5NS!!u#WM0)uQj@I$hrS6}wisJs*9c>5AYmDDM~RHaA%nN!YM1|27gK3mwB z4p{42A=^`53o6_GT3X`nfN0zgx8AJicBcgRXqxpH6X-F?Z;te292W!d*f+W$TQ*tG z;vp%szo|*}GGUi^5L#FEi61JXq~goD>om$l5ly89S#)*W<7G5D3$zCHEi~1^A7IUt9OM z4Jgk#F(k6p2%v);p%@&5S<^&&?wn|7MET5Yy^BP z*(Vs@jCl!Q3Gn>=smoYMxib3mjlS|@|5HzBdcB$13A4Gfr*7{VTaSSSRBg}UY?>b6 zwv>Dbj-Cv|7G-Z?#w!4lgFYOoS=rtn5>e{eZdik$LFnKwRRG@b*G0Sa+c73Aehej3 zb5B=_;dlMMk8Y-v0OPv?gdTc}`q3g@#Y~jMOJ7iQD1iLImz+(G;A>#3)fjGZ39deH zbiGx8PZ#18LB$c_tE6hPTQ~$aXI!dxHyorxQu(+LggOoIXBblLauklCLgS`u> z!8^ijqVM?hGlr+5(Ls2e_`>4$>()PG$Ex~c$$WviIEEa=@U`SE}ZvQ%&7&j!!+{qk1EM8Yo>SN-Di$Ql4B z-Umdb58|Ba^_?h!Z)b%u&y8AYXrunYx*(;bB$E#rM8f)8Y z^o_4|uZinkdcj`fP~u%3I6LK!BKVqo)p?RikEvI7oK4tdhg*XUZmikJuda%>!uGm~ z_uAOyUd7+K%dFqdP9xgqtdq$Xhml1w9~K)q`&efV=7gMmoe7NR3I41A;~vje*{{k{ zrW1YCS0N|0e)xV~W(vhGevxa?mE$WiSdz?#>h*hWSMWQ{g&${-!9}()v44F&pT#f~N;~h6d#akMQYfVhsa!Y$?19q~pFbX?iM}%1!o8^t%-i ztn()wNPUIVm%a{})gLBLOw?8BH6hWdpq!C_Y=QotV*d z>{GIHvg&uDjoZ?E2ZUWH0s_l3-jv<0w;{`f1XKyQh1?8`f^W~1LW59C6y1#Yx{*It@yo# zI}mW8up3J0J&Dm0=A1A>ARg=3jyf<&%==e#aB+?lv^JHh*km-xSG2t1&sSAY`t*vI z>pi$pK*ULFNcc|Ou_&{q6#&T$O6#vH^1;xeE>hTf+0&n!l9rIzL0_PIE;}`7iB5(6@PP?j|P^hkScaUkR zs$p}utV4ME1WLCs@MQXfa-Ou;ssQfWd5$W8tm40RpzTHY8~?w;A6kkUXh^L1Tvw=} ze=A3vaglevJhX3|?$cGeD5LLRlNpzt9t2(BZLLk)%b`cj+;ikq736G=7xumy+u>!L z!3<$jCn5F4zmqa^y=TB>D9Glyg9(qSHurvTMVqI;<>8v!+k4cQxi!5r9gra@X3~|G zT;c8xnVh7#f@K_)KL+92m5x0M=U$>%g7O%p9RKDjdM{yHZVvQlW1r->@7rl^2BkDx zN>axy0aHs^{9!6tQq*ZTvVOd&mRCR6p8e-Z#{7?yBzAf+!@c&4G4i5Y=KqYUwi-DO z`*dq>8*N7xE{8K=8Nc6~z*9&~neSe_O2)?VTi9Pfcgbgh6lT5Y6+~m%&B_InqL29> zhNUv#c%#(w1}sP3DLq+XEF9{g-uh}68i(0f*{qLFjl-sBGk=U-y3TOvNzwK&s3rgh zoFVE@*xvXdDG^)OB-J~jkvM>`m7Oq{s&te*WViPLmr<&asQ+~vVfpdafhiMIZ)4N! z(6rF*Q`9+q#OnTi_qzbc?#pXBxl#NGO1qNILbFHSQ0r1txq2JYh^c+tYF!6tBMc!b~0 zo&n3qaWvL$#5M@j6KvC^A=NP~66CUqjEvvdPGg8h+&Sx{3Qj5XzH+UovbVbQWkkXA z-p7!2?DLfmuWTwM58iy!;2DBvyXmEcyzvK(3<}%Ws^u5yntV7lB!WB%HtyMV|4oTqdpWc2c)EN)UgI@|g) z*qC-0ma37~u0^;y9r+%pi;W;T=4Dpz?Xr@cYx6b#H%zNbAy$$K^JYcwS1a{d^VI<` z6N|V#y3-u0XXxB(VYAn1X2dq>*GRA`YO3HHr2(d49!sg8o`lMUpJCus2(^{q9~3Wj zyUV<{R_X5qOEQ-7lYipj7Ud-#i0P`Tkd8JduqFV9Rrl+#x_)^=h7>>>%)0Pf`Ue#F zDQl1ICs`hc(RHhxmRIv0=dMf^Sp)s})ZSsHj{OKiuY0bnB3Q9O&l)fz-hq@q!X-~z z(Ad(2kA+&yH4S=PHd{P!iw)P@NPkcVEZw&EZg+K<{%EcOo;+59)5@>n{a#?tWs>_Yc*h4 zyOD~br@~D6Q{LXnrTQ2;7V15tA1uJ#Y45rKcc&&z3LO4CF60{Kc$ukW>KNBFU7+wz zG`Q8>DSBoC(Rkb8-Y^dFsGm7Xh*KZxW^+i(UOg(IU)i6sTvZJosf8+w{uPiQmvjbr z+oS`uXzhW*B!9-479*SmKeAin3)KWLf6$>|FjgD}#i%9yg#+|T#oc6QvcT$bp z;M(}Vyp)wf6RDtl*hn-BvfA5{qtPbu$AD!4WEba$Vh?a{ac+RM?RaY=uXZof$nwZ& z)SOO%7?a=-a3D9G?BoVu3>b#fDmnmHl|NUup{lw3%|RY-5}~xylmRypyZLeUZEf1x zwsC67XhvA^>-?#09Fu^?=+hQY&Wx6`5Z3S(Yz;3Z!DL&C3)rKX@9f) zTXVChr)#&E7i43uAtE$c1U3V(f`^nO0x7-8c zfy{m*I@raEu@>Pi)Y-PzUzg*8aih%>25()RV{QgNLX z!n?!m1hzmiJMw5ta4zEoK0XBWi!GI^R5IGB^cCfY241~wX^+4&kb`|APG`sbcGliD zv;Im?3+s4?r?{Q--7tr*hc0wBtA7$L6Po|!hrN&FGZek<$sv^>Jq(;S@o&ENZa3}( z_wmb-;B{@2w?P|*X3lF!lwYtMz%oPZgv_K$u;`UMM)oqT->5_{yg3&vUr|8lo{UGUQNpizzn^?guYseGA&n-ZaN zlu4x6Ne;YX>{N&xq&&Jq;{02J_u-f-!9aC?JvyHfAqZAo#7)WjWPeu#m1_6m;Re^?IZ8{pYK@g zqSyNR=O)^BLe>KPcr6vS^UC_Clj8G`)HXO4^e%JB=pMk*O6U;dDY@T-ubzz5B<38f zkJL0M3?C#KW*Ql=_@dm-e~X;><|sD9DPXwa6Ga($+DfqYDQK2eClpmin3yDC1ODGG9tnTLB zfIaQ;s*ZiBA24MPWE76QR`v?JKVJg{B)|f2*Brf7M)Dy~E57|_vxBmI&Yv39Qhza9 zIWKV#!D5m|DBEa#)RePzh(vF`?bh!G+~>FnN>hq{Y3<~XkYbyci(WYOAMLg)=_Qnr=;gI5g5=;J%hUUb{zGsQVmtnMCJ|)m0^bTa zEgkDNT1Bsvn=&`bMQliDYvGhjL4D(MUuHAxr62U20*;T(ZIW^fwg~X=6~My ze{5QSEB2S-vMst^rI2m4Ib5)D0{RrwDlv*%PRKq**z_jIjqh194QDmG&J$ z$fJpqz4j^nfje5>pzQRC?o|2S$Mjf6_uzIpqlNWPA?y{U#|6%JCY30OPrm+lJA07+ z&z+f5fE^!qlwya+VE|zb5Ku{+y3&5^2q9=r%EshJC>XGgc_0SVyqYsCwOF155bBNo zXo$Rt2*=BxXK(-=DtU#49PII5-rc93f8Jd&a4&1e#~fALOW7O;;y5ugCA|alYFZoX zu@0?)eIu#hHUUpk%mEwU07Nb$t@Ke%;7oXd=RP_~#De8K9Sd3t$kG4X4X6Hd!yhU3 z%Z?`mfzJZP#G2_X^|gL*uazIlYL`lT)?;`N-iaMR5$VKJK$o6QoKe; z)VPAp0}>^BxokxYVFPqtWfc8)x`X|f?o=wH>&orz8;*KYgUi+#?tZ-s9^p(Em@a?> zvbT!^XClnCzIJjPY_zZsFi@PDq6#3dfMA&BwEW>e^ujjtj`uHPWdPy;o6i9i;+VMn znqk7Oc~Xq(<>eorN%9B_h|MymiPJ_$K;Q)#RG)D z*k%IqiM?x+PnWQbPU~1|@LEaRHb!$)!Q{a5v;D=2-oXXF(!S|3OvBH&Z=*&3EAbw` z3@lCb+YLa&jV>6Z`oy70%~h_K^&V~m<≤p4Jm_fq9si_vgcHFQ1l*{9nDXbvX~f z0S)A@Pj8L^;-u3iQ~h=}p&2OaS--mXUunv}TE3uKGo&3nKK!n-&%b~5^nQ-N_@do| z#Iuydn^)#zR$4RU{~M9{$8L?)1b{&J0e4qB`I!RiL(MmT5u=^*U1fW2K7#Dkji>$Z+%n1iK(%fGM+A$PKtB55cyl=X-If3TZ-Lo4KqrbKeOD@(}xGxx~P!${9XbHN_6BJzBv%C^!P7g?9)Ue zUpbSmMtYQCa*hE$6bm;{SCtj%{(AfMX}|lQx<@*-^2=Rk1o|WwYBinQ?tgIP8wD1i zDD(fhfHG;(yxaPsLx!r#Iu#tTm}WjPJK*}EjHRoA%wt%SGVQ4?2MYfHOf zAM;sg2zL~cHQHOx{mm)zGoQ6*Z0?wLuO~m#5yg68R0a{xC&sAo*ntnj4%{BJ{ATAF z(AT^;%u-HmBcr+`njs>ptFdziy~@mB%*^!@*#XPX%qM=7TB zq_)1fd@8FvO{OcPR;c~cA-#N48gq{{={W4@CQU5x%RRQd+Mu>_i z%j84nhk%ey723(4&x5I?+9NBO6YetZCcu<^^hgJB6A05Bd3d4R~akk?X790_y|bMZc7oZqwYg2-d|8n?0}O z)|@o&(cj*C2OfXJZ~!I02eiIv%2L3K!r%SrA1ktiSzQ*p75S{7lyguw)Y~|V8L9a1 z8~ksoi4<`~GJ=Jd| zcL&^$TE8CiW-V(?_18^An$!AzP#%rPIDN0xNm1|Gw7RBRWsub2Eg1`|zTWVR_iqf} zy9B7j{W9*^{5MbPjqfxPUy0jAU10ayC~48=b!E_B;ZKo?oTBZQREGt>C(f?HDd#gQ zxV*mDk%LV)<&&kO=@*5ia(TC22RAav-bdw-ZYst9FK;SBya~ z29-2(<_>R#SeR*6c6z`DhWsxN_Fd8$wpzX<9fk)C;MssX_bz06ONkC|3FdU&ywdgX ziZIe*17l!#$a9TFfi}eXpN)55)3kuYbWYIsMup-*RI74cH9lYfwq zxVSLa%l}%8l_|X`u#}B@FY+-PeFOGDf=nhqLWLww`hyi#K(oCzVZI{J!Bgb3FHE~t<$!JMIFNf9T>DcRZcSI ztl!I?`xtS@(_;IGt@TGrWkux77yMsD6k=yRfCl+KLC`CfxH+R+uN0cN;}qx$EJQ5a z7GGAHi=rYVL|=Eash+F+;m98EIa|gdP#aaGrp_a)sQr_`ef6q#?CaPoSj~C=%|b6p zl1VUQbfaTNcdy?2az}{?^Nn3XLBd*&`>DEY20~UPK>YL zhg$#>gWa8agVeRnjC(|@_AoQ#Y4j^#un?x-1hX-`xRDiTO-~aSqo4>X?y?7)=MdUh;sq-)n~=&^hSJ=+fG z7ve-sj!3zxJz1g9AWcW(r5zNHEfwkS?_|qRUH50a!2Q^EUZ796%1vJo znf<-iCMEoh|7g_p@X*%XpX^8&hwxCgDwsZ0QBlvZ<^gYid*BQ+R#k&2t(njD$qQ}v zR_oX>aY>-vBu7Z9)zX6R?2m1ZI}N)5H~z0Ju;r_9!`OFrdruac)x9<+s`@yHz^77E zp38_S6bUc++TSS6o%(1(-Kk$vVXJPBaJ77=t6NjNv{cP@$AZbC-#X^Ijwu^^L&lcY z=yT(M=8v%FdGe;6GsSG7Dzc)3s!D^5m_egTQ+tPA+UuAfhx6XOw`oHIuDvd2P<~jE z-(vTCRn7KEl=My*Jsa**(oYyL69q_AU&jKYp??rGGvxJodtVR8^LM|^3LFUEf2x1| za8Bik0{_rpsCSb}#?`!MV>6V%PaJeAvQTT23*w%#ddke4g6&(J+;-ZxHNREaM#;8$ zucdRi3v^#>F%NB{Nmyz{GoNB1w?=RRx>WP?jPERax>Dzd60S#e`Kc^ znZ>lLHo+gWS+o*8qhOsAUf9pq z?udT@kbVXew|2+ujtDcz=c?r9JrCev*aO-ZoC-=R|*o9l) zhr9#xX^2lZakfvO(^ZAygz5{tXC{lcXc#|Ux1!?wYRH5IwJ)vAJr4mmi2)2{O(l(u z{>1B;`%r6J9&a$Tlz%9)G9PJ~GL-_$wA%)AmZ^;+AwK4Ew5z$vbyq|%e`JtpD1YMcBz2BTt_)A=#ix-VWTuMyccT2R4|iu2cN;! zzzD<(CU2`?nYuMLGd^>wH<=d=#dJ8W94nNoRFL6$Ss%OH(Li9^g#Hu#{a1oSigJ9J z&WhnDdEx+y+g!B^A9T_gk0IY*C@6fg1lOe|s?2useSh>iacQ}HXu1k$G5ta-YV&1; zS%jm$?dR47jV}^p5P&BH5z+O1pO92Qp8Jm5jB)YX$WQM>iOauJgibxscEp>fp_sw- zQ8^0~U_alo9Fwn6tboWXK!e6a`)J&5x>;umQf}qwEb#X*ba~FaNlR4wZe#cKpqQ)Q z@GTGru#n;a&hr2Pb57uPe5aR1mv6azl3D8c&;AC9K(T#<5+if3fc?YLo!H(QUpg&f z0TrLG?i-$ckw-c4fQS&m%2}g4&N=6c2991SNSh))@fFMo*0w$wOj|d*jG0X{i3}w^ zX25LPh5vZm{U)VCj4$<)!C}sC;wDR~Xr+|$M9u~R?kLhX@RdZ)ifUDNS;;fehM~5lEYO{Ar(-F|H=L%2} z88v{aC{7QsU-@M6!ra+Pd+h~cOPg~ZFIH`KQh5z#$wz7=X%UCW*qlEx2L}UGl7s<_ zEjIx94`cc_<*S!14Lqh}vfuY(tDz^s&qttk|rlKrKHrd@JuL$7jO+l_(1^Hgmv) z8;q(4=B5^U>b0heCoDy=*~c$Qtg1Z)ldWbEmTo&`q+TCQTS|^Hxh5~#GoO|W^v%mw zUJ`R^Pp#()Oa2x_UdA_QBz%D2?Y>pmr~M)tNPZGxd%w|M!di5BX%71rOEXguzFc+J zU5&a>{&c6y+bY&N`37_~378N~fa_!(jH>MbuUN$|8zwBl7=0%i4_eT-logCs;E0$+ z8c=eMt^l$fet+nClY>f5YFq^nx*Z;OMr$k1C?~GXM=bPMPHoKJ>>sw5E$k0(R63

m`|itZ=esVqnyzV~uM<9m*cycI%75nBcp!YR2S?$La2fE?BVM)5 zOV}>_Yu`tQK8lO!&xR8(Y7t-hTbFs4Kxo?2v9jf8TE~d@kXUR21w1$p!?6rBOo*n} zyl%{0>yg@~>`i~sH&2uLK=FOcs#p=wzm(8fIo#MaqZQC~%wA&TzBU?HX+vCVo7D_( zJrGhs=%g?RTDzH{I(Pw!9No6-m*69>yL`=5l!KL~*%Osz(8(ilH)d+^G^zQt*2~}q zqV-Oir)uqIXWF_%Rttn}2A5#Eu(Jb#EA5ZL!jiF3;jq;e>j0OrAl4NtC-`#L_8b=R zT{@HC(pbwr-pS@V72asW74y3D$AT@E0z9xx$oD}e%hxDC{?k=jyJe?;hvG6UB5(W%z>D+SnKyOs>n!GJB?1V$uyEEwO`umpgpxb~1F zB;D*QlM*2n)0Dx9ejVP}Du%G4i8zQ*X%&6z00l5#L0vO5o}9)s-$-)l+jR0bJCU(= z%~@Sjh2=15sWGvPniRP8(IZ1sG4Fn3m~i)_0Ft`}#TqaJctD`EH=^RDzty)@l9 zU5#~p*=OVNXU1v;!p{N?#8fv8(}jQ$-q~eJUCgerA}K!}jPjh=9j&=%paGs};2=3y zh6=OkCp$JEg+~D?{2h9ks~*r&BM7_Yj-@K>JgC~s1oM10Eq?K)OryFu*+Z)vgQ}dU z1S|uj=x%>yAp=aMOiyUcFjmXE`mQXMw|-WDnBWKOvFX~3T+!^x%+Gp02Cr?YrsNxc`E((R#kfaCe31674p3&a-(cdy(g~;6ELY1X!EN0p z559Czhz@FkDO7pTg~<~gM(|rS^zMNacT5_~&cFKZ0Cu`2bzJ!=j8JlX?ck{3Icz)p zc#Vo^y-1c$E@l&|zlJ2Qw#f`v-B)=P19!{oxY?-nvaWE?MI*Bv?u=IO`;jouxWbBJ z$&bs;+zFKp*0@AqeHe7e@GvWh2P-im?}VhXl*MTNBw!i|NAW3S3jXV4>4mNS{(Am- z&I}7(e0{Yd%*G|Xm^k(a!)P2q9X~o(c56}_lN0Z*LKb*3>yh{1!3i~bD`dOBVO0aB zFN0F>c<52z%15CiyHCWwYk>ha)eiH|<2+x{Z^y6))dN*U{%3FbuUm zR5%67zO6%MRRRfUVZdtFE1Sh*X19_t-^mni^d^7A!MDAH!^&c)<)8caXrHYduyKNmN`LuhT#zw>$9HK zBXq^7>@iEwLlKEl)ZqYl>+t%Jzgii5CLS}C2qH?;w|-C1$keBL;#VlyclJKS?U8mS z@Cxe>A0#Bf5pdAK|KtLs zCbfA0Jg=Nesl67Nrb+k}bdb@OBrqkKPwyim7U&7)rUo6Pp_TIB$6`f;=37=F8Y-2R zeR~!~Lc5?UhPC>!+z!f{P{l#tUi>%s>#s#-b&WXfriP_>YhJMp;kf}UT%~GP@>SVfbb0yufxcF=C{lH%Y%#hBo5Yb+KdTU4 zxR}2jFno8b!pbFsP7rFp}$XhZi&fW(2cgz)X+euKW(sCFNXi%2F4AJ9*!JHLSE$LSfTc9ciW?Ed1 zd)*$ZasjB;;>NCE&pK%ysJ73We%eNwZ*|_!Z>S4XVWuM5!prn?+Np_wJYXQz=Mzlg z*-WEY_<*-_X@#Ror!Hr5+Z`bZ_jNNl!Y-3?+Z=fylPcAKiZEeT0zWN#HTu*<@HNt> z3*=S;9V%Jz#!R}KW6S;t)2sU{GBnJjVY=~3cK-%9Q)tsNvET<>DW+u5%57YZK~&ZXntc*zfizA~90$Y@Tq0d3mY z0673hyr4?L5l4XugjzYJTLhkGN3rOWb{OBUM(Rs~z6zSqDV}7OG(L z{D9#ly|)O4Z7<#|yEvL{`HV|GTv`U=-vpQkX4b+#I~lx-j+R7GuEp~=GhC?0wCdld zWY?~GY8DI31W(CcsMV_5m;S~ouQr}@*7{?(CtfW6U9H2dj|>gb`zpU(DJbO|7}O$W ztx29%z5C)t4o*A#K;vC5?|3gEONTbLp0?AAx=_|}Gi>AuFjyMYlB`((-)p()(R&_# z)iIi<<2ult*n>bFL)MQUH3;H6kv|plwTRMCcAowPfO8wFH=7Wu_2gZY!m5nG<`e_A zcVo*ziaXjU9@{HBmZqHXkOAvKZ&gjwDmrSS8_?ut(r@-P&ERcQ1YccYGJJbAqXVEa zjWKYM$IFi_Mt!v#^>XWj`>-=&j;umTD#i6HVWyDmtw?*yo-&Ij%ahEl-OjF( z*G6W0Dzf;ll{Z{{sSI^iq0=ADtFPY=)pJELpQT69@b~{%8^tJR5a2$JcTjsx zX32^jZ}~>{s@H@-+gC$h$EVz^WwpVtCc$RfLsh)LP$-CMJwRgS_jbpM+4>J|(U!$2 zePmhNJyc5S@;6kitM^nmlAT$0#kFKv`gs19LDTdH?VX=CAY~ZxX}9jP+k63OyI)h6 zyZ4@a(PuEcEIORPtErsD9OQmlK1O>Mc%R`l6VhWdtvOK*7{HM!zb4`9g`Y&yJmPZ~ zP2Vu_S;fI9DVpxy4G@BRN7G}gW9CNnY!J3)JzXQprm(qyk#R_7-?+NcYRmXRnV`-n@IHVF-feEPg;U$XXYvRAc9PyNV2@ zv*mB}ATyw3(r2MoCygT`=#_BoV5euo?$Q6F>pi2I;F_-CByr^lq3+kfCZ3_KnP8g-g_w01eDM_gd!x=P(p8Su6upYx9;^m*ZYUH@{7PZ zXJ*gdGkeBe#)42zNBvKnVkG;)|GdsGTKokcAv5K9uqmo98^2(N800Wf7)Nl9=%qfY zz)1Qf8s_b2OdlgYbO;GZVfT)PZm(HB(iK19Mp=&YXW^+=J zN{1og#D^El7Gk|6jFLh!9+^_g*4PiQf-qb1DaZeor)3L3eyVfaL<@Sd;w=KbNKt5+ zR6IE1-fG#Mz(#q>d|Th)2ZOYWg~7c{WTTsQ6_f3WCYX=yWmqpHgDj2e*C!|G zIBAa7wCTwYr2;qb{Gw5;l&P9orAw7n@WOdJ+7^o@6_i9EW%s9JUy{BizmW+_c;(-)X4?f+u zAE=Mq2?58Ay~bQzezo4)YEI5D!#Yq}B+e{wV>^WiY*IUxsSWpd)R4CYUu_$T<^2LFo&3tF1ADXFOF?A)s|!@g z6qm5j06miB9lh)*6QA4D`P$x%UT22ez5*j+6V|fujT3(^uDg&&b6_t4wOab3_6@F~ zr?o4lzQ^h8aTTaXJxw#So{CQ)N+XP+{@Vb4svkZ2cW1@;L;OV%yl0B-KOqm}8x5>n z;Z`1Sn#u-KzoF+;{rf@ecNP(aNK;6&W11ou)ZSD5>4d)$|7*R^OZ~t(= zy#FQ@ihqO_e`6!A($K>gV=DBXeSz&zKj|E8Oq&p=%XxyWn~j}<@lZHvaT?y@B51DU z(ES%~uO$0kPkSS9+3MH7`h(eg-ihq4H4ZS5H7Wl-XDL|nE3U4c=b-6pNVfI{4zWOP z&oiH>oXYCNctt&-K84GM+E+ib5i5K(xG%qUho(Yv_st3+xg{_5W^2U79)YG4)`OjZ zp3i2BuT@B^9*_TALu5nun|jr00wJ9YrGDzE+S>IgFvKs!_ zR-=H*T-Y&GrZ*hGZ2BAaW~GA92vx5Xnp4*fo_F$}p?sbNXeZ=99?qu(28t}&Nq5Ir zmWt~C7Wg>?`xc$2oC9Z~xQLRs?>5jPe3`r%bM=ogVDf;4nY~gB?$YSvQ;C=n&b$$u zOVwHiZ0dmqrsb6@KQTE~wiM%2M0rp2^a9HVWjmgH3B=qUNY8egvmF}ee@)% z*XGHKM!fg5K<}8G!>!G7`(Js(JtBRjEXt7l*&WF%-#@*hLDa46yqq6d7sjoYRjmWz zjki2o5HA)yLYX`)PM)pXjQng{f@T)ejNBtk*6q`=O|viSeKWI3ZX_1PLhj5|N-mI# zg$CEJRKTmPupTWVkM@7YCTDaSdX*MqFK#Ni3TjHn4eU zqz)yRH2rS~6zF?ZR;Dk_YnnM2WMjRn#B0K!6Tc)}vM0skog>#I>iWI%_9)xl%m+&PiTNpsn`^KB{xy~2%g1|Z z!eyP6!6R(}MtwA%!9hIBbN7OT!{0nXHg>*;-^Kp~M?i5g4^hl8o505O5ovxNF4OpY zirqvk>R|OZg#$A{xkwaKn5}{c8U&>!q+E_-=CoByl@4xcHU}z=J*9kUKRym(=0C8y zzIx-ZY`A3c@Wb;zJD#hKk}^`((fMsEj^}@r%T{yOC1J&E(}#(|2oHh&G<(pr)3U+& z?Ju@he~~{(_0>me^FRKkA~h4d`t8n--o`!QdZVfbzdQ_o(WR3T2aE$(oNkfqTBq3u zGApNBGZP}*#3=E0c4um2Z|XVda?3;7nOpGT(+b>>G#-B&AG@};d$@OqexC5q&my!O zp|bf(LfW?8ra}e<|1Au64GQo#4tzTm43FRXcD-L;mca9|8vp2b#8^b$S1u;M(7Yr1 z4Cy>Qt_YZEMC6l=cD(MtFe92+mLut5`3+B}s;a;A3Q8k{2l2RqGRn-PQiNsdq{Xx9!d5`d>BoO8ao*136g&I+cd#hk z;#;0?a5&GJS0hKIAZ(^UpG7BY0=wGkKejul|EQz!!IVoQxtji<+J%shbL7m!)&Hl0 zQi<^RqUx`+TN(P%ZMx>!3ZkA7D_45ioNM|Eu}YhtAUW{b8D^D<)Sx6f)@q+z&q3bg zUP~;p<>Nwk8xL1ty8e4jvUO7*S@-pqY?hv7OdL!1>&CATBRL3!|KcbLxWYx5m=WVF zS=e%yXa5=^`-|6Z(WPl5Xi^!EEOw<9DKL}ceXGtl&1O9VQf-j-amBhlDU@+CkwH_> z@&;I}Z&cGGk4c4s-~^LO{Z)Vgwox$cAdvjcv;xFmcVEI!a$ zI9(o3VZVa^mDN0$s4=O3B}#FqRR)PBYq2trlXk-nF_Koc)G^;*oagR#zc9_loO@pj5CrjD26+LUDOzs5dijBylnlm0^Bh!aY% zdn1%yt(uG3txYb-Hh;Atw>KetaeUP{|DXcom%>HVaC>KjF{ErM3zX}V&|bQIMzODr z+tzX#Kt4Nu-U|Z%ens&e)ynrR@;t^PH^CErzfbAvd~rx;$6M zE4?|STVp#64bEeRzm7S4grRK67FL-mQqw6_T4I44;jkD&ZxhuQ$awkA<%DIXqQ`_6 zbvu5xMaTY2$LHB;ruV#X#88YjX#64l*@yqig8x}egL9UaI*LRHO z|4NzmVt1k`=2JQL6#nY8{7ErZFh0QTx{YL?jT@)bz`AFMQu3nd?7I+2?CB~-h1B!+ zGD8cuOg|M{!=!a}yHMqQmNRS7zVT!EH42fH{lynHVUgb!ccTwCscu>z=@eZ4-88jw zj51EXm`LqALs5)d-hSTl^1FAS81v=vXI;~3$zt8k)*kpuWVAKLtG6=P zG{df!x7wqZJ?!1_Uh_8d+Zcn&+tE{(Oo@fc`fS^OmGrmm(`3Zo1~#s75y@v1*G;_R zKOybPwROt+roDCB;kPPfM^iskNz8u_V~Rahx^@z*;EkOK)H*g)ceBC`u_Ez~vrcHh zLU*6@L3UJm)kZ;Oz0jb9LY&wGj0)UgHJff!uUKEPb~i-%ywa{}&|$h*?6k$K_Bg$V z`jvv^FFNW^SNO*IBCZnssw-TXg=@Ubuqox-b(0aG`{6P@r{)irS9hET&;gX)mlbclKCg zDQIANGI93#rAr=Xr;ZD`9X$DY1TltRl7HLdx1U8&l-BK$jTJU93JiOk!R43!FUy`T zcwh4_)e=+c)P?0ep9cGnT{hvkuC62X`QejCxd-})r z1twUrS@`Y12l#ZSwW_Eu6E}Am{egke2 zXpalKo5||A#kE+%x(-FQDjo-=Y1Ii0KFlmW5~)i|X`Gx#pH+(S9!xY0j(%*c985NM zT*tTK_dj|c--2W*vZxw#w^PxqdSnSp`ZIF7W?!4R^yBK=dn@ff#UADT9JxPg& zeD#g9FuDm{p)9xS=7O1brhkeFzgupm_U5+_uf{-JoZaG)$mD~A4h z>zIAemVK1-CSY&uj{M$gn@>y&>T~fZT4q7*DR#6RUHPWa^tIVUy_dHd|3`~@ zP37@KlcyT)g>=$TA0NhUezALpnn>S28B8{Q-6x*RFAP|gRTudlPyKk-oLgzZzp1}j zsesA8_IO7b8g?9=T9QQtOS+ONcHYuybqRnFywBi$5mo*7^z?q_EdkaOov6r`QaWN+ zPqo93m*xAlXvluTyYzCL`(~=bNv{r$GC}kH_=%R1H(*)vb8kCPSx-2CfhN{k7NYN@ z^C=dvV$cMug*ijN!;%0ZcWb8g)^gr{sPh@hM=Md2t$m zr*4#-E;OEClwT-)2;HFToL+cwBtGV0i?7Aq<2?Dh!$&Q2 z8n^6~d2Ym&d*$}$*q=A6AY7KjL9R?j9Bd6+00h!?hB@G!2o165`CWQTmYTt(y5=l^ zd23OGxa%08T?3G?h=Rg>lrB*Vxb>a23SWk&%3c-byWo8+`&G;{fMs7K%5YL0c5B1o z^u}A=8-xvmgI27_chGy5pNHVKc*{gfezi_9l>^|x+~+RU$et1F;USe*uWH?e*QU#mUkpuMo0Q7{TLh-&aRNIaHDg zx!1B-QmvJik?Cy!6>RMzivZRRN=890yX)Eipmqp_^6E5DsZv}Mx%_oYB8%btHx7y- z#HPe73NgZXrqxd8gYSlPdu*;c0g_L@F?xMg-oL*DMgrhA*#>FI>Of9ZHj>P0m-7w$|7 zP{VuAm6{?1Z+~`4$Tefr8cAcIbw z?DJ#d7e?-WI>`%qi%*ZA5b;PmN8_p!L(zPJkTzq2kSyy8Lz z7kOoFu|F3casDhwJD}TtdbI^hUmGv@#;w+1^bZ-WI2R`#F${bfjP+TNuPIpxd zKt#B|SnR6{lAWjv^qA@!_>F9wn8qN1&{K@#vGuPwG})V22^;$yPdEPbu<~sdnN8jy z{pnykxXMVivr4zR>|>5XiBo*oxaO{%1AgtiIuc5TuLfw6QPKcPY@)NjAQ0c%GuE1< zr}E~_yu*{ErDh)8#svrJ6>*xOEw}`Y(w|X4ZwkqTWLjcUG|EAT_KOv#L+6DwfHKN* zRWx7>KW>@I1m-e4$OoZz$ff0Nod?y%(`K<6N2ROB8wlI%lSehhOOVy$d{HK!r}_2# zScais#HXIyt2vN3$@&dxUOwvISii16&#M}nd{o~A1)zL4cE0-#4xO+Uyl#ye$f|qe zr*npmMUl}8(i;kXcAVc8zzUaoJlGoR#oiN_kG?N1!{ccltg9mkg3394J7;2nS`Pnp z%KyZRe?IcKgl_}|^Mwzu-E8~7aJd{XUkZiakG4)syK|RBUGkeb^#O>u%R&Y7m?F>Y zju9a9p7s?=n^j=#Z_%EjWRGg_*N&8}4LLW`RVs<0B?!mspBh58pfqD5c=J?0^VWBs z+FOc(fa&6Mc*XYm?e@u{{lofecpUW;|H5;-ajQ2nW!M4TalI|rwRN{7h|SdE*Bg8# zRyU$b9J$rHI517vBU5>E^XI{qY<_)T9jFp$2YAD!<$A=pF@XCTwfk2asTCujyouYT zXR12b(H%&m!H&m@Cj%54mGbq#$*;PBmywn8U(nvkt*{097Ln{kSDtgUyMU6a z7ZimT#1FJX4A11Bp)V4}DRE)5=xaHv_^bXW-xi(@9ZNka>`{DJo zc%C3DA)kSjKmoAg=NZ0C$8-(cu}Z|h9!WGEf*#`t?6(P!WlDOdv3qk4f$S5{c>LK${mXxgOBD?XMRD!`#@5gc_m=UBot4Y6<{rpqh6N4qvH44dwWeI1^?5Jx!VPC{Dmoepj~n`Vt3k%Q#tmWh`Fb_uJCh;gmskD3_oQ@-Zum&M8868GPcMKN12LB7 z?^cfv9b*h#8gfx-S)Ck3&OqoR4D!Ymdbl+<6@PgQR4;iA*A9WhqG;^*!HurCDpm5Mg_#O1^BL`SsNz!W zbuINl2!TO$d~16{!b(RtA9aZrjT|J*0kAgwFc~ee&56&XEk#BZ7eBGf6=qT4k`FA! zXfal|fXXs>+gEmtyaR;d0!Tc!k!G?N?+2>5sOt*a>1FEl{j48~mm>CV0*-Wb4NF>n zvRx|!U}VaGPJzspO!NbS@zMbZH9A7g-=N0jZTO4|Z{oWx<4utHyaW3u2fQcE?MwD~ zxRvxU$h!EUxdxm)4m1{ro#F*%%OBP0np5KK#jy{Mu{R&dG{3^SycdhT=68pMIT3wz zwEi^x^Oy8wYQwLp8fo~@_ekE44A&oGCz<`dD*tHF1=Kxa7V0=ac9!u`Vc4~B3&+Y5 zS4b7xp{?V9m4?m|nO&bJ2+IeTG3Pou0er>hwT6dYUm*MyTw~>b)C_2HGjQ-Lis?I# zE^4P&aca>yPbiehRG_|aWBdya32(>hu*UMI zlZQ29$4`7Y&I{br?>bM##z z)T8`VvUK7QnGgXi9{vfpuM*Y1`s}=fq`JY6S783TB0`>iLDugp0Oae)2)&YfS~s4h zB3@{{YT`byXo+{XfLQMW6wc+3KJ6#xtZ?2P%_sx>+VE&;QTX|m1}kS5;h{!UwVj15 zvazchncJu{rVoy09VEMzhyumgy{Hsu2k)DbC zF7_D)4r_y`Lk1V_>FR9cl<~%LcMUrdQmI_&6oXvdgWHeTQml{A14$J z*~T<{q`Wn=%G-T40*bepgDUuPH!y03s9g3 z9DSng&n>js`>}-oMoTo)@h*Sr6sieh7zjID5Rrd;QPw5^7JYO7o%Y@%0w^FNY75Ku zvb)K&+s2O07u~ai4&Ceoy`0~3dkyfF@?3+Oh%FW+{^;C)81O*?o_vKafW^r7Nr0;_ zA74)sFzJTWtDgkgHoVlj5{;8hEbXVE3Z1nZ5ltdKiw_+Oi>v!%XQtI`bK+ktyLc;O zdy(~qWnayMenVEbJrMreuMG`dP4n!e(qL5q-HbO^hv_aF_jKl65As>tzPiu7FaE>? zklvNYxLul82GHFy_Z`_nZ$@sE2Wq3KlHMVDJs_Vwe!vZae6sR_bvks#Wisl(Qnn;go!>bPTM{J&xs9(`7=!xmeS{#HSRbu=G)o*1C2 zldj0IPprYJPpKP4<`5}R$c=@6y5$?03?4SHQdabfF{0xJQ&>F#8DdmI1ORhn}Gyex^zm3pCs=OTm0 z;l6;~(d#cxM#xX;!0;kWfzCarh}VrWGi{9~AFQcpj?Sbr-U)}n=5&+KGHg{At=`QWy$8Jsjf~XaZ+T!PO>g8u@IvT?c$PYXzqDqj%*Tgxub+Ez{S-?VkLjhjIAF}~&=6mqdniB^ zj??)QY-1U9@V4+o8C6y)0!qMd1SAnqByfZtI&duQ!v!0+vNKmHhMI$Ugd>MYzGrhkgic=)TQtRd`(cV~oj>VrIKQ&P(Y4^uYR^eU=kmx0>-h3f zQ?;McaWlVO77RubU{{KR5dp-0Rk68-C8hKT_TFqpscC85!JbY86BTx}zIsXpBCk_# zg^`edn)Ylo9^XA`NV5jY~VgS_H zHcm73;o@x*88K=`$L?A_sM(l9We_ZcC<;}GB@T}8rIdM%XIHj0%U zeDpE;Y170(>*Ix^$}|6`q&FvYUX?I~Yc~p1CLi1TMSOOx`5@uN>m~WR>mW#b*dwVZ zW^a2-WflL@aM(XddveeD?Ztt9(AOJ4Yr7@c-4Y)$r2m2%4Ie3Mm*gtW#OLFEdR(({ zKM5sJbxyUHeNmN9y5LPtlc}=x`6O`OPd&!SS|nXx8-f#4R$ZQu=UOxCgM8cy>H_&d zT-VYV>3fAQkYgr2h_ZY_@c}aQt67@8mmn?+fxL~^AxbrF}I41Z*n=9X0i&#ugVTcJZcGyv0;vMnfRYP0%3bsF~cB6*t4AxSvk##Y(-S4RbOm%syG;iB}Tl17E z{|5=qpVk}sX6Y<_G+t=+)3rwK^}xFuri^tnCf>CXQKV8u++-h04%8& zrNVIit!k?P0@gesA^!Dh_Bz9Nbo;&|_`$FN2H`PvG>iEEZu!e?h!o1U_CPO0iTwI2 zYl0;lZCja$`7h9Sj~ZI3uxRs!!pg8e-@|p_au_XaLXm>;9^=$E8lw42Yet=UnR2tZ z^iOjcB*kccozJ;f`7LODji{CzI_n2Dr>R%R9Ifcq%g*ce4u3Y7Sn`P4WGjV@#?P!M zvmX6r-kh#msJztaO|1R#Qy%%^kcyXz@UF${-3_%4u?THk7U4Nf7-y#k+c%Xicp`y( z_qfhaGw(F$mN9swv_S3cwFzM@8#+2-CVySBhM{do8XNJ@9~bj)2|F;06eujB5*!=& zV2*A>i0&oQ^oM{W+3n96kJshzdm1YzL}GY_M7Ym}3e!NW=|y_}p1o1aGl4srvtx3b z#B>a9p3=6;9vKOFj`NsT$7ao9k;`NvJJdiJIyy*D2k_3qTfe@d>f;)Z#>z`U>Xe9Raa7vwvAsbBRiEn+4dGA{l%q z&W{gGRZJR}Y3~w}E{Tf2(ttKWd6;>bzgXSVHC|`6&Od^OGTeE(I1vm`o)SAXCZ;V1 z8;DHpoUw@ZU-_3D_n?RBA=2(XNe%HBrK(oOKupCv_5GDBRkJ)c*17h;@bwN$Fxi%7 z%|E+Mwf}gXUexl1X+|B_k zSAkYUq4>7W5zz#nYIz#fFaoun41EoYtjV_#xp0O$%erbOhFMuGcF(ktr0lvFRe5{Y zTGccPx3eHAP~mC3{{wiGawvZ>v$#~HoAePI*}01MWE)TX=$*rzNORmU_|Yl|Ml;AY z4h2@&3lG1jOPNi^?xTy zQ!j+`@n?rMhlB@!)n;dy)Y|p|^mQeE0oObx@gv8rebdBOk# z*CyQzgv8Ke)U1zp^#mumtj>SwzD}3G{sb(3{CXkaosiapWB_la44_Ug>qY=X=qN)7 zihpV%uMLC|qaaUp)uYJz@-1_<9H8!7z<%C{wxf`cB%Vf>QZqA1l1kgq?(+pYU*o_> z-)ytj=8{VEuqTsgjfnz&U^yruhs5Ocb`)ii>8bkUyeQTfExDG>9c$GcrHHK((WW~9 zi)SE!6~E-N8PXpJ-hA)_g-?n;f4)=9PMS=McMR=pI~0F4%vzzt(1!q=&=U5xj`rR6 zlZzCdsLffk6IJ}!0&K^D>9MiY9fvyyK0;Ih(m2w3{JyYdm>vdcLR{~zS@25&P?(k< z|H`z?K8Fk3NaABJp*xMf25clS50@VXHa)QSc`)~)%1C6QrjtK$Ax3<(#mV?UM;3gx z8$YCyCb=jrCN4~~Y+;?7Ru|>PoR@nv6XE`8i!|M}VRA%J22QFl)O(yj)+eaZhzset zA}BNQs#QzryXyy$13 zn~AeD>g7db^q`3KZ@*HJdB5mzy_jA%d27rVpgelnzW}Oy!W`iDk;+T2aDfdfnmx&1 zS7Y@;QjzNlp8%fdH1aXrsf$z!$^xLjqk`u52*MmnEJ z40jOu>-M!*3CfTd^f?#^x!T6zUjwnLgbD-j${&C0S$*@(roJ>dK2)7U5CE8T$9k&%*16h!RTxY5KKBLvZv_E} z%XqFYB*Uxv9ly3MS5@zB1R47lkKcOkIfuN%o`gOWj-~~YZvvEs!_@FcaI@yg2PyvD z8~NB@@A4YboG-f=0j41wfYWBGJ_2(WmZqkeRxd-OouwNUx;$wH%+p&JN-W9DEP;t+Z*uPO(_HZuH45X z&$;#J>8F9I>I(ry@ot7QbZL|qp?3XWYwEz*r9j(|dE8Hw95a+kCAC0gXwcuXHw(Po^A)_`xGrUI1%UfI2(WE00L}Sbw$dwXZ$%2w0Pv z9JLzA;Z<(n@^mWovkiWczB>S7beZ41#V%`sDMu>`l(#&Wug}`Pt0La`Rq8SeZ-BI5 z9W@bvyM%FI^c(qUykVdQLq_Slg-1SwiLY-!VS<2V^LbvV5`fIhGT>PwooGCoxY5J= z79^`EIz@hFc@;c50SmcTsR--g*rH#@M>T{=GzEslQNSsk*uRtUs!xx;P_HHfc5ezI z95J;DBBM`z2JQf`ysD%N91jy#An&^Gd&=be&y%*v10kN z@M)97yU2&jsH%51Bmj(MNJ%S9xNa>J^#nj3ydK!t`Et!hnq~MN+&g5s-ay%ptu3SW z<~|GV$7qhhc=e`V_tBLmimfy$CvCS5$hge_0R7Uz{LmTAntFYFmjbyC*(>onmH4Pp z|G{NVnd28?Qgo~co849erGrOKX-)+t9rus2pNxMO+?{yE-(dEp)xvZTmSlL2+WzAzywGLQ=zW<2Ur zpqi*u)1$%fa-z1jvBI?WZ0j>OMNnbc0}zoGv3 zE8~Acb2<$tWehz(eH?+y?fIUxxQLTyq)E$Z!`UAc%+$K16K2{H3U5RbC_;#zSB{Si zG86r7Ab4o-Ge7q_8w~d}o-S@d$tfjGQj}!F(w!+8LQOpL>tnzSvU-qPdH#2!50p>w zA)hwwgbLGaIbiuU0O5Wx#MD)9`69{Lz?S?5WLo^m>#FSwDqNJ1FzcYUM_}65hToO4 z4U_H8-UI2c=Ui_4@X1+BfunZsk8kwmzF$DE?(GwX_P4L>qkj>~v-9z0BI{dgoAN4h zo}p=9>wb3}CvJI2oWkN-C@Wa2RS_Mv=bx(*lu_Wz4jUdwHOC$7SL#&VOHf0aKCTM^ zEG3&=9VAE4u(C+NM^fJT<$CP4bk#dM+)E{BXUqnmABe$ib12{ByqhY$2DcaaHn6)* zVrw9|&z^N}oBY_{Zy&*pFAmt4yQ8W6Z-S#136ESaVD7Z5jOBjyk42jcV|WGD zq{H^tGM9^UE{JG2;xDrcQL{Zq0|@ke>SiDJzgZIW<~|LY@KPIDtGm+-UqUZ9B`J2U zJ4Ctn7dMo2*_xr`w;FwG4(-z1(#d*b&_F_2zy$6t>ZjM56+DMlsf!x?4as^ov`!M^!YB=p<9b^)LQ|B*YeJ@ zXZ91<7La%2r1YYg{$%N8hkI>sau{aT&N)|sJwu%`{o8ly81UuW$=g=dfyfw1!G9V8 z%tv$T5~a37d{edRhTBPxKtw?R?=TBSMj611xKH&9S#&r!#sHwAI*ezI5SFy3-#8%#g$yt1_grXNIGm88w!jXUVt+hEX@t>Z>MM_?!)WruK$O-K2d4i%0`6eR z+yB!G@Lvs2JTk1$fkWkC#E4nAfFkaiHqCi6en_XL*f7;wppK!hT(>(%zo$LO+zsgX za#MmX0OFY}^=a*hC)RGD@>*x=?^pnHwWy5S=c>yplLVDVN2)eI6|c=EpJTI-I_?{> zW=srf`+@|!)Sc`SRR$<)Y^P)Uf&@!n?&eu15b-1cK7{GNh zVuFtt<%%>iq=)P;o5fm$UM{aaf~(EQ2C&HfV(N1|0oD~#h!bqb(mK^%^2qsbUf1@1 z3ZdO1ow(uEe3*)c%B0RkrzTR2R8`SfPSU)8Xr zP%3$|F@i{i`sNjMY(Q+2ohqv2Xm@B6bw9X5@hNmKv@-%k?{?|!5^JCTAdtY)ApN$K z(vxG*&`bBmuD|;y6%_wGD^y>;L$KBY?blEeI@=I1$xAq`i(e`%{}IWW$FvC{l);)d zCn}aqt_0tm^l{!b8xQgeK$o+O`OX>nugKWm4ESpnKvqhv-_K%Zyv z(~lv}$5qy+qWfG(j1vGW`%R=T><1l!)z{@;c@f}tv)c~TI2z57=}n8?zpcm`^f-SZ ztQ3G?3!m}Qb2!;dr}<4b^!Nt&^E%K}0&Q8zAg|ES8VZm=tjG=Aa!h^WOt_~;{0lU+ z2qgcWgP*jOlE%F4t)s$jE1%yY4F~S0{Oa??6m}Ph&=)PQ|H!vyKh2*wjs?Q)(s^lW z_}pde_`s`rx34%!IVuNc#2p-6XiBUkec_|a7E8O3xwlVEMMhC!5q zN8hm>f)u_|5}C2n>lb8}M7h9B@}z`ASSyynE)f=h<-KJkO8Us-)NB6?j9qNj-Ng{{~KY zW{+U$FEguD#G2qh69*!&p|eC8a}T`K$2za^GeN;r}s>Pt^@cBU^K13B#( zfsL5{_Drt6X^(TStQ$G@bjzU1rXe1WTUyt&9k89Dqb0`MzK0f5DoxY%Fj1SD;#m-9d0LmJXYM^;2`Qok4}5x;Ctx~Rwc-ry?e zG3zUxM`iU?T@DTfCXsFlt~N5OX*QD~7oH++O?C@BomFL#l@S|ipzRYoHcKP4KJ`r) zMO5dR9t3W`+TS*yA{&2jx_e@KnKQe-2|0-$qzGK_{T(eTSh5C*8p>Zi3g=&-r2qa~ zeZvw!)yA9?Sq&H$)C^-S>@VjkeH>?Z0~}ZPX)nmLH;}Mt4*CNpC^6gxy8ql^tjb~; z+ikDTnTT>)^sZss&$Wp_T9iSM0W7aqm~s@#etN42dxr^Os9$%8r8B)|KH8x&Q~AR)Pb{iQRKTzpz; zE3?KBJnM>rrUA|#(hyX;53~Z8d$%eBUsC#x^<2LSs>*QEnHQ@5l+0-v+T=MgmSBAe zTG44oHQJS|iBIWa-oTSgbu>$#DhJJf-_28ceBC-uL)fDUq3|YR(k2Q z$9!@o_qMPCNW60uM1FZ8U;};k9s~KrKU?r$`P09FGh5J9$|Dv;P@11Eb?CFcE6eG> zW4)Q>dr{VoTj|enT@YY`rsdn&1c%ktX~I_?^TqT~^pb8?E-yfW#%RDL<&cU@T=w7v z0B-WU`@Q2}#{+h5zHsFurOl_PrQ@cITccuGXQz5lLgkj`S)+KsH%q=}`3}dvezH|; zcUzM*NN%+T7aLqkAnY|ob4#jpkfX-th4|9)hD3PWsl}z^`P{C+%CTl&lut@wRQjDr zsx$P}iD!%S_WJ4->hgCgx^N6U?4%y)3T3Zf_ftOd7{dpmvK(s@LYt8T)z{d$QVE`) z?-F7)$yfYv*M@IZDVFtAiT+*y78<4!Mq%a3GhUwnw1%wjHa)VDFFoc#`=SQc;f&H& zQ=^-;pSN~8AT6j~X6N$zFQ)dHm^ku2g?nx^BgzRJh^L;MkyQcPvJQ z;W8HK6EhFiB)=Z{yZPz~Sv`uSPi#$!h0V?s?w>cX1Sj%V+Aa8@)fqAA!?Q}S>{O$$ z-AzgJS4f4F4x7y#p48xkw=@czzqOPwHVtM$1`K!Yo}kR~7VOGo4um)Uu+3WlVdkAr z=C}>hj;UZid~Enho=>aIQ&fGq&q~|=o$AR#PJ!?m{h2)=!V(nC2Aeg+-{$%~Y73qcsi-aBsM= zIXm4RjFyQ^vH5(8?Ge`_-_v8!iSUh&<RUtry}Sv$oVfPJEcKnx0;kkmLUZeLmNj+f}SqZBF%bsDJX67W3V(_e#e@ zymh0HJsrAkK|ATLJSoPw3ltvLpPT^sr6z@Z&#Z%oSbHI|6`l{zONPh|WnV&nQno@+ zZQ;-0cSAi2CY?th6nfu@*a29}sr}>T?7ZDinvoft>N5weuZSP(vT9d(mIvVF!^ot1 zNF6QPDdx*3*yl1oEbnE*k`cjz%Ht~6EELU`Ju?mZn{PSlDPt-6Myc=MZ#yM~-; z?E5QrCUnGDAnZ90y1+S>`-M!!a{#OQ8bZ<4IW{`lfLq_1r45-r=DP!Jse2wSQE;4% zr)5{wDA)g_@Ap>Ff&%v-L|%R8nOEiDf;GUB))Y?TXEt|Kd0xB>S_)`d7u z_Cp}qY(bI0%}UFBF9@LVxgEZ}6s>MJHu#9@d_sOhgDg+^HfS;6( z0|J&6vq+i6{!7~%^`^VP$cb+Ojq}P_@*7Y!Jh*oh@6hWUzhwqA3P8eI`S5Khxzl)g zC?u@!kVFUC44x$XS6x^NJn57cfdwz`3<}OKw(3sx*C{6|g(em~hb&N-(aE58gT9-7 zX)6YR$l5W1yw7zqp0(PSuv1G%2C8bhU!5ZH#X6%;Zms_P(*^f_7_g|#rvz-E>WhhRfL&*JNo2gQEzS(e&mxD}ik^XcZc&IOR03 zYWS!+jJbZ~M_fWGprZn3W8XRo3ln5Q;#xMKHc-n_UC<*`ApSu*S_GgHibS0dLRCL2 z8|iA;@0=S0NBc>OBh*qbFEf!|5C^_C08jjPe<1}6x9#PK^{sQ!(x!9J+=uMTX#>)n zKmf5)R5^(aliXPJq~XzF&O5%~Di-6L$QqN(`6#1<>fzLQUYiXFnXD*A$%T@ik?1UO zvq^afuT@TmnYSDGYC|+OGJL^(FC(L9IJ>GFvS(=lsauEwk3e9ko)ff?s?YhG+as(&Yg|Y^pJ01eqTjAXZhsbBhQxAqaLDISbZfuA=Af} z3KT|;e-9mUIBl`Z9l<+QLU-uX^js=-6zMbBBf@aAEA~vZ|CMMaYsPP$XqE)gHEuee z)Pnl?_YH-$aU|ng4)9Sx;)795%*@LYdf1CLK>>L*?nLz=?`etb)6a|gAaaYNCS~pi zeOs8kx=s<19rZt9E{|3heVn_i_myos_AqI%@cfGwtG<&_vyFqx?wb?QXuju_yY@pO z_IDxzCW8xNfjvjefTDahMXjsj@|(&1k(#3_@tYDz|ALURn@6;ZJ6z%Tmt|9R>5=*q zN&F)R`c%0whdEqut<3)ZF=`l;ERD7Ig#^TDhW(|f;UhqRdGM(LuosPS zwZTe5KsHDB28jkAS@%@ddf4`w(l;I!J-Bkfoun-y%DM6Krv7{~#`Rpmn z36Zn$A635Q2aMvlh25uY%~0T}Z-In8+nTgEIJc+2^J6`aKEM9zX6{P8yN79r%r}3~ zqdBiSej~JJ-C7nMWhLI+n%yeMD3A7@gkds`1eu}ao2>XOu)>G^1{YAP8K(R<1<&#}8uy7Za!P)cN`Afa~<)O`80O^+mlN8jq%;;|3teJ`NZ> z$Wk3>KTbP`q~qoKRu){r)T3>7PKgZ<3wQv+g}R=4^QZ<$XZ%*{wz7z9_sgI3&inX< zxB?_ZdJ$D3@MNSrmguhfgRj4{+%=U!$6NwnTZ9k23gAcJFZpa->os6&!<;X+j+TDo z^V~uTyEviq!@Y+9uk^Ve?!XfBkh!N!k;7jYK6t*>zbG#Ts0=$3bQ;doDDc!XW?evA zHrmm3Pf5ClZ@w&g46ws3DQOL+k$sLX>9?NeA9(LqGwoyMv}b$WfcaVB%Sju~gka1h zY(9AN@Rs3k-{dQmtM{3S&%e|?{h?OU>KFVrq6R0OoyK;{;r$^m5=vg@4-1Q;88O~) zHUK~!K^BI163th&rAp3$zNm-nN#`7Avx~V9=i;iEn;^It-@`SOmx25cEZ; zK_Ki|C&1^v9}mnjcDc9(y%j?crb8v&)Jig{ag?y0<62;%gn zx-4UA2||Zg&AValy4^w}rmc%T{P`>K)lw0k@n@$kub4C}r~X+6!ic_$LXRQec6mNJ z#GFm7UAcXEN3iq5Q~q(j)Mo%sxI+hw+{%x-=f?p#k(rS-cv4Vd6c!*uu{o>wQYg|W zK1TPnvdFSp%^9Dho6P}0sILkcqzkZb)K^Ch8Hoq)F48fLGJca|OqpZdhTGEKooi8l zL9;RLIz~i$j_8g#mw8qdOwrTl7v=g76hBsOIavy0y6t2eNVY3`6w*?2!a(5(z(W_6 z19p5338>D4&W1KQ5xi2gx%dQBWb`C;3Xu8f0#h=KUiGm0g>@ z+utGNUdyn^FZ6BC>u?THp?3Nyk_GKl0&4-YWbA5_`OgCcKsG%D8adcdN-7=UnU@ko zKE{Ph;Nxh?ymBN_(0$wfCT_$a;>XM*X-%ASzAJ`n5`=iNwMtv2U%0##ezz(VLY z*g7#tJIVl=>MLNqnNv?uB>@L`5p+~0v}T!}?0S)&Y;@%iO<@~^?0td}^P1+kQ@zgs zc3`q{jhE*cu>huD!4KxcT4hGFu6EkO$(Aq1b@Od3d+al-aD^1Rej}lYIp-?x&28Y` z(gzpdH36xJR&$xmtk

013CdfSpEwf6p~*EP_M)1lqy*_d$ zdq@M>0QvlJh|)yz%Plo|4szd-%F*hre?0tT7i1hP0Kgb`=DI_x&b)h=0Ek1U=yhfz zf2IIGZEWq=Sk&@^Iq>kJu$M=`DQTMSms5^wstqgOl&nu3tXMiLEnTUI7k9q0ElH zjSK&m$qlRGQwb%PWFKkkCeSBJ|`-v6j!q zjf~kXaSA}-3m`zWpZg>`G&gZ%w88w091!N?u`vT*$IcdoZxiUM_+3+iEA*VE%vt8MpJdL-8KS@WM&_;&mPz%X1oKZ zzK;?3Y?U|iV-d@8A$oRcmo*&6?S3KHq&LAmVE?CgSBEI|66OZ4QaBTYC&I0_D|2p}a8{2*G z=~agBc9!c)5sdXFNEB_=1tD_5iJ9}XR<=wtJblG*>OEbp%# zOu-u9>0HvNIMr$7|M=yAMqu|ZN9RnKJ-`4M7?T>Gp*?-<))@J1s)0JwJ4*e@n~&vy z#);!tLax;SpZ$Z+@s@Ig7z!O_IKE&3 z0vcBp%p6i=x%3j0*CG~_j#iU^)?gzxG!DZACG+#(+x{Tbj8%H9|3o5SUjWS3cP-|y zt$B# z%IdNSlQ@#QZp9|M0(YdHySTg~LR`EC9e)5eU61NvM3Io%TcS2&j~I!uG`zON%u0^N zAit8#(^5zPeILPB>Qwg!2@sWcu3Ej#Ol|7fbLGP(9T0+(oD@BYrI>JS$R5wDqMggSRpcOiI;m67mSZ!`l@TxCOdo8GKhrU}Qg-&LE@CwLYT!Vf(vp*pK^Xm)MrFcM; z((MouCES9|%h0K0in%%?y8_jj z=IrUnAja>!%wpTS@i8S5Oa!L@cAm(QtKSZyzeqn)mI;>@xB9j*OLWdD`athPj~4H$ zj=h8X#0{R})8M=PAOMS?ouDH;7f(P$FZO=`9GTYDvi?U4a7=eYU9K;8){`kj(XSxx zibt37^j$%p&o@TD>y}n30f7fwCTFpwTYrI$E)V<#&qp1StM1^300nI*Oh(kee6)R1 zxVHh6*sD&F^){lEogKV5ru;PT<9v8+nHk0#&C$4_+to&gRz)sRU?zU(E2uiS=0bY!p7OjDv{O!k_Y<0<_Y zt}MhxFpkT#&}sB*YWFW;_!r2FMh4kKz@3z=h2qj+ z0XJXUX5fRb|BMNF^S$3>2*vFP_^2D5c+_9Lo%J?#r^ccBvw1JNu@DLAX;FOrr9uR+Nm65Kt`XSKL^ zaP)4!{Z+f14Yp~e#eeuHLI*}RJlgy$>Apdyl;KUF(USiG?|y@U@l+1UXo@I+aiKhd zF+e`=9;~rW382tM3%iOUZmEj4O?#qarFNdi1)D5?qtVY=>^TWB*ZLiH{j|2g&7jZZ~w=iww|bVuk4Gt9R6$DoLa6#Q!&lNaK$up2 z;;O?GZ7qMk5$AJPY63PGN^BHHp!vL?(!M|{?yp*sU8ai8+rM9)s?8OQk>Z!$ z#4O68J9LkeU3Vs9fUsj&dp8!Mz+=T32w5FN9L?}W>i>;r3f2F{N~D*}S_7{r%K#+Y zeQWbAWYTlCBLcFQ5NNsrHnP7H!#UA#fy>=cyjuJ8E?pskg+15VDu}85xx*FsY&@%+ zb&42>UGbbefwsu!Xe5?+Za?cR6P_SYPbXEfl)IdbJ*g3;@Ut z$hyE5FR8{$!BGL{qIv0vPPwj2y&h#D;qN?)tu5%$4!vMmF>k_5z=*x+-dSJ(RE9Tl z1!H(Xn1`R=`<3$l=a&6%r)Uxaoov$1qc;aV5yw{rY1uhgLn+r}u;V+9j@zp-U_rML ztnV9peg{2SX^bv=27A-u%<}B7OAHUbnXA=>+sEtkIqYj;3LhgD4~@Hq%`xt_fd+`?O*{m~Rmv=`GWq%cD1ys9 zQYEC^HG~q>uXFZYn>x*h*9^0kAuUmPbz*~IculDNWH&bRezNlXIk&xr(-&3QPqiJy ziB!2bg#Qvzelw{+(V2W|0`OpqpPu!GclU>btsFdXEmpF`!6<0}ciQh|Q*tXC&(!lcGRrgIcI*iqdB~o|w92Y;K0{Mu-3A0f zBynStM(xCN34b3W&;oWtgLW)u3uN`JfB=AGYva-2qlflq&1HGa3|=X3_xa5W?&A>h z3ebm;ho$=8)GHTYqS6N#GSuK=(c71_yZ_WIY$G2 zH%aKb%!iUheO?6w=GXXa=Svu+QESQ=N5ZcURKSRk?a#<1u)DFpzapQYBsw*JEniQK zlaa`23*=D+f->0JE33nhndVgZIX8$><^u~nTWSt38xxbsGHQZ-tCbl>euX~$KFJQ( z>;k`EcMRo39DWC}u#Ez%AVT%9tm&Z^zuPa)hx5>f5%C$gHc!k}Y-kYUcqb;Xv_6*H*cF~g4jdi| z%MEW1@AG3slQLkHmm3-;YbSeGtfkKMl-+DTxS58YYLae#Ijzjs-iOQ&X2zG@J1H_X zvB*I78S^OyV~d>61na2EjS3QRCqR_HZgvokNfHE#6WT<7Bv{N`I*OWdDZ4HfZ?20ZUivoWuhYn5(WRLV~jVjG|EEf+7kC*c$ykb&blEOz!bhOFS1 zW9GDjc6VX(jd*qUnR2yX0)>c4h{jdjCxms+g&F^RqqWober#?-jqSP8_G=~{Cr&q_>rOK zutu;m-#JYkEE;yiv_EBh8GcSQaWVf5MAt^AziksZ1(LTrwZtwfi>~uck#Q4$lq%s6 zGh8@hF-xYbp}4Qoyz;ax)nwu0P%&tA~e1VfE5|6Xmx7(KaKuFF*kR%0&daeN}ABU$~vNiF1N2?z*|t4*2HG2wi5lDK&a<@*~m@X zsZoaVW6sBBnEFux1*MPjSt*x4L5x@9X;KD-I({;}t326zxbO}ozVq3qfn+Bhg&PaS zz6Dem85lWWUo>|5f^zD#35pc>UG=UMLrG9mfk*Xso^PirMx1JA)K~Z)TJp zI}b$Z-b9MQtMivFl21A|n@1}2M>dykI6;V1JK!J%^I z(tZ{0%Oa8q^}|cLITpNzDc~P&dq+L!{2NNp2^EXa&7sY~s@uPF?{`8OFH%9cwqd`{ z5_*g;W6Boo%S2Svs!liC4T?~MQdp%?@ks#YA!swL`a;{wTzJTUCb5m{&YYiOXnqw_ zKM;5hmA&@wMp z8s&j>3`2-`F1>Cz0?;@)4o0Fy$Zk*^=^;X8HMoZa5Vt71g?a`4Sm|C;+mjxWTURyd zfahQ76n~;$PDj51P23Kh(FI;$WP!%R*yyD(pW*2`?ciOxz-0B24 zQWsjd`&wA|2pe$AV^@&BxSB9Tu%<}P*n9TEdDwH@AbaPxy!R*?^7^%4vnmRV+x@mv zZ7QV&M?z?e!Prc#>g{29+L6@y_Jr~7X`28QoIYEKUZ~+N>qM8s$I(#3H*Z1> zLmpGTv!r;q35ls&nUTfNTupsO5lm9PK8QTtU5mK7h$-!?rn=AjGZDwSyi_8SSj2c+kmDG?4Vv)QNb;-g>k~!fZc~_`?2P@9C-3g3f9i7 z<T786tqBIf?{t;92G^ygZg2JW(y%khL>{^`x`s0#zYXVcRmXc|2M;U0y%i zR#z|n_gQ-h59knYn5S|qaXf|1i1e+3jd9;PGop3j`ilPWKU_d#0z3KAi z*B&lervE5kN(#4g-WM1&*pq)IP4)2IRVrh$WCEuI_1C*+DN)+xGLXH7rE#wogJ15W zO6U27v3`=*OZN;P_AF;IUA$ohp|B!aotRukZbsdsC%&3e1m@ZE+rUJoLC{ICtoPE! z#-r3njwhn36g&yk0oAOKaqoAL&<@MO(nckz&Sxz5y_P>Zc<$f$y^OkXBnA1>>e{2- z=}lG~Hzg}>*>7%LE!u$XEE|V-Ic##r_`kK1M{}mp-w9~7-29_xH~=&+-%deiy&AoM zX%g)ru^?2u-R3uyKmeUu3yQa7z|FT!D7`%uh3I@L75NWjF$Hc2MHA}zlmveRXVEbn zj#axaYYt~>_LNbgkD!EuS2!gadd!%oQ?Gi#v>`Vv)%O}&zI(8(aVUFM%q}kXmugwE+7jF# z2J6qDzjo8C`SYLSHK(R(c1RbuLrm0%-N(v={US{C?=+;{W+(ZXPiN-Y!d_?=tqmapf0 zFG8h{w}XQTE1tzsLf>_BV|akKGn%Aq3(iD>gpLqzf64ytt}1=3Jz7A;!9{bey6Rji z`Ms!A#tH+O{l(^GB^j3Hula8K3cH7ye7#ekn=EL4+H9!Z{NYYcyosRS}n^HZVezK^WiQD?RO$|(SsL;f2)1)T2|Tae*(GwT|J zDMPH_WaMgMvc!8P0Xyi8#AtVscC|jA_lZbsFFc9>MI7aab(#0ke@d@$Cq%tbUi}kyi0%Jy*Iy; zS!s2BG;pitcYoAz8Ap60VW61Hq=dGzP;}i`8ZQj!(qFQ}_h55yBa1HrP)1MrZjEpQ zbqHBZd%$aa>Rd1TZKxz^jXQcSopt0V9e&azEM26VHN+k*t7;1vH;K3Gp5+oR_JQnk z)8Fq_-+KU^9}wr7>AC{XSN^@mUHRSKZrAbm0jBuS&eh-q`C4-6@Supj%p-wE2LbA( zHjnB7vT>(v20^=ZH|FrSn@VsM#!XkY?&DN;EV)QlP z=f-kHUIJIX_Iwn`dm;YKdvybLvE%{WXmTS!NCPPM#RB+rQcmDIj5x$kb2G$`uO`yF zk75ON9&4O&v}y{ksZC+zznb76h7T=LLCkg*!R1X^*s1FTbrL!(h~rd+8E!upo$@*V z%$Pw*;^9d`;K_sBu;+oQ>dd1N5*Yae9uM^k{VART*w@DFKTrqqNj;D?|RMVvC(Xtth z8CBkgaoazywwxNwdce6@8u!!`xTHwb%2v!ci&u*g z8l_i(TOy2evxcN4H=A+v3VZ>;-bF5mZ%YzCPqb2DFFhKb?N#>SIF}=sm+>7;Q!84s zF^Tp<#g_@6&2_52$$PzFLfd_^{Y#*FsfCwjx-1;)H9y5$nuwanQF&Z6X+>lG=?zVd zEvhLqUI?Y+u4vAprWF^@QO7Fg>V?+w=W1QJPzr{) z!`C8oj0#wM5J2M@F^npEimJuzXyg25`5`G?6-?01{0AJJlmicgyr_oFqkwMb&;AmZ zLh@bCP<`&(KVl-Lsdq{jF~;Nj)Y3;YpDj5!bIov?As4^vH!S%_-0fom22f?1NVni* zieQX=%LgYPpzxvE4S_`_mTA&!8&XjkmPhXfZp$E9ye1)r;Lcs6LZlDmctlkYTn)Wa z?7JqxsR{-*&$19>_wcj~-zfKG5TaEgA3Ww;|MD)5d`B^uqfkE^oZGJ4 zm)1{v_-C=Pitu`Z0LJwFPVdW1bR7P8CVJ0m`@sgy#B*l5vwkSXGxJNn1hf&EfS6ua$WTt&?dH2A_t-lzGs&@8{U_op*yo$4kLMjgZ5hze%ZOAaJ}U#CBGj zu5Ru(Tl~Z0FjWI)5NNF~Drl`DbgGpQ1Z5<#O_-+=_6*cQ8=m6=%iWc_;>KHj2(T;;Z}m!m0E2c8!(5muKsd46Y-}inUOd{QJ3$k(M-L}`mJXROoiQ; zXb15_WRJ^53mLcNWnD}{iKWF*+T%&WZ9e%Wu0FZP6}|5)wjKm3_RHCO9K=Z3aDH&D zO^JC*T^&%e19_vR41)4=B=Jjvp;Q{ zJcjKss4KoMX0=ilPn7Bz27c8Rqk=T&o6Jaipolai%1@FiBXVa+S)+`}P1Qeq1!MgMJ@9yOLB^T)Z4C^kP3WL`Z^EGcy=ec%G_`6rDkcQQ+i z!W?FThn?oDpoNQUcJO&?H+P5o2!0xKxak&010l>@Ec9ccVqw$=?V1RL-aVKWd?)vg z&1x~IE!K^DM&nL>N#R{}h3OB#h!JvE#X^Z-VgaaU2C|Eeg(QvqCK&l(4`mXo{Yxn2 zX-c4Xys!ET11)nWwapLF4F+d9Jz@Rq?0)3vK=f$uch)`Z7|d9? z-az2<*nj?n9iAuMq_VyTa{gjaXOU%@K`CKz#``MsVg<=9m`^@RCmXUdkS*>FMztL_ zbI$28^HcqSG@-X;RbWn6(2uDLbR_a63isXjAyWN8;Dimu%hXIM77~hX&IjofaK`kK zGCs|3n%~`1(OdP0g28jE|C~(M(Cqi^7nQVDkLege3H40G84+UIYohz?@qu^O{Xkb` zqLc0f|F051x0I@T92y(okeBMr+^$qIkwD%}CiogJ_>aNRjq;tve5>Kn7`pUr`@lco z&HbeeA}@IL$GlnZSW4Kyc+`e`hZnvVTuEmrmwzb#A6CcOwsxEt*UE|+@jUV=WMeJDhB`unM z3VSH}I&~NvEq)J}{+^XJnKR;_rRN=FeY|G${F_Xsd!G|1V|5r*c+>7jv(dQ__mWuQ`9>AiPGcgNm1PXCc~c|X z$B25wUw`dU-oIMg4i&gead zTi$OlkH$@D$3O$O%eLiA{gU&~=5La2ymXr`Zk`6WVSi4zIX+>`^x*;9dnOSa~mQq%1UYVe26q$L02;Yd)&>L${p= zuH6IarAn|4LLfhVYg-uhfD{<{i%XPKe%9Drl+Cj*lPXW7_57X1d1!@5H#u)lru%Lpet;yj{b67lZ%w-tkJ^4ZcVvKB z@s?@sdi$FnMbdFkRWZJme_m|iC%GlA#bi|o0b#I4u zyVpd>-N?dq2UE>T`xDsR>og&#*}|(IE>@oD22g!G;_aB^r=!2Y^jVn&j_-uymujw^yH9otOx zU$ZK37qbPSVc}rPf)+&4$+>r(TXn+GnLeZe6u8=uY6@gApgGbFman;|QXEhZt`sA{ zN48bLz(gui=pXNpPX)@S(u6!O5OyAp<>h=am`sOGR$kG?hF%s;hwm?*l2op{NaJ&b z2`i7A4il=aLaBnEw+vPlBZUU*5nc6|@s2gioYkKyBI3g+Q7BneEPw|u0bZ^0NA>dY zO^$i4ljNi3r$Ci`PsyzI&2vLyZl!Uub`1<~9R))6J3D)yu5@MDG~YIoKg2dx?ST@J z<%dhC$CFNc-iI!)M~7q@A6y^0xcpwPDz;tl8(oM!cJ{0Vm_+2fu72Ya4}SLr3>09@ zSw&59J17C;sa;kP9@GBpZ}go)y`C6wqWW;fE=Hl$%lxN$CfM&dNKfR2ppplgzqT)5 zFJjI8&`G_;z;()f4zu-C)yH)0Yhk%xWGTc3oON4$OyZ7VMXMZW0?qvaI7tBbf@T$n0cz8KtUvYZ(@p9&I*6 z8T`xMJ+l~n&%21YTpTrKWZi`Ns}k#Z*|K>X;CZf|+`9QG?Nr@`x91p3|Im9HT{KvY z|FH7oc5)hn=mCnz`VC_c9hw^M!ayA$r)iwk?(B6FrgDE|UjaioSJHjq_r2(gh&BU~ zAOsLp%&(NLsh$0nz4eZjo8%SG^sxbb-_fF>nXSnm2MRA1T1N(l|K>b< zyw2GD{FstpEvOmLd3ye`gQHXvosVjVHOAiq@142z@ZxCs5@{_Wlg7$ht)PRmAGiqa zNU0M&kfs$OeeX+$7q+nr>}6RWrH4#fEWhXWXs+7c{nOzPNraG{w;7U}VUNHZ4t^`6 zTP(v|9x+jf!3(adt|pz0Hg&bo4w?67r!GhkzR%dYI@8ZKKJEh!j2A;yX9`qg9({a= zRG}Ib%%J+>NH@BwIaI)D=fCImf~Z<5yaGlZUyPr4ej?0*zm87_8R?yCLS25YE@v!u zl5IhDO;L}HB>0XhEUcyTG#Jq0y)u>E2oK}SC9O2-M6w(jGVdEMLv5?WP8VzANQcyS z985%(Xw2Ve?`r+$@+D^wSDyFnWbGYIjV;*&*eb10G2|VU*YWaK#Y1{^ z=n2EF!z?AuT?Zi}^TTrmvl{92Ht&AyoGBvaIxasFl6- ztC)@|Gg-kq&0v7TO2%Td6M0XvEw{C;F@dc*!6 z$JXkHlQfgx1dszM^60bgy`6N#SWt}B&uswKY$azWee$Qk#bPXYkSVcPX=3hu`Ph3} zkTnuBePOZ5KOE7b80HmUGCEi^vRxb~+EUAzKX@g?UQA13d}WZPa(z!_b@lW=1ccxM zG6y6@7QMQYyf&;EAi)^|O2|>FkA46fY=uh@+405Bb6YvAw%|YpS4h63f?Tr)m+fd8 zH)fZSUcx}1v_dL?;(lSFOoXGXnRx{2^cXZ1H<9v3`YU+p(v>$VE`l*9`^*wO3|v`! zRs>xKp(d?wuF}|PW}DuAYO2-X8UT75Y5QwfmMIgoK3(Q}kJFieEDQA6!&eJJP3g?| z=E5B;?5D4ylnJSK-#1nb*(EmcyB|TBePnt7G~!St5WIi3G{2J3Yh^aO!XA2%$2hJ_ zMNxC0xEan=H|2>J5ASLq1Bm<7E~eV6tu5g4na`m!DzU(G0AaWDa?QT zs0RQ1VJYTF${OFaa*iWQ^gXfZYhgYU5^D`<07|o56Td;1=|q@PTJ?*kgOc-_EVj;~ zS4(uQe>DZ0Xl&hb#S|P%0TaRtAJv~UZbw~?_drOK)Dhokn{6jpeO2dQ0gC8>DjEsjdLvAacN`XD*E! z-&P@z8HgYX2puao-8%*eYg-#Xfg@RFa)sMSWJ`y=0k_U!1Kjf)Uu@hS?BHwnsKf)c zXT!OJYcD%iV87`x4xGH}a6C;QpuDN`PVvX3>5($hg1pXW2eI+^!6$M~XdZc93Ep}I z)p^C~Y61Hzj?pEDM@x;v$%0*@lVAB93+Ah~OJCJz+Pl6E|CB$Qd&$51>7`M^wNEZ+ za(_5P?$+^0DcNn7i~LrSP@8s!SRorrISs}fiSxdYin!Znq*Lgj(_A*T{Jvd$EjM3? zyg29l{r0nLl4z+WBZltIq)QJ2b!1yQ*|Hf~wt~x)iEP(EPU9mgg5* zn&l<4is3bB;t?7fjf%SJc&r6D4g9|=#i<$5Qa@EE&#n*OsU2@U)wy{(u}Lf%jDueG z51_sXeKMRtE|DNg-2N?m;(q@nubi_p(IN{Kg9+&5;Ys%LEUkQ812vQH?t775_+!4? zBa$44?xDue@pQ5Y9_i0yhGH3)oRy}TvLvcb)Q^cjF|?|Us-$sN^p01@7u#1)%{W|@ zljY2=XsAqis$oCJo~C=<>TF((5OM91zKElwNKW5HqYnm=bEe(OVPP@HK}(BPZE&QF(MKi@Qp1nK+uE)xVXOKji;9y6PEv%d zXDX}W8~Z*TA_Ti(Qd>*UG8@^m8Fx;W!4Wuyg~$R7|K}S-Jbi2Qc`GZ@}Z(al&57ghb7m=S}N@w&AD_CWSP3GiLfQSxEQU{P6z#n%K}t9n0F5D3~?#7}B|V z=$vsBt>WU{2+7hYOnE?%asW8?{0K4%v12k4oS3D_8)#>w8S)cE2Q?ohGVGh&W6KK@ z@CHCiiOL&cP*RvzY9A)-&tnhUw(D5q-2VKi!9N)*;`Jvw{Bimoe3QfhSb<*1C{{j! zyZ?k&@i+x-fAf@=Tb}=q2O+NhsmI7JC5_b%?#cx2ut4uGVdrSz1CmJ1gF^;F5Izy> z07X{07Q|g9HDgRzu3z}CU;cj`Lu>bCj|fFs<-l3xm$19%b2zpA^i*aMzuaZFI_CnH zG{f@^1@^PP)rXm zg0rI~xcT*|>pet3lxxo%tvx_g({1Cn6)-!DBVh&EgD4=fE?Dclt{K=jI0zw-UBjx$ z77g8sNAyGZ6BV9QFSJ*{sgmHWwexTNWP$)*W>!_&^L}sKhWgg7>F6KqC!b_N7u$wU zm;Z#-gnZ=Rhh#sp{|d7ckaH#Z33cO*xzCT?h9+bzoAW~eQ}4BBK?-R>Sjjx5eU+t? zEp856`D3rc9@->3ctiRR$AwuZu>pJM7eqou!VosDt2^IDx%-`sG}umepG@DKck=qI zTc`ctbg>8HAa-pmpj&q4t@9_>jH+s#^?vhpIj?QPV+-^vtX+-!XX$bT(a9j67tg-d z*5TV>mrt!MPzXtv;?Mv0FhzYlrWx2PWF;7X)?BKNA1JsrgbAs@+={s%{VDZfXu8Mo zaQ}leRU?zp!l8O{4(DcodZt(Kh%p0+v=z`W@rI2?30SPP7}4fxHrFj9`+9ApNesEA zOL{V-5RGoB9P3^=sb*z6*n+&FQIcV@)AltJtgynlsE~YOLZ2t%*x5J}8gfp&(7gEL zEp!3yECXF|q<@B??{;=*793rXPu4IU{eYTt8miP9UEKA4D;qecWC= zRY?-_w_j&9N}DTtHUcWS(WX5^SCE|~rkhr%xXrS=8(msgqkIRYqQ zB9yq^ZhPy^yi5x6>DhhAv6tUz-Zu6mOz^SOge~^5_?Z{T@YeNTBXy7>2r9GVjii z^dJba7s3Y3Dv3eeHP=8Z41p ze;GpfBe&)EsR8f6V8I=iy0%Mi6a2UIW5iX>An2QiE_H6-+XnPVCYr?^4dZdi zdpa^^@9G=H$<$pX2i>3qa@deLB-McG-@LQSvId*x8Lt;@7P{(pD5kl>X7GwJ$%9cj z=@8=V%;=ug8-#XC&LV}lkJomlcRuxo?pcG@zh2(-g0>UY{oxdWjyZ~e%}&?4K;1q3 zWNM3Tr_XtjRWYU`p0q`a(o!E5$H8SqW}%YOLG~U*Lluo(+FG1#Is&GsK)oNMAgton zSCy?E(3fqHSY$i<#;cfsYih+NNZ%$Sg^YUJ;;MLELm`c)jsCw>vmiT2eGhwKr?COM ziyu{Z!8g4Q+8>8(_x%MYiDKU|fMm2X4p-2aS+G!-kG!~Zr?FMx8@bPeS8F;B@x<*{ zgz{|{g1dhkh0kWYD;8)Y4A%N|;X=GLGdNdjH;r(=zF0L$KJ5V3U| ze2E&|AOG{v)gAAc=X;fTK+yEtAwRG**(_U*&h*f| z8wec(!n(fV0t|0Cu?XHArqejIyoe})m+=ShZx8GYPMO0>^0xSb0GaJ}q5iOnk1i*3 z18zhs+{RiEN|ByvMVQD3TMOBH1=+Edz1M&AqwFZ4Cikc| z5!O7=N|4fD^6?JRh~myvEK|RFjJj~vfO8Y46NG|@!~A8!fJ2L<^ZLt*;sae*g@&>2 z&qH&=2!F;Un2wZ(#y$!BBohm&j8o(Mg-2@;zDjIX(Z14Bb}psd^h%<+l~#v`MHi%P z6sb-4)6!q-dpS|ty`hZ&AtU?IT`ngW5AYdv0`m`c1$i&>$XXvu?CffON8+MC&wjDV z1Cr8K^`)tA;BA;0OwSen-5ZHz^uk_UpX$B#s=003Zw$?H+1q<8B95pyL&?+2A4q-8 zJgTh4Ogg|A{qwA$PKBZChlV11inf5#x99j`+&T4 zy5EVqGC#DVO!ppu7|5$pQh4uY7yoQcii<{XwCZD$<1rDV2;bpi6&_^=iOrhqG#q7~ z?@A5$+Sl$pBd-$LBVh41I2eirnuA5NLB9_z!dY+;E(c2=mIfR#lM3`E{@+%@ zFOlBF0Mp+C&5D_GS?vaYGHE>xmRQMP^2@NakP02OMC>$8jnEbE`hAX8GSgU>w7M1| z$!$qpSq01v&4@7&(ki`^W^BS$EmJ4-xq8qa`(#$=v>H=FP%ahaS~l-C67teYh!o`_ zYIwJw3S+$ng_){CYozXBWVeox(#u1=8pYcl{9&AhD!wsXAOUEKu1AeLl_@B zM3@{EuG}{Oou(rY=s&{t1qrT%W66e38VQM5!}Xii-40pJbu^2B@v1@a7O?qAOt$qG zEgfGZ&(B)b7CllUgz(@ugkmYukni*HP59|rC?m<&3fF7PZkCRFT&6(=mDp&sOG_;R zW`fuo-Vq1;$`S)!U3^F=@nAGCh*U0yxzhFAKNGEY0_lLwQgt0FyVH-Rg=A)ZOq|>G zo`|c%wJ^}i9z?3h#DcJwk+ja(k5ZQPLt7tjALvvh*#`sX7Uj12%@K7OD@b*YA=jq|ouaO_U7rFy(d9q_8IWa9FQK`mV`VX& znNq*df6K;(bDh@p)gO6@EE|Q)6!f~}maR{5EKyXB$ACVYl-MfJ$7(U&K9{a3HcODPO2pq~>2tsrNgMCMRg?e#zetdPAxy%G2A<=*QWMTx>1HTyn@i zYvlGRkbI8H5f8JKu#H%TSbb<|^+W zLG-k!lhC;4MnIE&`-r)@M!Wyd-c{2*;o$BnVl{ac5bHVi;j-X2-thuz2fkEPzT?C4 zE$t*x#2OK6h8SnMc!;6eHiEid81h_tI9yJMl(JYgI{u`rJm(O;uXOn~xk{?#VWtUX z2~l~WctBo7yG}W7C^?9djy^?jfwUBk<)&IP%*W&>9cvRD=P>w?G91v$=p??9s!Aj#w4v*L)~& zI9_D!H6aS0fg3Q7sdXu#2$U+V$nrm8czMZ*Pr;z*B}aCGX_gR0l&QxRG3Y|`^^V%K zr~{FrT<6Ax2UJubY$(u)t>FvjT|abABfd7k+>6lSkt}0nkM27FiZc(ZT+(^1X^&1z zDmd>-Wg>|6EIy{u#-{;%<7*aZk;UzlF?>a;$&d(LS_sww`#$3ac12;Tt9#^Pg1M;1 zLDxuSI<&J*75MT$^4-HQiyiuWsg5o|Z!y6rx{DuASNxZJ4WT1L`+hFfHGchJz;}y% zw+gtte?37ynhg?pSiW1xM7m&>OVF=}=lSm4TAQCgEu;;XICtT{R|*t8F5;UN%b8m` z@@{O6-@0q8rmS7=Grj+Q@(V|h;Vi$dXSX*a>D5H|JISQOzZ%g|U+Q$S-t*Y2$-S2= zKb_0#)qz@7|M>k|&UL~5HOIUR+ka{S>=Gi>vm4On4B++z{*?w5NBxPwC|#0VI6es{ zSTL54@!rdwv6yYo=&P}#`a<7hkQGD5O+N~W3g4!{#jn;ttYR;i?=NvLBU-)MR58rx zfT&UmV269L*Uh1pS%g-JS*hRzb8dQOfSM#nK}7>Pq+Z+N4A+tjCf$I3FRRx2(EY9< z&7vVku}9#Ziv7X#@5Z*>gP*9mFIKflwjoPMl&;*!!>3J%)-%*-YnLS~LuE3|VyS%O zkJv($zt<}Uf2EFGeA2pjLen?XzS5E7G<3Hllda1+#APsGjG4z^k7YR|;k1i0SKo(| zr{ok`9OK)>-@;yY+%)Z;bYoajBu~v;|CxMR_2vlUA$&M^E3xz!ox1S{>~1tYZ-6&T zxT~q&6Vd(x1X$vrua}ABDsf*@-6nJz_T-Xz!bBO5HLkcUN=V!03r9;dd@A8unC+0*XV z+#~v>H5h^VUPO0^2wl!71fpd7&x`NYH&KKMzi9X3trBeUKG!{=n6|A$Nb{*x>H8*U zXxrbkk6fcva_gF6fgT--^6NwbMU~fS^75m=K!k(0yN0CNWt;ZK%YS{JHBP;U7XGVX_dbD=`^`< zews_B`4MwogpPMUa}cUMsQGV9O-cEEmHL9bkM=I&oZSxyMEm;k0Tq{C{DdUAKi5BF zI|!FO;^!?5AwQ%G_KN|~ws+hQc9RJ%s_xJ$>yQdXsBsM3?JHcp&>UzR$kDKA8A zfwi?f`@G(HN6p|y4cYS`&On;YmM}W1ev9UTvSF-E5Q}P?>xwd1$6ed1NfeUNRZmH) zZ|hYX*2~%#`B7Se^v!nw2*~vL)-h z?VbL7pq*SsbI%)xA=Ojz=_?hVyb0s)8eB-x7LdLw^dFCfJ~DCFLJGFFUkTcn#!g`^ z$2yj@7A$$wrN8hf?n+qvLU%N8cXDLqU`PC@k&dVTnR+xBfQVaG{HRFI@baR-FZ?Ue zy$z2NK|#{!g8xQsIoychKlk2QxL+#OLBlbbm!LnfRj8!)rdc?wsYN%n&;;f@&5MfZ6drG?~( zbVJ7FKS{B~ugb{s6)x)!O;CGEf22btJpStSRM7l{wcn0wV}OTlzEpT^e4IfclKUqj zII5vIkk|HQLW$D%0pZ&`tntad9Ax!nV7$QXY)c*`CnV_8fZa8m5P1>029Lik3Q2H< zY%M#<;6}i10H@?D!ExK8Xjz_e4AlqTYYj7tF=2ilExm5dn|mIFlsHI%2%)w>Mdj(Z zPaeJrg^eJ}rsE1TfWwUMX3v^_dHbxN6#5WnE`8TMP^dn3--6W9^793;$HqrrWEtDKOW#Q&J{!= zR?^fvADFLR*m4#c^X>ZoS#AFhn2LXVVV+7HaFbB2bWr#t{qIfs zm6V>WwcdRR$~jxg!;^TufZ?m3pOv!r?f!|;Quv7WTXh&z0HFGyk3^6=Gg36g9((nHBVd`^x}Kfp@`#W&hE(|i)d zF@_V?Bk;uR%2l!0&EmKBHgq{y9@v=~UW!IPm^X$8pEL9ML_0u&z zoK5dOJ_sRol0wwVa&nX8%4XT0+*L-Z+J`#cptvOdY6j7FXar};$$}gJD5Gx61IcC7 zPsEYydr=!K(0_9YVfev_(cSEdM82bVhh69lZSZC&u1opPW)Y1^p|~9+OIq(tYp0G6 zl0_)|9^$)mOQo{4(A+xnsyLNXR!_WLyF)J_k1xPn2v`1c!ec}LZH8((J*DT4)>F9i zgqVWlV*71xHGd&_{nbkvJBB&Ejkmp{$IvIArEb{2Bz(vEP1(30*Lf{bb1WRv_&imk zocBByV~h3^i-Hrq&h6qWvs}e0jY5x@jHR?+#(KEV7nDBZG;ZL&BVFo|5hl}D3}^*; z;U(R2FixQbuh8x$iCi!}c=sn(wIzR^=e$I=8HBC#>5+2~V6+K1srG^Cyy1jxKR)0H zv*tv_(*sNuc$Uq!htqoBbFzkmVgk6z%bvApkdrQtWhitd)APL@6@7JrZ5MU1cN>{q zft~4(QA5J9{PcEknilbEWP%hoxc#jKGn4kZ{*u~^Igk62S|*WL7Dt3C(0;x|v~$EK ziEk~WVkMx+6B_sq`SQQH75@H(3cBEF=2>;A$xoIS6`*fxT&ssC`r*A!Syw`ye#b+9 zNBA+CC_L}yPW#AV9J~5C+f1K#nesX1L71op1E?QO7u>0xkc3qZH~emxMn?+Bkt^SI zK>H8>%{8*G88B4FXEN9vhBWkm9O46o+(O0pTRoJVH+XbDttwR{bk7PTYw=HuVa6Ip z3pjqH4~7V!>u;(cf^VCdCuFHgt^M(MQ_3LAnl^`wN`RxgUG!*VTqbP*V%mTRjPEqMuOF8S;F_Ew$7nm z5AH9Mel%C2%kl`QUzkBEEb~7PiwG{H+!b9}XevmWx_`|T*;T&#{6JFBALH&m)txcA z3Y@7UfSyp$r_nVAc7+xym)IMT^p||Mfa?#|47N&r#kuX=vW6VRO(L$QMbARI{`R${ z$B4nLBSO4zxAYtOaM`2uye9C?^BAu{?sv;e0U2~h5C15IHUbH}mU>KGu7CCm$NIi> ztU7h(OWkZ>2_HK!S(D)JBkqHb#5%IO7=YZQrQJKUM}qwASs(GqxF>fPlXqv57L}N!+-Yuk}2V>(+A!-Ec&z>=@O=;aSuz^ks z){LkGmo3;bl6|tXpbaQ9zb&BzxHO^^_XsO7e83aK;*)ZgHSpz6z^UBf~?bz(g}^)&y!Q2i~p=23@1W)+d z$c!11SS`!UA6-Z(f~qf4oI{55bB_~0(qm_!1m#O4=m1|^A~`UdTj3}FdYt5p+4|$C z6px=a&R)o_y21fe5goNi5uvg9n?WIxt6oMSs-ZKL0%ya`-x+t(Y!{l0kPHJC)PjKo&b=V z(Wfi*k03lBI-7Nfd4#9>x!B!7&R*O6sxT~ud!4!vQ2`-ruEuDZl?Hg17C^2S4ieM# zyP+0>IJ8hl`UimU2uY$Nbt1yyeTBs7u+v9nLLb%w(#>?$k8?zwrfvisCR#;aKD zt@+wF#jiyTAn{rJm_n4I*BSKczd>I!ZE(d@bt07Ks7K+waO%-wn}x#jL)&@!Es|m? z!~Mu1stx>AgyFsOCIHmpU1)c);w;^ftRPC~ZN+kwYSpmYcTm&G=&r6Q7Z>rN z=ftK8TIe8rBg#^Z?kJ<5u3axb3h(B2P|tutj4hKT74#jsaQpo2FyD^r+$M#Seq0BV zQF6=}M5J%ttAtXxSXk}8`%3~noAL+Ys>Xbhdp_4y3l5Pt@s{a|Uha3u{Bz6?EX|Lu zfl86qoVcVDB-!LvGPrwSlJ5Z;xT*&jd(0Vb_=H0)msoqtSfYx#b%vfcSXG4-7*m_> zn{aIdgzj>XM2^s`_k>~6$G6zB+an*nUw`!MWhJ_9lF`v5?N6wK2!0m@PPmP5&yJ#r zCA;-&P^2=_Y%}$;D$txdULWsm31U;W^Gn#V59`v-%s9!m6?iv zkU#75*>&-H4L6cyGGMJ*5$(rha@{Lg5_NywaWSP2rB0){?h=!kBcEvkYP-dh((-Jb zmizG*svUsR#+^Km^Yc5!fB5m}7LX>r(9a2t51-opBD&wzadvT>07pa0jXU@0+Lf60`u3c{ zrG>c-vzGYF)z->{P?|`8i%hF3%@2WJX!2S=UBmrwIqgW{S{kT?=O!GvKB0k9|9^~| zUx@aXoF_!XCzSX=Q3+62^Bi*)IsYmL?sZI;Rru?aAL~H>zMtKuf# zy*&OrYndo%wiVO6z)r17Le7CCewywGO)?vTw4a>VMJ68c=PeR;3vLmJd+_)0n=>nsW2*w ziuC*xSK1GPy{6b}8!Gt&FdW9)C38~5S z6<2FCR&i|miB&WeS*^i5c`K!|o+(NW(R^5%4JdM=8d%om=rmTCbxJH+k^f27eYTkC8xn!R13(ex{ z_seFMM)X`yX!2;i;#ILqwY~gOH7edp1HJbR-D;QAt5sX-xLB#%8F+%}B8Y{%KRLtk ztUr!`2WVxUbZHq62XPTDTnV*gD`#7X=U&fGCxCdufkEOvvDs5~OTwC%31T_*zz zj+6@sH>B@yE+APvagTrpU|3KZ-^gAQX>zfzcn$lQWZ1)BPbAO zY?p?se9_`u4$}<2QB`NW_;~Dn{-4d(N_y5n@c zlgqN;cP$Dd8}t$(6N;i+3jkyibm;)X`>)nhAnx*CIt37`Nk1>>VT0wLK>SDhy*tmf zd8<=I1+3H^_2UGX`PQeOsWg^vgUsZHJX{XLsaV`Vat*y)eyq@>|DAP1qx6LxZ*|ql za|7>8s;u(w2ffPz=H2ym)R!Nm<7ss9h)T!|VylR+n1{}NEcJ|^5_KH@%jpOX@=eiW zAMO+zZWe~3c4+cyds5~2179lQ5;e7j9CB34^&P@wn%QTFfEc|P4Y9%qn)FkMhc>yx)Dr2QwksXr=%ywxWp09e=l{T^S3iRPS@F|24 ztP&@GdFls{^DfG%9tE!vy84~2eTL@_8YmcXlrTSM{spvJk^Jtux^;sOPj@}04!pth zc!moaEYg7F2sb1MRnRGzwg2fZpT}T?aL~NZkL;Mts$+r@!rVGD#)H|3hMp>`l=r8~ zXO4$@T>zZ?kA<$%13m#7yA!K$wpVOPEl>XPP}Mkv78$rTG_Fj$CBxq+X@~nBWV`qe#Z@c(P1h~azuz+%{YO|= zo8=#Nd!;{|wHGM>>unsdN6{{fY#gJr8ft2mm9jefBwTN`Ut+-;41%2tTTY%Z{_D!L zVlI2M)3@8by2y~;gu1mqrSGytun>!>VPsFLIFL~-OxX_5D=$B!-{P;%VZo^tZx0-( z2BdOZyl0EaeNK4v_;xqSnHG`jt9^gQ2m(c#3eTFc&ZXzM4Vzl+9lM1Z@BbqYG!AIqg-pscv+oWF9O+Ng`MFQVAz#bvL(24B6ihWJ#Z)VL zYDIQ`Nlq7l3P?UsWe!UauVw}$R+;WyMwA|tej(RxkH)ZttLOXU>@?7=T6>w$^9|r87rfC z+i#s-riEIFvUB0e_yUCJnu4Uh(?c+y(gkn^&P5@Y(q?pciMhUI&gFr@w zJXMRJdrxuC&V&3phu5UKlpkIbYtUY##{Hm|@?e{Uzw9)5`(D(M6p1Z}=C zK)qI8Q_qqy>@;)Z3O!~}z32JRSn7QPxl4P9bGt*(IjNAW=hA$d7lS8h@_6&# zY=i9TG7MhfK@ODMqa*$hbkQ}CT@sGWK#DlIH-mpy`8xW5qfD`UkND`UR!Q8nkgy;T zeyZlC^<=N}t9>;#YvxV7DzC4Rq7q7F6{)$!-x*2CpxA$hGyKo=514>T!aofXb-V5q zxiRX(MH}swujE;M1^jmHG6cY`v{{+W=`x5Dj8twt3)zwM7Lg3xhnT5zWxYCRxuKk7 z#r|6@JBN>#n5S7e=`=F3?}*Fgv`Xu2FAcV2ANaZ2yNw1!epM;4KFM>$kc(FKeW{Kp z{J#_)+{qHLw|OT&3!QbX#e9+b55{rA{4d6#XnIkNHaK3x=pM?Bw^l{zknik8-sksv zg(jRX@xH~9-|!=i1R}Pm`a4wtAfZ+`%VH=n5u?%6YB*eNF2Ib)ZBF%3*(`8d(E|OY z$_T*mfgc3B0j$xc->swD;)ZIRE@xlENj{C|Vnx033i7*Lq17DB(!-8p=QkOaY?=0$ zE`Fq3DS#4kH$Db{q^Rs7vfB8j)mkrC9Zg=nOW#TjtLQK3K4}vjP>>gfJ{lZ$AWt`> zCxJ0am`r~YP-|5xtvU%oJNy8y&kr25XctyhAHX5#&&xngY8mf?Q7Uk|lQ!s-yapTF z<*zezHzS+}^vUn2r+OMXIbg^W)P2OBtT>9#Qcyxr9ka06jrgV_bFNF*a1FmpiO|z@ zAo#>~J)C#;2&ecc9H>=1;8GG5=bNDq1MD6bbzg+PZbeN|?&qkP>~wpkK$JJ!GPd{M z*L12#v7!A7jxhTa(WNRhc)|SUe7e#YC{TiN&-mJk?WcM5V_RjzVX!o5b_Vf-OMZWd zRBTF)I=-#sl&zJ-VDq7~gSM>(+i`rGE5&8w$Ty0%QbJZ#NtC7EMhr&4R!ce zfBM|zMiJBsqBZ5tm6JDlKVG!?T9fBLB^r`Knwh5`=Vh5IXRSWpjqvO6b)xCL7)yBpQs4>b||Y>`XP=$jqYd5dVR7+ts(&xQj80qxck7s{%An+jU3 zmFMoG3}iG=f9HM>pE!nh$+geaW<+-Fc$cSeSqC|9%HFz*q9oDS!M#}Xj)T=c2YK6x zCA_|+5Bbrim2dJ);9SpATpyB~3r9s}ih<%HTj7St0NetiYaRa03`i(eZNY;)3MK@C ze*!thQB+O~fP$X3yXSRDR@o(n*bRGt{i^Svx+Rx9-AfQxX@-KJYiactqHEHIue;jF+XVq=m5_QxO^2xK< zCcUX;->ZcjAWopXmUK<8W$q|{6-66cuN>Ld)1SqZw2rSPwvK>FN?vD5EdP9b{1@-i z9bIfdd4d+q4z=D{5320<`W;+f>-}?Q!}s7Ls3D8D+TOUoF)*J)=fLf&FC!a{;d8F0 zVjJjQ_j(ojaX`Xm(;e$i1wuZ{B8a61UGOHug56h-y~K$YMJanO91`3874nCX1UIEo z+Wyw>yeP6Lh+-C{$-7O18)>3&32gQ`X^(YKhvC8v^{cXSn)N+5J8G2IL-vF%Uslna z%*g4#V6qn3gi`~<_`UHUc^BaleX;{kR*83in{(DcfNE~vW|qslZ`9yQEYPcxNvt+) z2#bP))WY%BB?l3gQ|t60SwakU5Ok4~IAmoJv9wzJd=W8xAaT>A$vD!ekl^|EA&cFa z(>&0{blOt5bju1!`kiKy%tk*&%sZ;iFq>=5?HUK#)I9yoi7VKvV@I!>()G%i3(*}J zu2mD+Z{Q-nsrwwiWoSaOfX5(1Tx8DV-*a^dGkcbnYf zYaPDKhSOzArWZk-uqt(ahDayz2n#Om)uyCpA^%eg@Sy)h4J{~Rr#6|tXf3!K;)7)0 zk&*Y}&9U^ZVzRMGc!_~>_LUkf&FPj$Imbx};n~u!#?&IfX2n|=Fx>xn@m0mn_Q-Zf z{xFX{`^u^4D~MMB)q7Q>~4ii z;{9rJXk+#zQM<&rJ>@j0`MMoZx$KpBHecS<3sC#Vfuz;PHo8%P zr=prP^hbi5Z4)D!9~GHgrPwx-+Vz{pZs;5TjUCoEyKD0%L$|EZ3St!WT`%s7D!*Y!{;a)Uf32>VICAHmfB z?t!aay1Y?>1uQQ=ZtmP4tb!|2GwLF(35X`|U9eacw49f!m?A!n(t4oXD2Py&rBhmk!bG^N*CfE-=Uv$?+1E{(SgZ z$eWkl`uR~zq`Vq7och*H39^J^kV5OQ;7Oc^AIY{WK%TZe`iYl=ox1Ox$b&1{VG5{H zW%?SG9L4gJpCJYI2Knv;C+4Sx<%u}&yGl>XX4wLZtiJhIRFHKJKuy+C)=G}Z{2T5h zr-O7h7#9d$v_sxZPs2kD#3lQj!U!d=dHlnd>OwL5bLwgkPM%evOS@R1qD2&mYSa` zPxr^CwMbI}0#9`F5p+Adjrw>Iwf8@h6f-(z>x0945^sR^a@TX~ zr_I7&U%aiVErRkZTLrGKf2bdFCxC{DG4PSTA~|OV^O!d~PGF0jQyrTz*O9Ay}<&+0L5PAX&HqdvOxH$QRgMPUpI^WcKZ0> z6mtrBncSB5_`9u{_tl`KQwaCJ!1TEb)3m|5F+g#t%+I;`1|kj2nnqsfX%B+4)3xs> z8z&@mw}gJFbupV)w!A-HF+^nD;9tE3;xp&KfmmZf-7o>C?#EL+r&kzcOr@yyWAw*|3FkU)ld2jvk?K{bhjTago{vA{;vDOIy&I7ypn-lw^r%rvd z3N%-7_Z)-q>`}3YTjjZ5oQ+4hlXBELL7NMsZ~M9+odOs5ps92RRFw($7?RvD@CF>z z*Y6MYOZ;JZ=Cty;TBWNjQnke!L3r;fX0d=sg`Q1s&Te}gC8wPo#1K0{gMZ&Y&+55{ zWZtHiue^1uU4;hvGA}@3)mzh(UX(TOU<&=q2Y$b17O|1h>D+Bl$Ts;O>Q~2#TgA^R zBc83%+3&Mg3Y++A<9irL{+hvTSzW(<#Gk&*hPS9|>4dHnk5`~u3mb~|0| z1_eRAKjmY0x(E3FIovj;x-W^}&})kd`(~(Qm=G*t9-2|_w$bbDZsbzJTWg^3T;A_U zUa9;+k-pW()Ejo@$({O%sjfF?;u4LgCDqYf2bUfFL@TzxUgu>so$|`$|8NqpoBS|xcTZK(0_Q{ zO(tYyCe`dG&z$-^qFrz4{*fvM@nsNL$1-~GGd zE@AFjE6*n;QgL}|KUdpMuT$IyQunUr6tsSsB`ostv9^h=4_{@Na=?>izS&0qPco!* z`CoYKZFpHdO7`A(uZC%&YMst~R-8cb=X^yk;IH?WrhTrhWo$V7X8UnZP%ToiT>ra* zRp!SGZ+l+lH%z`+N_J)sxSYN1yzv}KMNvYrPoPsEuYKkw{4De=_MwpsXU;XOA8otm zVqdhabcr>`yCUPSqe>ns8%rGG6IB#Jvdz7Z#ZcS`HHh< zQD4dAO?rKvM%4fvQY9K!^sfF!I9I{>Tf&bPPg7-w?Vo(^kzDSXIh}ozS+U=M7Dh0pQiQtU#Y|B|B*V_ z|D)GNX4N4@+?)^sDj(9J%m*JkEmUexwI`?6J9=dlfWx~Kg7Iwy?Q8Rs`>Tsy+J&-! zF}S4Cr}Is;l<93x1D)5=dq2M0Y794N3X~uPVY({|T$2`PUmQNk&a5~Gf_}oZoGZ)_ z8E=Hnd=A?3R;Mc>9Kq{+rgQUI4oPyTl0{Q4GSR5-wUDhzu_ z{6*Jp^f_wi^CO>nh8PPik8H_?2ags7e&jidKYQ8#$>g?!d+fZAH-7lE_MX(eAEkVQ zVF?!bQMZr2h)iyM=&*(U{7dOq?%Q7!SkGo%XTkpcL&y6q{D1dd{!`-p%??y37LyNr z{UM~4C%?&x5V3DCL|h-fHyh`D_UJK>@))2D7Ubn$8Un$-u+_WkNgbq*Xi%Vmb9e*q zPV}AlT`k{U(nv&wltrsCzRF{{KVrhTCs&sv`~}vEzPHFZ#l0~5_}J$_AxNgFJ>!Z` zIFF>t8j)2&c}n*lwQ1VFv$i~Y?zPiF;h)VLpo!c$=4X%(lS6tl{i=3sra(nJobTWl zYLN@qkx%x=E_LJ+g~1CJv7iEW6{F$G`*k4hL5FWF42{_FJY&d%@^szmtd*yfr8^s*ba=+5H|X6<=!brz4?zO z^+GSp&n;rRj_(CCvQPJZuNlMa+_pN5I!3DRdgX9s2`12m^Ypp>?ffxfmWS0hsXqls zS|qd>NJjbW{Qb(vYyQs*;ry#hmvk?|SCTGxuDa1P=1$Vd|I*&0jjgERSXQ(`@H(KC6S)T5R~}V`eVV?EaorY+k?oq z8`S*eKlOOOKeI1OR(rFw_E;5skMf3#E?5pSJ%Ii5##lj_?@8&el}rPP4}3Hy(Yvr% z@J;gMP}~o~6$ct9k#+GPco%|y9ivLdcPD{?W`zT$m!8A8Y2Xq`Jog{8Xr|9Qml?hK zkih~I_q1CvCCfvjXgz$uDXb0pFDjQi70TV#%RW6!_&wThn?jzx4BiiNkCP|Kdr{Px z|IJe>nMgQee5?k>&7e3ctkp29S?k1vQ({?BQL3i4kvp)sRzD%7Qo?vKm6@vBmD#MZ ze8x;lrEzO1^3aCG0~6@d6u6F`RlJ~Fzix#NpJ+WjHs{);r)V99#Ncl8cxj}1#fidX z@j2|>c(^G3TmNky#Krzej>R$k+QYG!GkB4sO~N8Y7)AcxfYQU1+!bbtKD@b~ex6|E z9hE!c{y|mzkiN{~ZquCyIjn;{K0D7BS(iQaY|c*FHP#5bKLq=+bn;6^XrEi{b4??+b zP6{sZ8Z-EIXc&o2*ukz@?K_DBn3MtyW^@HfgOSO=Z1h?lWe z>+EV=yy|TuIOeqG6#J$3Wp)r+zqNh`y289Z_y7F)Z|v!UAD1p1Ms1%mPtPZr^!C#R zvq{41ooo2-d>BHs$lgOatr{%)tlRJhq-as@Hg7~Le|e$ZZE$YN}xP8roDX-Jm7I@>h3M4FO> zvK9M@*QvrLFTf=&be!}A2)A3yJ)?D)xed`@lgWVrriPVMj%v8`i3cEG&31vFlhU`7iSo(}mB}@8634l|;l*AuF zHFc!sCzl?`FJ8BIiwrvKY7Cd~Dr6jTp`~h`mp*I58j*N3W?<_L3TP9%=N_Tz^~AVY z>`!Ivh+yfCU_YIsL|vf4!+xPseDm(b^*?+p%Ct29*K7n;#gMSG=+z^Ju`w{*Op;UF+ zZ?i(BvTzcmBz3$*i1IM`n@ed>H!Bx}L6=gFLB>LCMouq=FQHVlO>~D0^g0D4O-y3v z1xrrO20fJVTdgfnqhB@9=6?+*{)gel=p_=r6)v25_2@UvG}G@{XMA?k74nMGPGHxf zp6jKU$uYin>-W-ZUs0509=k{nh-v40spUcrVuU9;KbXQSCX`%8ax!q=E( zWh#+pz`H?Rp6A0)ziF(=#H>UGA_Ii*D}fO$2iSA}gn88Gn@E)rijpE(p;a1Ax=B1D zS50D8_hzfC+vQ{^irWwSPuOt#_zEF!;@KWtb$kvyV#eGaQgu(lmvS{= zk^y3RMEH6ovj4iwPaT!d9h0sW?^v0>>wzXk#Y1=+tfs-{O--~6M|Cd zvf|^tbVn}KG)~qq>ipEKkxfuyajU&Z8@zQE_)lw<|MFV`aWqQagw4!+oK0FD(4e?Q z1l23)nxb38xR;9|H;;f%v`u2qLWhaEE6anDFmXE*XFybSUb=3Gmu}%~I@U?AQNH|2 zjmY4h=x7&X_@}VU6&2V2^7sGmnV|)zi*qU18V290J1VhT7P$|l;FYQ|Ne?8xS7s^y z;!eeJ5a1Q3SWj)zX!bk3{VWI+!(&`2LE47hn*iZYi7L8g9 z*6ryZI?#1G(7`*aseaN-$L+Q91}U?U7D*&Fu0EiOgva0C=kVV+t0`Lm7nFCEX1ki+Epz4 zzT!)SS~ooBIE@oELDjbXPp;x0PhI1HRy~U=N%JFql5>jpRjT@W)*><5-ML_}SE5uY zzVE<+>egpQl^U;LPqR-OL$%)b-c8G^B$Oy#AIom!Np%|d=tf7FC4>(@;U4vK%3L5g zS8IHlsEOTh{n8lLR%upVzpQGQYj>FF&xbAQ;%!;H6_ACgXeHH0XcCBem;JBP{7joG zjs>N?<^x5VV5GjNo*s~X7`qs}nbDdwy&@Ej#mdLEv>0!}H#61@Z~r=e+26IL|MB?0 z!E!NhYaaV$l0rBwzq%-YD~H0YJ}|d@(n+PqH^o8ZZ(k_$uLR=e6yRG$C&4E*${u|+}dEq^FBmFR?q z^iH^Uzx0r6NV)Z;+0z`|-ubfEHKQ9{!AnsV1r3(HFYdTgPk?2sC5&?|Ccew{jB`{B zVuo|V0Pg>7X82g605g0p?C3>16LcQ(_W61Fs+N|DOER70jy-M5g68Y=dXY5TVtKa| zbG=y%M4b{3rQIauS_MH~ERar`?lB=jb|hdk>b`3${N-kh$+FIShXGRRW zlnLD$2h)0snDss%lbYw~U6u#bYgmk{X-N3`Z3SF)BA)-&u(Q&Leud_BkM<#6qqsB= z#W}l`v>Su2wnvT=14=V_iJOryo9AIG%<$gDz!-o_78-YrO7wZEJKJ|jFlu@+_%$b) zUv}I#p)Qr`!V8bdZg?=l=Wcbgo2{o#yyM z^T_u@%@Yb=0C+?i{@XLn1U_Of`-c7dJK8>$*U+@c`@>D+xnN8shOGyp@JZSv{xJtM z!Q4+w7Gzxj6w1%RoGJJ|8(&oJ6cSCmN+G-yFl}TVL_@A z_1|dObzaT^juGdB?q>@x(yA!EVzwm8n*=|L%u(I6z!^Tdl;Prb`8cj|>A0k;mCI>m zJ26wC@@9zf+la>j;5H?!EhccPvFQ#I=mJgi?ta<3Gy*Sa@X+QqF=5NDCgiMmV?8*p z+-K=n#&QzLIt&|QVkV1UXt&`5-W61KXY0KlT9$UE7nZOj%1d%wO>p=R5&s`jW2krQ znm`asCKgrVjPE(+HF{K>)V?hmys@&E9zHBTHNWjUw9>Z}Fn=%8xo^T1Ak4V}zQ_HS z|G8hWox;JO@ae##ogYs@xs@5Mf#!`)yO|&bmqN=fA)&kpU#71{5A!$-?J+RBK8(4M=q+UIvlV_uYO zA^}3>XAy{0KhQiMUa-u*=At6rA2n{!caSCSWf!M2rj8XRuXxZ)`t_jCy;gY6Xax05 zJ4a#2j<=FS*ttGiBV46jx7jz9Ci_s=+cR*HM40tNrZBhKUjApoA^-{v)`GRReY_~D zy20b4FhWS;`Syc}4>~(+Zf?9(a7gvdc~NKL#>M5lsuf`#OeJvx@^0#Rm3Z*M&amA?0HpSny zgV@i7*PBko<4!>kf=P}T^HM2%$;vd~GF;vfP90!M2?saePy&BA#;GNSlQ;d_m| z2?1uRgUWKvB6uaTw8Tr#QLHN9tY+yr_Mvj6~}Iw}+dq<{kNsUe~{}oaKWDuDRzG zJ|`-4>guLyy(_pMx(iCWUjS&r2}DWsm~iNbKQvBq%0~hmWVX$bO^#~+{ioP?8Ot7Y zo*66Awn#S-yZy~_@%gbEOj(hQs(fsl%BUa*u+FXiliH)3>9+v_tj-YZdfyUFPQhxubD8pg1u9vBHBcyLyZ}IZPSZ5(tK<&}C3gmXdpUAn zV>ZTB6SOi~8=) zv>l!A5d}!^crtFAG5*S8JpV3>xz0yIY%;^Ezbwr#IUi|S;CX*mIbpLVm>nur!5wYb z+)`B#F2jDXYpg2zT^Qw+m|3oTnEOa{vfCZ{Qzg!8Mfn`$LAxzR5QoRc0jhMnwriTT zfZgHK@XmQ^CHM%b@Ior2GIs85s;_k0Lg!k7Pvckr{6&}ZrWf4jac7jjN%k9It(KZ% z0@UIj377ZuuQ)uu7$+Usbx8OS@1$S_P`^D&l71--yC~^)t$6S20y{Lx3+^(NSXoki z-m=VX*wHgTW{7buKm+zM-BQ`fnJH&EtrD77s@E(2x+f&i=gejHyVuc266P7%yS@~z z;o4V|nW;9NnfsZT9ihog!jEq~C#fW@R?ub-VJVGj2Q{Jms{EkO#(Scc~um`%S(P`X8 zOwI?tZHVF~-crIk87uCVS$%82FVMto^J-u(TBVeAFW139@6mO}&kbo_2(+9uZtLNZ z+bi}E9-F3fn6wC)vh*2}7x(9dFL}t71JG5SMULd&ygSc#UlIO+1;AK9lm$s`Hn#$ z%<%YJ*#yb1N)@&x5%>Jsxx(a%`4=d(2#JlcLi^V7=VzJg*}F_t5EE8gXHz^c(lV=_ zE>@R|zwlnTX4b6tWevKk`KCRoN_{-=jiUcTD z1#*;B8h%lB3H#-9kxUfit%MMHn92k)%$5E|IL%UDLQzk-T+0PTY^j4>axgFH4qAfm zL`Ya2K~nS06iyOW?;zirpILZIm}vsy;{FGFE0#;x{Wq__&sF#G(a969J2wUiKHmUI zDtn}(HN37p*2bdM_8-o(?ck8H*gHRtp0 zAjFn#*HF|lj|6F>AIfL(k)Ur(wDIM@y=Pza-QQi<=6>#k2+2(5Uauqpuyy;vs*0JY zy;-)+Wd1kqmR2L)S@&N&ZFIWzj5Ob?l3@snw>fNx9gYPhNRM=qiWGX_TuVflsPpN8 z9JdQVLnZ|yaJ;C-W!T1`QVLI~e4r%Y7VcwFzbAA=b4o+-rBFBHgQ;U*HoN#%0<^5v zaa}H#y14-<32mSk?v{%p3;#6FV!tuH?b>iL+t7U2`>P4{k^<%t;D zeFg0m><-l)d{tUMYXhL8P2^YJa*}V_Z8aAZbKDYHMCOhjCS8)}AgiL7a;RTRHVih( zx&jYdM}s>8)4+v3o@;@(o)!BJSAI3dmtc%H-YR`&uD5K;lW|I40YOErZFDqe?{9yG zZFflGk%=1m1y)?l7X1-i<5X+&sPv`x z6TY)|3a%T3>^13-()WD_p2VER^Gq>o*jPPzPRxH4 zFp&^K|1rS@Y^a2r*U#pqmcg?0CEs6}!A61$E2=o4*J;IIBAc=PrAH27 zd&z4uv#3KOYAjTjCt!>?PBlcWOBT;_84W0uKm&OSMab09=)`$yXwFj+V!)Ur9$@SF zrQ>HOYbnMa@`^(kFWWQtV@ls#-uS;>ti5V`KL7sq8C!JLLc^IsU*O=EkjgnwVdY^5 zHPXfJ3;J^PP}Jagd^E;hqYb8 za8rnn6^S}biuwSQ`a(==fAyCGJ02c$5KfuH@~-W3KVM7flJvsXl=-9pq0Oo5a|zyB z2{QN~lsGLmiJgyN7A`Q;XV$JL!$n47z{Kg7@j?K!)u67W<;Nufm=skog!aUg2z8~_TT@=gbWXq9J4dR<+Et`zrJ$ogsMzbkMeb( z;sBU;l*29J<%4P5RxH$m9H5o$?V(+>&Y{MuXiNMl)>HWCg?Iz>22!u7CNrp1#6>-; zsfURAdDlObwVGMOIVv1NpoFM+HuGZi!Vis)o&~rZ$ntt{EsmHyiv$gQjT)5#h5@Uu zaX@XGoOjgY2%DX3Z81-TBq#8UuQnIT+7!7_bhZ7FrS+r~9nL$G=04#+Qb|D1+-bob zz{=VjA+ZVZlGwi9$ye?!DK+QT2#kAZ5%Qu!ues_hs?65SVnmNsgWeLcU;eMS@tbSv zDg(2|W@XuFym7?y(o?4#jL=pYvq~sa!tv91T%m;2cGQpL>V+gPlUo?KcHq*O$$sjE zbeLg;4o-vcFpTnc8g>Cy)^TB?Due6jJVL%KkKWUr6aM4V`&)+hO2LJ4!%id$&Mmc& zv=d5=14*q2Vd*<>8op9@wvrbiQb+jbx+`N0q<&K2Pf!|6Igg_JxMjGY-8d@_^2m3Y z>u4+6>TiWE2l>Ljp3Zp$ud^0n;e;Yh&U!b*gA2KFz;k+^HbgIY(`l~W6VmwB%)3 z(_Sc5Nz^-WtO*2wPrZ5u^SZZZx!3BUtAHy4SRTQgP>qw4ltBW|74(XgL0pn(z%0;d zzHo?yl$zvn_e<;wtOXMhvU|cz*v;?{ztbT8bIsTVpk zTUyT2Xj~qla&4BZ(HJFE*ix5a5IdC5bLt_oD}Jl z_UH53CGm2f$vLg!>6^v@y>;f- zI=iGj4;pGTLzj5dGuAch!TY>t@Oeg@^W1DRv)a~tcPpIw|FU<_V2 z`!a#HZh(EB-OumY0x;akDl5p)7mpehJ{s;>rB)kH4N3KT#drUNJpm_(`c)KkBW}=` zHJVdzwyNy+$>KLv_L2C^75&nk2(XM+Ys-2YTx?hnVTQTMp() z^Xk?es&Xu_z}-D{<8s>@@kj8{m^_fTJ#AdZ2w{*pJI6mFKDF7e`x8^k~#~jNrdZ;G80hAd-7dd z{5LN0AJs_)ZlJ57%S;V+e~|^Gp9TJ?kJ`h9la$VPqmsrP3O59d2$?3AL_nQ0+}$P{ z9Y^58YxY!pxDa#wRUkp2e>;tD*Fr6K%o24C+^s8pCN?!!xw~Huv1oMi5pO>AJwnn^ za=wCjsN$`w#TR9pQJ|?y7eNO9o*3D-Rop|amzpA!-yAQhp%LV7NiJ;H0ljjz8A0#N?WBGE1UJf_Ug*XMO zxP$txq3&*7PSY-2pZYQ7)D zzW^^w-ovVy&8%lf$VB{H&s>U)-OSt30^fFYTj2n}tU&(M6FM4n`T*bo@qgz5eHy@i zF8QM@^@?21UGHv&yl|UJJX78&Xv3}SrKvXH`mMaS%ivh6&-Sp4*&ESDYTep*&CBRK zBGX=pGa9qVgv3~&=+%|r!pe^M`^ACaPe%NYgNu~Mo`5NFy+fQ(LCRyNLYMFPa>|=| zib-A)YoApt{8slq0}^}#EzIyY`XLyYbH3nE(u=-0MG=jday5AxKXOz+~B{)S?F2OuWjL z9~OXBTi@65WNsWog(>f!g^YfNWQbv^(_k_#i?IuNjZf>t?D)BJ0b%wR%@-#qCCl2U zqJlY@kdl>ar|?31QQJS3FVOaKw98s~*!Iaon!lw_{;4)|>AU{~l%mvLR_v*9kuC*p>eud}bM-`P}O2r;`5tvgasEEZm|YyV7c z2WQs8Og(&r1UU&X1;w&%w=q*SM*F1kJmLV}8khK$rT$w2VqgT&G;uM=uRV)54z>#a zzK2wB?gg+DRfwY)9m4gkZ0QzhSex|b=>l~h%Uqg%{Z(|-6)13z)A6Q9diToee6cFq zF>`1}lp2prW$qh{D#N3c&zcjMoS#_Ppt)dW0GKnR1nehCYrG-_A9NwL$E+2Aa%P=D>;@x0=WnvbSoqoSPQ9cTZ5`wLtl(ve0qP5Vhf33ThgsljBD&1= zGTE5;$(d>0FYlKbv)^EQ634@o;EAH2yGjshl|x9Ia|I8%uHA?QJt$^nHiA%Gj!i#bRaDLgVFbvf%wXoa?L zKA`XG9(b+dhJOyoF|e<7Il*V&UPcFjk?D^%3u^Mrc^jYW*4`pJG(C|_1nA8lyAov+ z5bt->b4)##G3mz&VV02~Z0g8lXfX$w(&yuo1`dgKha+v)ztHu(AEzJHSs1CgXu0A> z+nM;tb1NDY)s`?JYL`TmBVpxUy=ok8SVwy96lH2o#-@=jtQ3Gf_|*$A!!-5k;1~1E zyrgLQ&HAbV9KDMqD)1;IJfwmj7xh~(#y?B!QJ=tXz>28>eg!>#@9kgmG$4WM{^^uQ z5i$Xr&pyqBPNjX4VD{l_5`w$MXuw%e+sV`A>p$V zxQ-|Y5^rl4NZwgawsrfWsr%}PqZ<>}XDLaI-=Si;dOszlL@{;9c1UxXg+_2 ztfCgPlV3O4VO|boxF;k`vi1Vfl2Uq&ikF&)lu&&R+g+}{mnQ>?J-<6rB)WW?r74w) z##@!ht6c1k*>SP*mwo&%hQ06yuOP}cMXT9ZE<^c9GKhJ`L;Y7C^v-M$R?h`&xce zr&h&oJ+JrNJF_pwX=~(AYb~c;h0VJ)v4~pN{HGgAD8qaoyfbrjpo*ukx%wTxxLIUk zwuyM(RS7}i@81JAqf??3h$CU)P=-ihTl-jmrl+!^z*Q!4@+QBO+*EQ}M_LAPekCYb zPr~|+BQnl|>n!o>&JBL%q3LVhX1AO2H5HE#iwxi?xugPssL-v@@1<~7X!v3{lr*nm z9B{kLbN}^bn*^1L0CW%$o-wnw$j0N=(LV#WQkUXpgYYX|z)V)T+oplJ9+%8qSVUhP zrZJ4Cy=Oa2o6oHhtxJ1<=-eu{Cmb~JQF4?x$azfsa(UFrpU1aSzXhe!ZofUC8A;*L zLyO)+^-Wt1siz;Mj0`C9l8WdSR1darj98-}Lb2w**vBV_Nu!ee-Ab`nCF{kb&MIdr zT{>Hp;(DJzN?7dDmv>JSA_MH`{3>vBt+aOLOI< z={r5nui?WXelu(b=C_6wJUoT;1Xro)&%cT@s?9;pWf;EBnZvUz=qw(azQcCzR;10( z>t^M2q)Eu30+Taej`e?AIXV<-59LY7FeywNx-dK5SCN8*)Da%+<<2MH4Y?8!N4dj`746oCItOu)jA+=Rsrf)E6lE;neBf1mM1w`cwQ`?w4*oTpK?1Pp@x!QPtabU zNV?84J+fQ4j(y`k6U)1GxpV*ls`$Fna|&;@`;gp8S7aI4D`X2tc<26>JNvJDJ5Li`A(b;PiIj2<)0#@|H>m&GeRC|U3Ol$>YLY_Bp}+lRH`Csu&&9hUqO z+FX1=G4`D}$bRdsI?rvai?#Z(B$M0~LqlDFmDcZj{NqOiVl|Avo=}*+Q2(iM{$|<| zXtfGbNgSXIb_q(9mPBuS*n2IGIP)X-yXedC%6$m`Bg)3!q)+e0?}!Nhy5W9*ao!w) z_Iuak7s*rq@Mi|EPzp;dq?D#qRcnx&7H@O#tO2$UmVr>f+gUfM+RVZt<@7F-k4asW z>@Bf-#Xim!K{?RfRGw#gNat#4ZdUnQhu#G))eng!yGu|XU#TY~em>>a z(^7Rj%Q@z@TP?;Lw-Vh$kS6}HvZyG zul1;1M=fX(c4ch(2fgly($2x|;xrOQh#*}M>at?aBnhMyy!>i)B3o?I(4nl!>>Xih z>idoCUe>hIoUY#8heNbopdenMW+wdwz*5sslM~4FP@Vhx@NJ`-sU13)2~KPH11>Kgad&(2DQNm&x_m{wx8H-Wc-H<4?~%ip8vi^nRhB zZU-udV>UGf*F%-+-bmn*$n}x>2_A(vy)rwcBl6RflfafcT`^J1EM9K#pf$mv zcJ(!?JU`L5VPuk0t8baL-NxQC^)o-OG~;k7FQmc-vgbWVIcb0xb_ z{nZh=-$7S2VY{6usRg3T$`4JZdT-=u>p=Wx&$%+%9YCoNpd^Uz{ z4+#y6^0^fPeAi`l_9S8!Kk=*^z!b~9tI+B)>NHg6F){jVT0VBm*;mA91YLWhf8nH& z@kW5ZNU(WfSF^epUY7nc(4#PK4;We_$G-A$IY2fyfs4E^;m@j(%QMa9Qm$1ldP5r< zz9RaYW(1EhdQv^wMX!L_LlW-lFAp#Ya7nT0Z4&hwTIWINg(n7S)JZ0oLsabwy z`_07Hk5@-HpMOiNE0OPOkxY3nZGFkr%B~>ooJGAYs}OFB=lF`o)R?B4&5D@KHbQT4 z%oZkkSv7nnQ0N_--|M53?jsX^=9vUHvx(Ep@YZns{gIE8vg|7}6H9-4;rakxmm^=< z!&^}_?O7+rtwWQSH2?>{X*-z3vna32+>{S!9$lmN}|I@{BWyrufu z2_ydXv@0YoNI;hJVJqwv5W#1AIX9`|peqH1nGLSD8k;00ME zPgCdU?SLPvW!?WTp6x*&IHaT!zraD3;loC-Ydv|&`g2E$89uY-B~NS2wDyLC`9*1B z#EY*Oz;xEBB6hWs`idxKljjDG(j~!1rWzMP*g(TLZmzsk@M{!o$C&yF5Y%bgeTxbXP~h-yikSE~z1ut6^xlY1a$@3;9tZilSL(tS0o`gRR^A#bLR@ zp`Tn7%S-}X+q`;P1V)9<&cM*Gt|;_ol|i2y(qA7gIILS|r*jOgTUZp=w|MyUzSFaIMv*g*r0B*Gs3_PUB15b0(!>%RgRlz?%I1 zMb3Ooyu5iQ;37<5X}Kn5qaYIQoRJHs*XYG)-#LE~mUpRQnpd)SU|E`%1hj?eflU9G zm;A@YaJ!YC2he$}DVO}42>Pk|sn`@h}T*=2hLK*UaW*ZAeB z@f^L;5Msh^du(#66;@{sXt>n+9`e3=1G0G6_?_6OsER_*0!I)#c}Vs6-<(?%?F?S; zC#=e?E@@NCuKA)uT8S~)SN%r$c~p~%127|O0RY}T(d3yFYAQMG`((4A)ko%ddQY(iuBt>LkwUw9>}>~ov@6?)WV^3f6E632w^;RUbhu&Wz&?|l#oHRr$iMwKJioL?6U&LA$2 zZ0n$Fo4mw+LoNMRM#sQ?K~a9b1DMCxr)v55l4W|7zKYkk0!<9O1ATo6Ikx$FVa25@ z>XVK$YvxS}D0c}L84qAMk6=Tto*7CCr>HNt?Y_=hX%HM$eVh}wbQ&rQAVU54eiN1c z`I^6Wxc6$z+vmhAxfb8c8@xY87J4efD31&p&kVB6p&L33u_-DVY)RjgcE8`~5ud9| zalI`nVp`=wNpJn}@qWo6?(OC3K($Wz0t}>?eLg=^C&l&$y*1fO zr&+R4@9Z{OZ$IYzuI9gdv(?|s5*^;1YJ`)CU6--2DX#KADAH0V2W5vg62DIh14aWC z!!t5oWco`mn+WTHQAwO$W#84Rf~3nE<(6Z?C>QyxVYzPP`S%Bpl7e59wXLnnAbw4< zG2RcoeVld?b+>^aRoP$iB=9VLa(3ei)3_{_=k)1l>x6E@gihTAeo}{)AA-jS@E0FK zlWmed7`{wE#f1^S+gWm!!>xy#{*ZClnQN!oaK~F7k=zYTlEQ`0hu3 zIX>j|0|GO0YAp3qEz;5Arj==HM-}={v%nSlPvO@@9)(8%m&8?JW(~0qh75lh7?5sa zmboi;U2>y1`(22277ijZcmwh!X^|H$3SW%1{7lensh%sr?mpQ&$ep%YCv=i#0Wsz| zspcs(!%H+Eq1C%!fus~H!Iofy(h3uh>|q->2)?iOR${gZ2T>DSE^$Cf4Yg7O<$i{0Vo7@NmjyP)bLH3(VsF<*dsWD<5fqE= zbd<4~%r(<+fy7kT4M4RV7@wgLsUF?lJ?FiJII72&dq^h?xt^=N{PVz(=RhOR83+7U zZwv)jfWggb1EBWye!jh1R1E}@8fj1p))s1Z1%35>k#bM*FcF&2{W5EDX8pES+q^xX zLUSHaIsxQI(Fk)EIPqR$&NUZzU^s|3ZsX^6RB51E+|fCIHf9qq(}4ZfsoPrNUqy4b z^B?m30U!sUwu*;mHx}2{8WsoF35!c3wVI@rv?_whlL2IwPR7!OLr1HCnIALjgk6<9 z`LI6fxVksjYD8Pu-kx2F*~oK$^Om2{_X{ld$e~a31_gutKA#e19UcH|FWKLSRb%4e zng};5U}U_c2Mj{L8{7G(f`P*?7^9!~M)o6>v@S^1MrTRH*OZT~g|bz<0MXYeTV&HT z2lO_w?$sGrR?+wj{84S<$k1fA4nfE=!jaiH3_!fM_FqeO3Iis0;9OK6c5Xj-8p!ue z0_8ki(pKI;W{1+#2RgZy${!xTkeXw9fN%U6F?nY1)>`bQ)|n?202{XuY$v%LN|rz% zWB~Qq%PvXei!4;2<+`~x6=eKivaHFIyHahhYql*lP76Tm^Qf2doay*3o}=`w8aUNt z`Qe0LiPh@3uxDb(A{Ee;lq9 zYN>#y6{c!^bxNIQ+N*np>?v#K&6e1nyMD$+h@dwz_JS{mx&9eC_N7M**(J*8gw;MY zwWv9MbjCx${A^ZE=k_{%Ci!hy{6-bPYYYv@0*s>qfq3!s$D4+CoX$4rjb}DY`a?C; z6UjLu0pUN~=3*(WRei5Dr87)HIAgi0mE#)#BJS)5yJ?S+(W=P~wY_Tq_VrY_!lnng zQDWIRG#1^ymKW#`%%DM}_WN}~yTHxa*#0tKjm9(ND|rVo>*rJXnZoQL37Y0Q3z0t? zu}?24;n(Of6g$A_H9Z8gF%PrhAD?9`x7ARzB78)-H|lNc~)4pCM(h%%ivO|XGPg+Wr9)@Xp^+N#X;XF!U#yOvRp-t+#H=qdVHFVo@WLjp$uz4&_#;QfO~h*LDE`6)qC zA>_E;=Wn~D>qWt<*B74O72$m0FLg7IqBkj8u?T%0X}a)a_Fd!COIPY@o6SxGxy<(l zdC~vQbbJ~^X#h8AX?^#DINsQ^ST`-=XMv1wk!XF!%`z@>l`B)uE6M$g(6&gc1YG38vnJD;NPQ}w*DWWSBN!mTlFf>J*s=Lpb6-Qk}slnZ^H zVVXgmsy4#9g*ls_Lp2WQJP%C}T(=V0aH(Q7~q61RO_ka;Q;m`J3cYTRTCNU2`*_euvq z2sPls^To3GvzI1Qd~xtj490KRaqtLf0dbKLr%sqvx=1r~oyz7z51x@@ZH8O^xy3aQON| zDJEc>^9LpIL?g4nioIJq6*b`Sk`xaBE!!m?%t222G3|73LP6UXAX6D~LSw2rWv{6b z3C+}O05^Bu@oCvkIx{EPDt4Ivqb%za81^fG6}t@Zg^c+Iyr0rM*0pRQZ)Zjg#(Kg7 z0DP2FJ|{U{%`9&?oHa*D#YSn|g}VGW*u|9aPdF(pjq0A)CsT$p=C&GNP+kBkFI*y$VxK|R|| zQ9#s94)%=ULW|U_qB}+yK%{!*X|yhNlKd5FjNcRhs-4G4(;wegJ-QTq#cG236XwK2 zCLbq>UxtrL)Sn=So?^<;ht`49Bu8-4wUSY)Pwg!_T24G%&c?Dvdwo0<9h?U)6q!~) zmAOB$@(fLTM0EmG#`&c1(f;TAtoJ?wFfwPZU*d&^KpWw={EVS>B|&+~o+~;~8pe3Z&l_~a zu`W-zxs)6CQv9o&={+-6Qi1?0iSUwP?+4%D1#ljBKZVjaAk5+d*;O7lK+EWBYr}2c z3n&&ak&OZmkhA&)mapc9yJ)asw@(xr;f#WttXWCZynD`#dB;7_?Jc~m49JNc;>Mj3 zhP?&r2Vsd?7oJhnndmS^xaRjnHqzSlJ}?DF+IYsu%6EjsTX*=}2ZjUOzxmR8fQ-1x zLW=U{!k^yFQrS8A!b8NzNbQm$I4}F8C_yJXV_Xc}3@4W%@wD#UFt zfS6|9v`ep)w5(a!Yc`96o4t;r%^s6lHB8FBylstTgg>fmUFdDDHGYumyZ~`{Ku7N} zfckWCOZqmT`_D1ni;3ri(H^JZyScX>h{odHY==)mRv0MP6M3aTepF( zITlgCNJ?rpx;Y(`-S4b5O`RGgE#OSG?N~{Qr<0wIq4&?zbICp%14ZT+cjhh}OmFvi z9G)&@OC+YYUnCz6HpDjUHd&^h)Cm?bH`4AJ80mgAcNi>Kh%YIahrCLfuR1^ub)-`4 zn&Zo=s~NE3`U%QkjCgKj+h{+q7^7?R;3-!asqG7shNnVy_k)?piy`g7`!ftkj1pLM z1cj`D}eZcGIe5Azz`*U;9)mC=5DOtN@$l7G|}a}+&E&s~Vx30|$zTc3W| z?v)U~g>erv3w9pS2(6=r%usqb@zQ;P;j8DyBLH*Ay_B)`q4lKph%I%57EOo|5#0M6 zya<`fro=EHzqAinbO`e|up}3L-Ow;mcx+~-b!gHhG;5n;M}!z@hX-u?h=cPs@3pIe z_mQ`1AMRnRKIxqZ=b3!faMHvyPjm%h_J|1lQd}WU)cRT;7rvp{NEUSCMmY=U9BSgB zfjRs?oaU^9_)#@3Y<-JFZr(a!IB|Dy zZ>CeUrCnO|u^sTijumMw8x{{-sGz*lKJtw&Uq@qC<`B^_OYdS=RHR}AS%|Ih@EuGv zFDPR6K3EL%5F1;R7fpym*V&7fg$eD&-w&T_*6v!^TV329{7h571V0bQRbb(dCW-Bs zFW=SZl%<*4=!KYxK@NQP$DM5rA`5A7bWbsKHUE=YsYAbcQT;m7N}j zJ=fjqgl03)o5OqZWyYFc?`^L7)~6m$ZLc0b%0wQPze(R(ic+cdtBq{Hx#o3pFV-+J z!g&OA<`{qL8@b?fZa8734_;1NH``Y5k#--;6;k1w)^-BzlV0Vl{r zEzQK>D;nju8X5L(Q^VF*^0;?4#56t)XncgZ7@b$y-Nr@pV(riCIMXxa&g_*~v|juP z>)1sfMJpDEq&4?tTP84oq;?0Y2xeM6jeT%uKp#Q@@(j%`*gh~L1pXMFcMbN#SVdo%7^Q>cXj zNyM?-sKa?I;*w0wbw#sV?XLn&PI%ZTf)l)EZgqq{b9%M1vyrqP&$JrQy3jr{>R={? z^TT1*Mn;JJ-k&Lb^=M_qkcLNnUF9~aEgeuXIb1hucQED+5hfw>Mp%bAl6)(Yq zdPmp%D*7sF91>9HutXn zvTBK(tq;ug+PdM7e!tWAqz{&c@FA<0#uB6tkVm7xv1@MlwujGK@Jp93M0ICzB-tGt zu*391Jl-z6UGiLO;V+4r?iCrfhaLsnD9-wg-=%FWue1Yk>X?rO&tDo9-uJ8Vuss+&H9^gQKn65!-=mX%xWVW#Su6X+{>4NQWNFI;bf8D zBuv0Met;kJ(a^_e$B8gf)1<=-_&w76!noGMuDKAhdbGZkeb}YPRF|E@t~18Xv8kkD zVkSpFj5T_fEHB;ceLL!rgIwMX%xl$({rh;QUC`gH16Kz_G#Gr(pYijkZ2a*vsMmOr z&b(kG%Q|br<7{2$fX}k6g|9T8s4pgYtoij1EjdpHrih$7H~d8;9lY{U{LH`Qu>Z4e zDCaCgTnp~yUdfNOkJIm#71Qf;xOF{p)|@vtYjoFCB4^~yJnh&Px|tWJ^Bn2aLypB; zQUeWKLw5dZ+G>!CnySdzccO2+8N6<6Bf0&z|%$Gf1{}WqIoDvF*B=VbjAOkf+od*SQQVr zpsVEXd+k~8=+E~>pMh`v$;SVD5eXjopz)AAzZ&R&zhOtdgK*@T-$ip5h%c3z+DxY| zG#t8JlHvW0)>#zz14f}({xa*DcWg{#5xIyj2Y#PH4`aoUBgRvkLtBqK_DT@3caD-x z>x+sn>TQ%-6~rt`s1)0|Mg+Nm-dj1z~;P~Z~`}-Gr23Twrp%ppUyA- z;i2=PX^gh?&F0eVvb=EirHw`!q8NB0R?rHH0m~MP{+0j!&i^h$pG|nmV0*t;wJ^L7 ze&+O2fz9*gw_`_Sb7xju>qquxk=TOqMOtW4;GdOE3E$(vt)S>_G=O<N(E;hvN!5FNR(WJNUo7^WWug ze{$i^pc8++eE+4wmvncZa;CudmdpEKo_>YtSZU6F#0RM~)O!dojZ$$hr^J$96 diff --git a/.github/banner_light.png b/.github/banner_light.png index 788b1f94bb6dbacc455e3e70ac6cb60f5c34dd2d..7cd686bb53184e09aa070b6c5d9914ac6dcaee72 100644 GIT binary patch literal 23647 zcmdSBc{r5s-#+eL3MtAOLdljjvSe&YvSkn1s|ne|jI}YPMF?54W@aM$mYuOxL$eR2!}h?)A()AYPvXzte~hige=#wgWiq~|cQfn_ zxy|P##L02KN9y-eo?cP0g@@HVz1Hg=rDMN8x)OWxmL9VMN06K*-?7J&0xU1jb2Cq# z|1`k7_^s=4X5j7PN5LLV$1iodKhM(mn$@4xuW`w9FnoyCJ{vG!7BKFT(QX$C4TY`} zI!cid_7!s(^<^PYTj+k&2&l*hTiJ$njl`DiR^T0cVB_Dv-@W?wMA*^)uRnZ6(?{J3 z_JQI2d41Ttg8Ap*tp^jc@WH#=NB{4-cs66B*1yIEC!Dr17*1grvh*1HbZ8InuK1HB zGBWI~Xh`T_YFqC0AQ&w7274|pptEy(_$zj&Iv*bv;0d1|e`I(xn&Z#dt)g9>W=#;S zYhPncMRI&_z41k|+&<&^+tp<~I}TBXkB9xwG|{UnN1nA?1xU2yZ2f3wc7aZnjQ5mo zXaxm9vVF^a=S++AdGRCKBic+&7AGNpnthQM8Ffm6kinnPYe2p0!Q6%=HJd-r$;la` z`p7!>ZPEzQ-uz0gz4pz@sv7sajlNz+mw&*DWLZ3^GWf&g z`@$T=e6i&dpPlg&;Seo)MNM_9#t3)dBuMZ~zyeu+zH)bEAw1j)$A=%91NueRWOi&qY;;X7-$IMIo*c4WRoz z@Q%!2-cykZu`SBlbx2AKgQJ%BsvZ_^E9HWm4;=At_tm8D|NeE4aGBJfQ|XFU%e64@ z<9PM&GkDFqtG97+YToOz1G~+PgbG)t<&69i&N|t`0z|2QSJK|Ldd;OuRYcPa19diS zvv;gbN-Wqlx;#4aS)zbTYTOn-Ixkbvt+y<7)oUFT^A6oSZ@%3?o!#2CK7IaRaFyIY zjNLrOd*4>mOOdmjK5*kgU`#T6S?NVY*sOi7A=bS|C3YNU)j~WSqF7N8*_LsEV1SQ1 zx9!YDWzcB;qVx*4w}|rEvvBSH_O5v|aRe@X(?*U7*AT zk$#u9D2An=>v*;@d8mQisEm>%cg|w)hxgJg_v}|yLvlh9|8#VUzg(9Etx%FOc&*Yx zL?SD1tq(#YW^^+^JfF6=k>|NL%gPXu2#E>)*^Hn{FbM{gdr`G#l(fz|eS75WpA^UR zKrz~%%Olx8K0SxW_`}w80XSfS<+E5BSCAVaDpR4`6~bQz6g&0f z=~sS_pSZ8i$`_jtl0w|&3s|BsuZq@gdb;Mvd#`}Ju*AFMR}fV$t&rv6@# zqX^ zK&DKF z+Wm*klxM{A|EMS1IyvB$ql_Z_e(OmSvQ-8<8cS18n7ufdy=D$y~M02YezUoX0=EX!8ocZW?%_R<<#hBLe8Kh${S zu>kzK6glPoPEVac!)V-uDK+_<6VI$-=qhNeQEz7!j-Nbtn$Ss%U-Ke7Y*^avAV|8*_E{%Q9u;JXnmC8SIO@}JvLksih zpEjOGEBG3#kyI=ry%i4m>0^o%F@6U-Eh>WN7BYAmBRi zI!hSkC#W4PBqpZ+pO0BSL+yXJS8V2L$qjn)cjADl+hdKOkGrmKW|*g{!zq`i_B#7D zR|EDw8r1x*8lt3;liFuOm(>SZa2Zh>T2uR9e{EInN9~^;7;@tb*l(lUjwHonOYl}x zyNe4EsydTgd$2l@9FH*~kkDouaY6O1J-EWK|+IAldPi=)pt*L|$ zbu=pfF7K0_$DN+WC+$1J2ryZhok^x4&KT>A*pe8}smacte(oYm0&{KGVnWU$i4~x>lNya+i%*MSr&S zZhImA-l|R84~zYYcFpcjRUwow6yhqxMSP}ju1vAYxF=aW;NdSZhaIy2I&t2od591e zvA?_@In7TNeRyG7Q?w-rr?cs{%0UG8OsKUr?m6KpTNGLQC$Oju-RTGT$ZbSo>Mj<( z_iL~^o_HpbeDIJy99}99My!xkQb;7y2y)yrq;WfCuy^Vjar;#hx;4meTzge=LynMN zSlJV0eql+n#7VUI%!{&-o-J4D7DmH+!sQN{nbQ+24cQig+zEx3Ou*?`QII`qdwAsg zA*YT9-y2{Hq8%^2op!#s%?Gb8UuJB7rXrfjNCvIqExD?_Y?j=$apBBaJ zvP9qCDa1|eUETjJx=&#=t8!@DVdj6wo^Pa2DdG*A!NfDcR5G(jzW9wOx2LJe7%lf;`{cS!T6CV zq5d37L|5-{iDo;cnG%aQrC9`5{CO4ACYHtcu|I?SN*>jnl*#?BurSf(Bl5 zlAp)XmOFrs?A|;h_BTuYl3soTdGp3guJ+ok5m|u^xUY|INkfT|KM6;ptZpq0yaneX zASv|R=iGy()-zl1P*;lviR#@`Rb-q{i}~FxYTu{b+oFD>I#{ftrl5I}rhiB0ww3ZR zE@!sbTpaVP^?)s8?s}W*$+?SY@{h%aOV^6)t)67~6I6hC#-d z@t3Q58JR+o%U)g;wh4NT8zy$Jl^L$_j~Y4jV3SWU0sGDQ$?3^JT;IE=Wqy$3+^)uz zPi?LJGf`DqtL!ZX$a=Fwi{PND!Q-8vQ|3t)tM@o>ITad*O`L{khJqV4#+pxtK#HH} z;U0Qnx1Lk|MlMiN`5?UUAwDDZjr%*j-?r?d817G}@7bH?!nQ@*<)X5~FB{K@GSuk~ zlf2{GLq*kd)BcM?5;KQ~qsS2td=oqy?>D&8z&-mvI>=bV+R8FS$G)5Yyg-JC4Ul|+ zz$gROn;eI>8$&$!AP0Uu!DUk*;mKRzC`;E05H$#ZU6@D$8#sn~L#49K?*-UQnx(OF(Ru;D0N_GlLWSy z{V{5z2^L5bQs)TO^Wjj-sZiIUOwh}NRFNr^mRqzowi z4r?@gDP=>8$~V3!2OYk9M7vN&@|}zc-8_E8X}5_e$fpaNweErn$3@2^lTM2Z^9cv2 zZya%1r8R*n{b#2nvyAn@-Sos$Zto-}9q=bUPl04drh976j{!?97;yW>_xDhh-|a1T_~YYy?U!}`Zo^4zLctV6Ki(-5ZiuNU zKfckc#A~s+%qY-a_@)$F#g9pgPJ6>>-8n6G(nCwB=pQ#vZPgH3xopg8Jl0X@3-anb zOO_#iZlb2i7W!5}Fv*{jjL5f_WmBri;kAUa{EzO^S^Jp^JG{Hh1Bq>sR8IZ99I(&QY|wF2E?PElj8rbr z0R}*38a7GJCnPXu)){Kk_FAm0I3klHAhoCDo%|)Wb!8v_lG-iJ+9sbCVhPpRSDJu^!F?Te?VR0EgLyxg=YQtRsLt0xCDm1k zUHBL}-L;m!37$drI@6{j@5E-kkh8$>#)kH%w$*CA*e3S062Bo0hu1C7)T^OI$p~i2 zKEA2OlypWFiB&3*-TiZi0`@|RlE02A<{3z)JoWa)N^0*&E_g~ggt0SHBPLExkA1~E z%+I>Jcxp2r7cM*!hg`FMiU7m+G)M=KJKr!`_?tO>KQr@Z9vuuw_u@R-?V|L-W8li$oT{WU|U zY{oP9bxK*VS5N~DWN+2CTX*HZA$KOGlbg%z5TU>M8tC4CE1iJ<{SB~#=L4Xq|M4+# zLR!rH(bVVe6MqNA#PlE5O+-6)wDw-8k7Bb(aP!5A@wUd%2d23nI~Uf!D|=BIXD_AH zGa~Zm&>36b-U>oQooM7@7<3gk(D_5>fLr)BoG(hWmv%hyYNZy7B5KUrJQ{@(|`M1IDmozlBa|q!ghyNwJtZ0>5Zl)6%>QZ+6 zP|C`jj^@E6YDI|uoZW~wNeqsfF!T5Cj_>MRdEZu@BiGt?v&oIxn&a$h-T*F)oPg#Y zxkfBoUTGq@etn9<+BKvU;06yJ)H@>bn)MMiHSg7=v286S%UqOtexcq`b$(}_`t~-s z;RjB~m|o)dZq(D{Ft*o;@F2+dIczoLhVdsA2d7erfwuJcJw;u&;4SO4$;ztG@!;g^ zY^7pdyP97)q^Av|W&d$_So33g^*!_lV}ju>_ED+F>KK|Rnny-YC>wo&42?;q+>N|0 zxb^e4XniVt{`Ilk?zzEsAAW;HY=$=_({#u#hS>^fD-2a|Nv*_Q)&N2A>0QnD8L0(& zu+1B<3yO8n3Ma-bHF3423qt2&$oU_G^Y=-Sx9`JaD=1lWRRLrP&3YFqbz;|CSv~|n zK(yDkN!|XV_0cs(y6a3)sO)0j%Zof- zp(9bs(@lQ-#_)E#CbzlMkVp^l=Q`?*e)3)?kk9I@xNHlW@Z&wx4?4N6d+Qo59HY!9 zMR^VmoFyt`y(Z9xo-13XEucD7m2%qEYhM+1WGNk;?Km13G1GjWkgod8;;5G}XG6u; zPy|otpGBN$$|LaljXSS_D6i50PD(x{TY`|1_F%7C?CG5}H9O1}czzN$7mB;vQ`X~b zKjqUX?JYRaww+tNJnnb>aY1;`%K7Bl4U@Y7z^7rWc)S%&oUg3}-Uz91fzC2$eNkV0 z&a)BUF}o!HDo^u{|L}9*&MPTG#P&vI|0?QB*-$usrVU?I#_rDbyOX)jB2%ogVLZeQ zv*nxlt?axJ_NN%RBI;dR-0DNAyU5deSgP*Jr%p$ov!R)1YP=yG6}Nb1pC3xw89X2n zE@n(N4V8-xPjC`jVZipAcf=)GSvUKq;n+wzPKFBt=Tc(xbsL8=(*X^Ey>MmZh3Toj*vj8-gr&1Hgqc86Lpi}cZ znXtRevqS+(6^95n1ZRkZmXP$USWKr@wGOJnljjI(UKv1DzP)uGTpb<7y9TD~B%?fy z3&w|#XKt)77@h`iHwWn12TwOaTI~5}hou(FOfcZ|&5@0__p1_iwvid7HZ!%v{QGr6 z4ezn&jq+O)(X6BGvw`*vM~5J`(K;7KnCV0{y3QL`Ovgl>bflnr`^G`^UU`ag>Z?Q^ z^GzPC6POfDK{?5EAVuU#)<=14_|@3r<%SV|b^wqV7yh|{)Q#c#i}=O(zo4)>i~ z_ZSwLC{?vXiy*tCYi+wnNeV2~hEkS07jEiVj zweJ$UC>9fet0V^Lw-=t?5I}2)MeqYzfWmJuu$R@LU*|IXwXR@X)t6PWo5&Bzi;+*=N{48>-DV$mK)_OeiTSu!6pp zqfDcVF5nOS!53joMmj3(Ha!*WgKXYJK9woKgxCY zU&WBZ&MaK0K&X|$heOZ*OF7y;L3Q3d0!v@gs)`z|{4+(seYH+PGP6;5*kE6s-T@Q$ zt^Eo{^c2@!SuP7&q+qnjdA#kIpYva2{=%Wsuc$#&8rW%P=tCfU@0@ujtnx)_e(JmS zHr;hIOb}Sgu!wTH#VOLb%KqV?@vwG>5x@u-Q_G#2j%QE*B(8fGCjiL5^6?gN;oJt{ z2%XfyAldXs&TSy9xr);8?_X+?nwV9|P3|LQ*Amc`IgEN)V$mv`uyk5jCm~ zv-#;+GucpLr{X$m#{14s=s; zxgxf(=4T}Shnz!LWbb%qAHWhdYMqg}a8GvMa8wAix25(W+ z2yOl&3UOipfd^=vlvDQWyzy1J7OafD zE3->YYjXoR4_DgAFpnoz$&aV}bnQ1_RQShr_dqV*s}9AjJqU>ikvcIKyW^LxmyPnt znZCY-Q8oGfkH|6Bh$GPorF?^0%LCxC;&8`Uvw|evP1>vA$A1Fy%JGyNy(LK_MG}wG zqh>Xh4I@>zn=dFi&jziUPWK<%T{e~iq{OC;!h$Ab=-_NfL8$4A@yEB5e<({^S4h_p z??A_Wc;Po13zApKD=kWE6aSowXuLKFl5S8Bad9Gmz}*asE`%i=Rx^V;w{yg5VGE>=^rz3)8{1p6E(|DLpg4dr5+ihS+s9_ZQxJV@!H{=K z)2q=%8!7@M{3+tXA<<*#hO#Fgo7zQOUR0bHTlw+oWlJ5k%(W6_`u#+~ppO%^RWk_2 zVr})bNDbhFKt&$ufsxv|Ie27p|b!W!O9iKmE*1RUl$@8 zlf>$-Uig`!nIkd#^wz1nq63l&R%0Q()V>YXM&zJP0ZKNn4c(+uH!!$Faa83QeC9c=~)zj5ob{j5(E>nscBNy}Nb>3`sVk@SN;FeL3YodmjEeGLw z!x1uj#0r#`4c$E)lY6|6=RqwdfCK9mRGGFJIK1X(7j7^NgNQ=tf=?>d8Y9mMWVCt? ze6NgLV$KaD{JIjh8IAbNx%YwdUEGxV!%p>kQIfLO&URP(4-f~IEmPj1TH zVgY;;-|m3HgS^$_zt7PUkG3+@Qg^FUcBheg5|1i@2sC6Uv54Wf9Vjd@OI*de?BWZ?@P)Z_}7+%{Ypnn1^AP7-?MaZaDPob zhb%uNr``pNcrLae$RF_8BhX$g+H5ig$OKG!RhF2~P08P7MsbFxHm$_E4Zs+%*ov(Q z=x-8tw{&ba@S(q`5%o&HCG$znsvb7VJlh@OY*C-M5~i2U%)>Om%hlr41R1oGRB zD!)QV!6RH$>PbjKV@8-P_{#EvU}wcmxF5Txr>H$f-o>Q{`wkLSz0KAu^u#<)Q|sQz z&A21585OEr)&SR!QmG=Lf+wgrhncc8$ltA{@4*;_{a)srKX=ho&3y)l0Ia>(p)@gn ztB{-rW;Bn22Q=}hJj4T*OoWz#nb%o-&1=7se!_4k}7>P_eRkD)0U z@DnffVq%!1qb#CDWWU-b>S8^wUr0Tu^EwiCN}Aiykv}8 z+hUxKWXcp_jrbg`W8SwTJxn^iTv7T`mY}pH44n-V{dW)id5Rf!C|bumnj(t z&n+JyL#D6iS$DOMMOYo9N?$4laG8bU$iL)3rs2s=HdeH>Y;%5;2NxJqYw|i;Auj#4 z5H#Un{7g)Ln!OtB8sfKZb(0eCfd@&8Rj;b_ z##PJ)(eJzCxOjlUT_>J@L=AtH@oY}k&l+Yit;W0U-Tk?C{nTbT_}qc)dGQ$Q?$aY} z(gWJ2{)Pg&BHXZvK&U^we5d%XXz-jBR#`D+(`uUK-`+YL)kCJBa2jPs0ZH0!Q`VoE z1M-jUii`heR{s3{Ju1Fcg)C#!RF<|`Dj6zdh$&0|&wP6b{8^vcrpi)nQ9IIku66IG zVeAoKp4@}Gn zIk8xm!K_U}Bq|#_PgMVl>X}69{)FM57qB=EKp4{_fqy3(C^rB3KX&nv^WYG=*W54E za!?cBOcND8BUM4)zDBBTsm}nkFat7tLc?M^M6h>j3#u#+@$Wdni8G{H;|G_se9B0q zoa)5U7Gm=*WT*u!%)+#Ofd9^#VE(9JfXJxv&k#7+PcwWC;(oUxVoWDw>#}bDs`SO_ zgu{*dJ$BVEzu6&0+015H0{$j%H zVnrDLe8RqABJ}xO>WwP7a*3jz+_&W1)Ju)@l}+FKxas(WAImxHy? zwUef#uls>3{u-*fO8+%fF*FVgRk|7=VNk;#OVzItz*6-%A!77@d#c16{^O}~EqIv< z>`-~29hLI2g8l*U4`o^m%co{+1PNl+vawQRiZC)sMC;Xc)_A zG0&1RrJUySNFeuj;JF&0S{vmsE) zr}!H_{pFDX(F)6|X=^CI{2S7qQ^$$L7v{KWD1ehLtg3SewJ`4PP8FNxe`q4n}d)x6e%ty)>rZb^A*w-_mK2weKJW-rR? z*+0c4L7eJlx{ME1VimL>*6K*}8DzWy_R?gh9{YD1 z&mB%e>a!YX`iI8eqWt1oGk77oogXQ+#tk*%>(j%cSOq{fPadu_0!EjUBMiHg(AKsv?NF&?BZGsUefD` zclB07t#6oOc+IwJxt@Zq)J((A=qWxyB-C+4Pl9?@fSOY7l6`a~&sNgjz}jamIk#O; z@{i{$e~7q8&O8N)rwCeljY^-gKgjyE)&hp9x5%J32AAVox*PCpcam40gBSbrAl|}m z%hO8Icby;!fZoYg-$3L3?vqxLFwCKAW;}UEb#uRc>8*zk;P<-&mD|^Nmb!aEa<}+?6LYQHgfVK6-N09(=)fWM zrp}`Tt2&*%=1$bAMESIK$g>ooy<3~Q-D0>mOA??_ri<9ik@h_}HME6ZQc3VvK<@ae z3|L09nJLr`cb~gK{!6vGL7`WV{|SWCiSM#smSz?ktN1zH`qVq>d!h0{uc|)jc1+JN zn+3A7nxu9~-67X4dw6y~q(NeM8z$2TEUa9g1xQf%ba)`^LhZ z_4v`DXgkdq;^GiOA&o8BRthSei7Fd?C!>e;B|7{~N$!f1ry;YU@Z-DERFQAPY`Yoo zh1=hQ{M~7`-S7Uam>WVzCKS;b=1hKB72#J#O3h#Pr+wK`u@rBYhPerFM5T9XowUJ+ zw<)$1f(Ja{Bg?0B4*Crzc zi=xJ;9FRgHRxvd`f9ICIIrS$5FXN@%W@O%2RfyiqoxVXm`KLQf-`X?7_4YqB{v4WP z6u_qyU|&05_us=`!e<#+J3Zgti2u>Q@2MGF7@m-Xg+5%TQB!yE{RKlvUl$4F9*vge zzMvk1&C?3lbe{OQ`f%gtw>=+tSax|7L4|smYq6$vJ1M)lXKaNQe&3+NR?$(nkp6+zw-2*K=P%^Kp_xv)^{mAwe|yIyl8xG54#l!dK{Xw7 z&EMH@V|d3EW#!>Co50%7>htYiv?pZYp;aA=W+`%exWzHHEpbB1x7J0i{jCfi`|bxd z;jbfG;T!z58Ha`+{xihbvMNmTzvdC?Ri)Ys2Kyh5x!Id_l_b z*>}m|$lba8NHeJnOgitzd!u8KvpFWrz%vqy^lf8?EgjxTCBrrYR_!9_ao>yH3T(qy zJ0b*TMGK6^)ZH}7u3?6p+PXvNC8h!aW2%_WJYH#bUWqqJSe1OA@%!)uIr4~ zNhk5-@RDyz(XRWS!YMjj5QE(K=}Kvd*&^v+)OPG(?$G@kp~JuTmEG~RcrY9KIcRS< zbT;A5zy7?7CxA#&HP`SE-Yi$~VWW+@X_yEGKuy^Q;^*;qb|&{NhnyqZj4<_GZz^kz zFa{H+!)3P=yAzV^HTh<_XVsDYsXpq6R$>D#vQ4uF1z#)L&~$upfEGW%lsBRk+BY}I zeNQR+^VQ^$kE%9J-F_K*$XJaYQAU04k1hC!i4K)g$W6%PcrMlP@Nx>O=M0Y7g?nm= zOzqx3rM`f8*F@OW*V)$IDt>zm$%sk#!LB<6d?qPJpL=!*Ch3!TB^kwEFZ>tiR`+L< z$3j?^-n;nwBg4ND*JY+=r%loL@)E}BNt~CsAxmefl||}CZDk~*?%_AB;2=oKSR)nh zhEXk0JxFPPR zevpf*7nJV*+?yGEPCXu~#*=Hml&bq49-6+LBObo(^I3qMy(3U71;x*MP&OXh{bO4L zP~sDx%WNr?1`!m)-eI zok+z-l`o48D>!`%aoNnv8m)gGdCH(=$lJ*gRXP8SRk zaQLs7FYJn>q1f3rGZ+rgWr|nHlzlG5Chh$aYNULN&Z z#r7sFF5PKVX}#%kt3rKi_S24HijvQJes{}q_`6FrC?K5;Bu_m4Ls0EpAO80G+15D$ zEnnegKQ8OV@86Xj9B!GKCtA2!){(5fQ_?Yld`IkLQY`BX_{!-xd4h^@5m@` zsE)zms>n{=LttV#X~L6#&QmD0Rchm$ zXYR&5Sjo@90=inddTPAC?)%YV_1QIBezvb8wSe8}wX54vmnW6MJ3HB$6Tp!S#L?M{7~R!K}EFU(3HPO6Nscu z@3_(M2~f<&&9XSW2m%;r7s3=*-$C?+nhBxb4o&Hxt4RIznp9Kd#t;;P7GfTyno)1y z!`Hhng}P2ahK`os2^)*T;8|K9Qkk>3x&BO|F*{@lxKobu$Ul-Q5 z%I#Eb2Q#}YuVwTEwq+UgSG4Uu*#V2CDY$ur*VpTURoV< zmr=QbPv{YPc>PmcrdSWBNb|4I+4}K{5h!IVf)eFOQ>RXwSJG*?j`_}hUMD&X5~UeK zDNB^PFbZX7%Y)t^)TWrW*|C@&FKyiB)D+u`aFnw6+I-W(FEy<6@YQ{+Psfw%>L8In}=u%R~OW#U6z)c4JImre%M zSIvT3_=#OLo~3R@^&d$!`s&-Ev|Ki#md_Z}g+$40e^(}4p6eo{ut{AzqV0!l=GnBo zty*&H^`)n>Cad}NHSs$6to~&1=K^Un`2K*um)`vDy|wIb*N20AQs`P~3>S}Z)!5)U zhUnflxj1ALL=Eq;@{Vx$voHIjLqS~=Qy}?IKHKLT+?ic)5)l0@bi2Tp`lsveuz;CriKuEqbG3A{vf_=xt-Z_P zzQWg${mionVeOndEi%XEGk&hb9)`|CSAYWiWAmnSS@5U#o>syV4&;XrSwi0@%Pt*q z`z@a7eb?#sg;3}WPw2j;I~JJy{BtYPXf!O{HpUV&x}6oN$7$fGOx+!-J0|5WE0|h& zlTuULXm4ew3vqPB)TZAr^FAJ)b^OtT7at566Jgx-pZTDeG+xMnH|L+M_RLh?rPQ6z z;3V&beq4`Otosq|m!AriN>Q@oxTsds=o+yi$@lGVip@Ux0rR8s=RnLou_Cn>0CzfH z?;`e2y5)5g1g$9RZQ7QYJRk$uJl{yXO26vXF!(92EqsyEe=K5J?l&6CH@rmx3#(Qe zZOakMJu#FjgX(cM;3QH#QuHIAw~+glr$f02dxNUc>E68WI(M=g^M)fT!1n@yxe%OK z0Htp~hxg>{nK0<28fkP_J5hX)BZ*D>`0g4<_G z_0LwAdkyIk1N|?Zb>`Xb9Xab=S=@%GP{xliyro2a8sFD;BAkR2-j*}H_3VExcpP4+ z1WDS0tzc<|mKfxW$Frulr{@^EI+G_8>PW}(_ZE+#b;r7U3dC6<`S*8rBnGZB_I zKM`QbsF+SRU?#o=zFo_^!!spT;uS4Ys{qM<|5>a$<=(PkdqTl$=bf!fN|@7I56+b7 zRyGz7PMlgWGjUh>C&E9%09{mwK;0U>Di2LID1Z0A9-O;<gUvIru^Ja; zs1LW>ZdyfbkINj&zxd_&uNwJ1*7moHRLwAdN}bP~&Lw$*=zhDldWoam@Kg8v^&Qaw zgQ7&yilw3udxyE-&D&i^Tw2#dH!_@XJ70HlB`AGzReX^M8Pm&rqJOxyM_Ma}^XLhqD(H6XNKNReH(5h_+F zMT$@Rd->+VmaR3*PC(%FFBJtaQ-v)D(hKSH1V*4Z0?nVBnrM1LQPcU8Se@DqlI^Wx zjS{|QXj#N~Q@*xqXau7%#z(x7a#>bPn4=8y^+)>8N|^6nhprxylBFH#-AQr4vNh{2 zI0EOE>ddK@pCx^3dEk*D5I1PY$ny)6b`?I2t@0C_6#*N4j8Ie$|pK}=h# zanl%3+(SJJoM+(??dz1mocw~0*06cw1@!mopwhg8`9!-OMMw2L9cI`Lb)#xOURQPI zN&Uvu2I1U4Iwn&?v^O5#s`6&XVz~~~j8l>#GI85eKOFOqq3`JhdGrStm^G6bDMO_X@MK^@*1$r~lUdcSQmoK2}F zv^(%UQ;haM%!nSDkK_Z072b2c+pPo#ZMYTm+vcEctta?VfSV=K(V5fd*Cdh@lS zk2T^lhf{yj5*hxUXP?thKoanC0|07YD5mG?{n?#Xx@u&UV0;Jk4Xrr|HuRhH)Z;)v z_Dw^-WH#r9zia>r&juliseH4+j@F#I}p#XEpmXU+ZPu9lABw z6>N)Y*wEA+!Kn7sC6OOZdq6%BgN5&(Am5j(BeUzR^`X zn3P`hdi&GoHbBU6Or^K3t;e6cqj*WrX@v{wIHEwR)rm{-tj{s2ilm9-QnFEM z>wtZYRKTyP$TlZwyiB%H<&C&T;sbQ*{6^v0Qh3krf&DPL>yftm&GiNI7=Eis9ZAj*y_~!>a1DgUkT~k+dyP=<^AS@IT@(BplDe^qS~FqVkL&VjVT0=UX$SX_tP?=-O>-M`1I#ON>oNt7{hnUFyGyG1U*ZX)QwCBW4QYViPZUWgNe6EquEzl!_&!b`&>e|vU>K&ql$>t?29SBVyn&g~4~ zR(lnTs&f2+62~Tt{DYBLaWTN-yQ|KmC8o!pU7>9^ zQYKv{YM$|#e^pV0yUJ)aeqJZUG|>g&+OiK{#^V$|>BQYLx5mT+B~@gvS<2qjJWjT) z)iF%OorS0L*vu3YlhDqe?**vO1u>6h<(YFB4EP`kJbmGIJ$ChO4*R(Q3L;55 z=EG=`LyhNs<(fRDarb_o{o-Y@@lin6T$%t_jP)Z7peSr_brcj5l{bef+KdH0c%P|# zUhH4NYNHDftT4Xw@J$y@?&M&zZ|M@f=Q4V*&lZQl@gwsanRO^+ZQ z>kX^xp9AILT0H;NfHNEV+=X(PY(~+%i$x-~zbdlCs72CHR?rENAy~;qjsx208B9ak z-c2`R2PpH?W%&lw8{o0{|937-^{57P@VrfuquPLC$G*Iiza7;awzqVF#|%#QbR33S zM(pj1BGGFrO4258LR;~ZzQXRVR zppVLbEjs{@rxGkrIPA}*&KJJ)q^Nh4D$=~wR^PCV->2HGFS%VdLNNNQYStg8zH57s z?CdssqmIL5A&Qj2k*c_NO?GFdC8zTIZE})1i)qJ47r;|AQ4QZMGwjV^cjzyyR zbD|6Ig$jIgu0dPFs*^i6p}B5e+d9+j+jq}T;Y1Yc9j=zk{WES1NOiOGz?pqOJws^0 zfb;^nX!$#H3z|-H%~O5U4srmEjC|HYS1H3JsEevzu#s%zdqg2fqQ^0 zhovJb(4GMVw7>szM77l=dEjdbf{WBfE-mTO$UpmO@AhfAJjAmY)n~%|-kyH;v(VWH zwei8)tN2Yh2e-L1GaIk*^wkCnSmgXH3oK?+jyOk}d%EI3!9u-?)sP`#Y}B3(=7SET z>c3~-W;meH4mhx({*$g8;?reY`pS+)n{GaFE zEUc$H=3t=%6JFn?&KtcK+$!YGF8|VS4@*fNymPm_LFf$rhws@r5F&06I;JDSS`0@p z>bIyiH`|VvCL!4C6Dm!GSwMyJUf&waKuY3W5yE{H2qoS&dxN+ZgvPj`SgFykZ(1#8 zjY*VPH+(8!Rcfci>YT5pR?`Z>CFeUUO|kjra{F0tj?!9AgqnHl&vHU#N}}4|VeNMJ zIS5l@I@_z!l=vi_{6#{ufa+%#VX+B@cQ6e9U*}tnciw~;=Zd2NJ|p}&evZPjMIr3Ie%kfEbC6@`&<#K z?xNwd3i(lSEkISwZK=kc@l9oo5fQOqe6yIeXbU*7bWyYQ!Olqh3eE^4W%tbD>b*Mc_<>r#29IfHP2$sHdI0o zRH#HakLrHk^Wm&@*7y&)mmxJf@ z;f*k>V}P6?Ff6i=3Udu#sU?MOstUv`?fMcE4Re-jL2s@76GF>)Kw1API_#RVpDq{H zCKh&cKY;?fkD=}lZOtubJ0W$4Fq72u)Lha?p97WeD}vZi+~;f;Z^Fc+-9x+=W&Ly zlHC!#9syGz2G6a;jFLyn+8G|28X8m$=@`dGFo%{V7p*YVR?_*IMF__KZooD%kI?9< zemvH1k}ENX+bBWYd`Mj>*N*8X%f6@bBuy3HWx1Nhh4qo7u%bc}nUX5aSV(*4wiT#6 zm+>i9b0+HG%4F&E|DA_Bwh!U0pw8l8?w(F}s!!F3yzws@q{wNdBzT<5q~zv^wdzDh zz9gBN;vQc$-)vn?8f3t%MJbC*9OXQ5zkNL84hSKbZ!`k0pG-z$+-|78?b;lu>c|Z5 zZ(xk190CT&^ip|JAOQ3V!Sz3NJua4C2qc?yULBy)Ra8LVn-cwbk%ozPvY*C%V=vVD zcy#(1&*LKZkd+_;%+`hl*$>mJ9NXIr#O?eLn(i z9b!go7_P@fJ?t>~mNOg{f?3-Hhw6G*2`^s?U)LppHAL-rBs#e2D5dmC#FwIqNviB(8vxA!_sy18qxnyWXYD!%Ir^{-KaLI-+O$9Xq}eV9A@ zb=Ar%Yt>%4BO1IHC+?7rlkA6D+B19UoyUNtIB~aztA=dg+rtCL=i$ur%-4E(+!N`jQi2qIeS!aL-?!~WUaUZ5%J(nGVtLU zQOBsF0&l3uGC27z?#;)b`U*6MewnjKoS+Kq)8VfKS>x;cxK~BQ6_oC##1wBJyT#r1 z&MT*ZCXs6}H2EdTWw$aDoLhwVj(((}pjHSs>Hd9Tb67JgiZe(-9X9aYaUCPVxh3(o z{693|UTXQNpAriIIva6!GDK<;)fvI+Mc*>|-@TJw%vGFK^sYAe+kq_>?Z84-i ziwFUZRajX%G94^wkanj*e|=rDUABtGXfnWvv?Vm`nSJsLQU9fFdd@CNQNr2HSSmus zQ~{GqJWju@VQpqByv0iTsA};NZ3;?2T7=jkkd7sztZRx7Y|6__f6;RH_4-8ZAq%wf zcH}Tbsv4Xq?pj;yJ9)Ex4VmTHZcv?SHBo}Pu~-$Ms}1qeW4izOD~QN;|NpM)c>7_<5AzrAf!R)5dXo_o_81wv6fD++^=rvSFWXD`)fc&P9i+=A=6yfMF|a(FKa1u4By_ys72o;H=`%E^OwO}jP=}cQ>_Pd zK`{?|OJUDD*gT7OFnegRc?D|Gt+jG1B~H!Wmz>03V}E(&hH^U~IaQw|$rhvR4(3NDv6?qrRX08c7$jYJG!( zrUltYmQee#)cSEx!tHK}FYReC(8H-}b;QM{l`L0nGB?i+{1e2GZ$Aa|C1Bpg`2<6E>(NDWsM>6Oy6tfA zp3#R-V@d%q!Ltz;YMD!jx;SHF66n^I9+fC)Sej2p6fZ(H?hAkd>XdYH1h_cuE(>@` zH2-`&&PR7IK+5VZJ3tYgQb`FwZTTT1J&*InyJu3r&1XqgsAttQEjc+BeMoy1*d)#4 zrm@Ik%y9>tkK{|bzUZ4^<0To9NYr!CskyAF3^8uB2p;b`ED;Ww0}{T}_Ck~kns_5F zX*o;LIr4MM(riB|Z6ocZA#Y-~jf8>Mpmh&o3L!P5>H;u_mEDJj`g-hEk50B^Q*R8A z_nE`~9c_oR5#F|;?a1Q{-O9Z;N?QzfmTR7?HYxw^QqAC~<6ZVQZx>M8m5m2Ig8Q=el=4y38o_a? zIcB>~DJd?UY^*4eKXBdIFx(mpmsHRVv74`1Uv!H3BNyTcs%~@WR98};w5Y%m)LI!T z#g|0t=If}BLN1h=zDX~9kuAK9tiG2D=6R;6+&6?UkZp8Kvto|X&n?!Ooio<%Eqd{1 z&xc+tDhA@%c@IdhH6%DIQ6LyoQCsC>RQofHQvm{ z=UmoALHY{=hJ~_2^TO}O0|*G+y!Frb{jfB1oYs=M6(=M67B8IIyQXpzUH`bqpo^b;5{#Us!v z!3+{4iYFh!1J`y+Z(M6|k8U?_D@1h*o#N8kCul)IRL#SL9J7SXQ-_mf>>P`j9>px`wslk8AJ%i}^PH$F|`bw8nE^M}>Gj2?8E z7&^*-nC>+~HGBL$YL;loq}23_i+~!`i;Xy;mDpDrX*$3coBn}NLji$0;k~Jv*cBkJ zQ53`4uRGZsk3dIIv@au+S6CH`ca=SVq9685C*1zyX744Gll#}{>mvn6p?yYsW2~9t z?UY1+G^=4YyGX_RlT$->pQ*I0w)y1>ac0c(ZG=U{qYp?I^Iq5l!lu;km?#zpO6lOJ zf*!qSL?%*K1$L`;6@V-03kz~2Z=|{!)|yHynLBp+oJiJ4my{L-PM0#HE@X4|+t;Oo zfEDu(RZ22CkNf9MozrQyucJL!I#@7}L657Ldm9inN>(qu<+}u?mOnpo^P_l{EW0u% zTmxax>JByE{BUzH*sOvtMHa%8!jtTk9wxX?=%HJ?K&2*0e+jf)-;?dEODF7>!z3mN z4Dy{=yDgVGmUm3{6Q8S+$`De(W)}46sTCS?Kg%C4!fhrBFEK<~hh%^lg2nUw4*{ zm#p2~M#>L_V1(&B-r-wRje$P`2xAi97m~FRcICp$4Zd=dwEa+>rD4_-yzk4dz|8HW zvw!m&>J$Hp7`aBFC4T^7Ck5juX7|d_2or`}2l@4+*$VRET{OmCk?+@#_NUFC_;bRf zm-*RNToOCmPsah1e$w{5vBab>Gu2=0=jwnrbzZ(IO|cR0%ABG1q@nQpF$&=I8FaP$ zS67p5_5H8zRPgx~++FJf`W>G4nJ=!*Pzb?V}P^xmil zkmp$$0nPp{CwNerJePMHh!xgQWC856RzLl7hFpazleEuB@#5jS5KHI%N(5Y^nxPr=nuxUt#Bf#5YI{>N5j? zcWQBe1U`Ejg9f1`@P$+N#~r9&VORZ>G>LnNlzpSr?5#2Yl+&cjL7)=?aTQLtCY~r! z_qA3}WTD#RxuvLdDg%CExg6-(0m3iPq~6)d<6EP z+Z0q=K6MIkQqC{(s%CfyW7Br+008nNm)u~Bhcc`(K;Gc@L!VU-JaeD3vazJcW%*^5 z_%t9JHc0{^Gu3@=v4dv4R*O?UqB!THtc~ngAh{kx|7PQ1X-AHmGjQB`lC1AbSY9mL z3qx2(3lOa<89~L~@g8YZ>bX02^eZut2Cr87Q7F-UId^Sg%)S|XZ7?`}i=K+K1M~zq z`@9_2LRLU&Ijx06rS0R4+%X(%^ZdUGARtox>13>@ZWJ&&Ju)cwWhoVL)4-?4AURP0 zFuSbG8ls0FnVxs~>P|$=Q?Iv+$N|`X7%;lb24WsBkbG*iG=ltTID+pr(1wilF0F+; zv5+-EFI48SZMx6K~|c-#*Zb~M^lu_3t0Q zY}=63%AVnt?sw@xd>yrGumOC3jTM_#IQKMu`qoxeh^%6$ZEFeKN57b`^kHaQA+MWf zzD_9yaNIZyLdly)C;O+Mmyu?mtC}I+#6p$~-Cl}Ob1FlA2^^)dYmdOfpFVH>?N?F7 z!wc8we|ABq{#x)ofl(|6(3CLPx^SM1rsAZGRiV91?r-LvbZVlfr8jo!x*gtFk zBsKRvTy0gN=CN`53iL{2$N9VhLj|z~gOinm>t?<5QC=Z(RnU3+B&1gz*5dOu?@5a# znadc*X!nn|i(zFq-bN?R6<{X6d-r=!nV2aDa{xK}2`r#55#m>JPq{*|aWuZ#E`7#s zb^;4USPL1WosO^=j4>hYRtvosF6EWZa~l^vI~_V8Qf|snY^f|opEI197KH;Acq`G# zm@Y-lE8wUMY*#!EHv1LEqiFoYC5K)RnPcQs<0-;X93N<29CN3QQyY$~bC%mmFdo&Fst5X&*{Z_k2 zE4J{PP~K(5Ln$H@=T`xI@KeUm^W6L{B>in19Y}Dx&!;#=68_Hh&z^bFZwa{6a>oy< zkyT0HZo5Bxu$}&Gp4ryAgKap(mV5G_tyb>vud;=W?JC><=RX>TnEY>U?LCcE?dX8o T{&PzE4-|b}6CI4!{gD3xy<0P~ literal 48704 zcmeFZdpOha|2IBoDM{s!W2GZG6_T?`IyfXoIjxeMNtx5kbdaK`BqFw)Ca2|Gm=&QY zhM4n0Y;zhm8=GzSTk3m%zn}Z}$Nl^3_s_5E>auoe@7LjZcsw7^$8(P_+E|K<$%%nL zAn|i&PhSFoMDQRGA6--!c*nE1+5`Bp>DJjRcR(QN{p)}E&RyEK00Qj>ojYyr5dLO% z1cq1layuBwH8^$2!Ats}?;fZimD?Y*m8o92;v%ErO0JrmoP2HM zWNW@D?IIOAs+&Egov-x002`jdGy#^>$E)p&c78s}%9$0=$0 z4^_{4SIy<+v}y+Wv6BtGmaG(q5#(|0+5Xz1o&d{7y+nowaqwBmO|OnO>WXn)Hb{dJ zUv`EPqP!yD^AWj8?3SzPqJOS2=AAe`N%6{dl#@P!8&w42{C;ddFxISsmFA-#k%z`K zES9PB(|2uSKH;Wip|9-9A9I#M2^7VG{#Xk+M2sl_lMusu?K2}{;nZfPrPlG8rJm}b z@bF|1JSAfMIl6OoU>D-hjlohdpTT?QzfZNv>nW`%F-s~O6V1%A2tSr^W|x6QYyb|i z_i9G-#lV-fZ$jbJQjQFpHT;p5Ij1tR9hH;_QoO9bVX&Wm3X^G9(@!1bGoYP`X)vIA zg50)m*mq_+6J%VNlu=w_^y6($r3ksby&b!se;jep?XJm>iN(g)&6xq3)w))(aMv9H zYBUe|;D(`MH`G3FKiCejSNc!X!Xj*sY8&IT;n1o9jd#K7^h` zi$;!naX63Xq)*&oS^4m02dlk)X)|kxL28Ml=hmrfMuX2!Sylu)8-&&qWaO2HZU+;y z;a;2;e`X>UYa$DcQ6kkBBXg{8i!t@N`h*Nz1=ND~<>PR*r&nkc<@6-mqj|Qzc%cTm^yS@!4zRz+OiMsj>1WXo6RfXE&D>=vwhX}s4Ef#aSNoKl zldWy+eT2?dA+|0OwNnsId#E2Uy=06&q^o9?2E9uCd{<&qNQCCh_^2GIfOi70thp*< zLuQF=lDyLB%Q@nm_qPo}5&VBb%T-@~I=oJzShTpiPlDZcI{c-u)&1PD{_(rx?=|mL zM?K33l3Y8Y=q6_n694J&NUCUjyQ-IpLdjr~d0}Esar22?apXABAJen;BU(}YAY=Y+ z4HKN72vfap8d<)mu=k}^tJQ--?WwkEsR`U!LF9dgO6fqk>Mv2-nZ(cGS5b$CpMnr+ z+HT9wK9{LnNPVzs0q_28o_H+;UXssjEoc1Ru{q@}G?iJdF<(OQYK};FU+?d-IfE3W zxASVqKNAxqOD^R~rTBb0ddRFZE(^TJ%Rx0Cqn_TTKML8|cv{F;5W9+B{Q1rP=_&Ne za8%U|t7~ot7>DPOqV?VHeV(nXRlYDVekhueP*|dpCRZhDNxSAuw}^_Shphz}=rRig z=>5pTwyAT1WXjG_4h^fmE6FEK)}z!WMoGQoOk%ZX(@6|ih^(FH=s)q?8vD$!5RsF9 zf^)^8nXn1(!Skq-4nIh2ea2yy{=6rpb9``@ z=l;l4f^Velys}fhNsB|I1O?fC)NQ1_wzg)k%4=tbn@w7R6AjEXUp%}&u@!Hd>AFk5B~3k&tfE@(p`g17&o1#Uq>H|!rPOKX z!B`Cir#qeENCHD?uW)A{WJ;!`K4tf#^CdL4)CotrcKZC1BZM|2sf62N1`1(TghAIL zAH2eZK9RU8BGQ(Bk2>w%*NfFvMzxycN$CDQ#WBF0${%16Yf+T@uXhPl*lc94@76I& zJ)C^0XP(^`^A71-nI3m`sFi;F{y@pABVu9+C(#AptP<-Uex{BS1rdJvK zbkU35s@(-&6?NdZx^%un)dpz2vvyAn4~4{E)D(ZXS5QvxK}HIifht0MLr~|MIVu5p*VXi&P(t@kM1y<6+DdII0 ztZ5|UG^N4M{;o-wY+BWkoK;#PLo{md$G= z4|GWs>N5S=xz}qyXTb!EXyc_qlZ4%REs!U%vdY0kat&Rfh1H)UidUgqGj;iw3MbC1 zmG)4UJcvWC@?f^1e@vEe-+k2;5#y+z&Di30m>-Ll{rYsg(r?+x+U1-BX2>Z0V)e}% z*yN_RS<)}Gs)J1vro8HPocO2s{6b)r+0w20_I#10Vjq3lk*zwvhHzdBJ4cGt&1?y1 zB^@|hi!U-cPhiQf_qdM*!TCvE6yCje+UTmbRZJH0753bPqJo|>Bda1grzT-?S#LE20a8q77e4YXBK5zd+l&lbV*Zee!62 zw=G5G?Ue~W7WP&WVwu-Xi6qDe+8gr~A(E0OlkmPvFOpch5v)pvwN0BW%P$a@+;B8k*+EXp;k1D%rrSJ?8 zyxCyfGC^5(I$`2)Vez_;EMs-W@e_L<9p$Hgiz^(QrrclN2-q={E%+Y1cIHu|#-T#W zNJ9Qvz|m&t^C8P8Uwn-pxbUm&;%V99Sn{`j4$narrz z0y=0*vD4i@SoYStw4>MnPv{VEJ6RfvZq8*8?Pm6e!t5UWVN;)Y$pqxf!9cZGyiVhHuEFBTJf+esZ2*c*6*fE*2)Uf8?a@I^ydq9%ghe~ z^nz-urmV_)#~QP0sZTw;?VmeK8CVc=_gr-NYI(?{6i9}}Ty{=(*!I1@t?t)osEvhb z%$BXp-D8Wifng4gVTNUSJ0NcXjThYU`A186abb-N)>B zCjIqY^J@<8&`39F`j^F)=rd$#s^4~l$)ey48yHc~ED4&}^U3d$y;G~fG9~yWIH9#^ zS6Ao#4v5`l8Pc(qFU3OSRhom8YwJov{Di2rr6elult@x>I0AfDk)+l*y4JLL6?WQ6 z96}$h?3@`D=q{_o?-uPe{c5*-%gptlosbezSEI?jXf;kLe%?HImg3izf0a7!-Z%Mw z*w*qavEsG?QC(g-QtDS7w@gYXcQH)ze4e^^(_X7~o>60cZEt}!A<(9Y9qldEDuqyH z%kA-ekm3WqWw%Y|P)M7thr6D|od+$e6P->^rw~ddsyZE}H@59@euLKU=nt(7w(?Sx z;LqiAuheIDX?777ag?2XP=Q%}WlHEor)2+f>oxxhvatmxE%4yUv9(UyOof1gEE`MW z=D6bl)Tbq6q$tY6F_z2f2KO=o1n5z>1|-ZUA|JsYImKgi-BTcIyx#zFy%))S5p zTw3I=g%`-9NSL7yL0?GB);pEuetR?Ljhq!o!qq+@8?@iM3;ikg%*qO+pWuy@W5D3BGy0AUR-vQA z>&wzpbpA|X`v}XGo*R&d;`oF_Q|0Z)M^bD35G3W$uQld9BVqd+anT^70^N6@q+1_- zT`VM2xAb)8%%epw0uEg?2;iK8t(uYgdBsvf13Kk{@l=c?h@32OK^K1O@L=@hDYoYB(#2;!DeZCZJ+;GP@27+bH8FBgA1ScArPo$_YkY^jDd_3> z`9ke18@BS0k4j7EKm%Q1>VwD^U59y$t?FPA`-#1wW&~OkhOAmVPwI95%(Tgs}9A9AyyLG z*JC8y_cv)=cH6s<08(rGC_0es{89CjSGXiU6CIR9smP&T*>&kmjJ$_7e~ak0gHl_k znESa3=5RO3+?ErGcNFc%Y}<9EIJ-D<6|$!je!(w(HBuuu(!<_h$W$NElo1z^9VlB9 zg}zt+Xo63_H{pf)E4KZ@{Edtt(35l!d}-``^^t!BzyaGBjZayzAO!Xwd2scoAaE^l zRi6Lo0xbai^D&MHuKbG{AkcrLWl+qw9jMLkz4m;9{KXa^kgV={B)k@Iaf)vtXcSAjrRm;NKefg{OPc#l^>Bq`jvqvhE~4W4;)9Z z&oWYw1F3Lso?!(px`{jcmN(;Fy}--bwZe@X4bx#~KnxxFx$~Kpk?a}{reo`HM~7yn z+K7!^0tb9|bhzcIJZx?k$Mi3W(sL@t#5J5Np==cTO=rocjutyqevnJQF{#H`OQH=x zxWDJy=AxFLij?1%Fag&-U3pW;WDX&5H@rNbaaUSxeh%8;;4Gx3C_ZlY-z$natoIIt zozeZ?T6KW+HttmgX7~dx6kor13@uhuF?k1F!DHFh^lio=Zdg{cvuGv!bZ^Ac8})&F zL)*>xgJ=Jpzr9t==n*prZ!GVEYS{!e6uul4be;*N8V_zRWSb^Ei=y)CZ=Dn6ee?@0 z?(e_`nLG<%eShaq<>@2gBwHIs{(W17(pwG+#3`L&GV+tCH6jSkt!uIDELf*Ru^A6S zHF!}1<8^g2$jXXNcTEKU*^iYrUnFNCY7tSgr5FgDZ%7aszjX99Ih!`UT(Xfcr7_q5@QcslMMnq36d;)Pk2~761*k+gTT3>I! znYjV4L7@CJ5H|XeJb1+C>)Ha(-aR(ch89i2^}EYOS_;7&a%BQkRGWJ`*)#GC2`QB2 z1$2UD;Y}G-QrzDU5DuSyhc4eNqd_11kZBlZVhCfnV$7Xb9;z)eJwdK~tPg#MlqX4? zB#6<6>%NTUpjWui9<+@of`JRd^#kv$u;EO|LCsV0%m(&K$Ri3@b2? zaW@4p?0y^@3X2mPtBD@_9uI=kkeQ`XoKsZ8+VN%WDp(K5nD^S^Emn-)_2^eU!ZvEw zMgA}Sr}ed^wc5%eCfYH^Mr2CeENS9IeRwXxM#1#;50&*xF%&Q2k$Azk3P7Carmi2k ztED&-r&X3EFMM+Fw;|QUsn(Xso4&4jm0A^Kk}}6y`Z%l1rFy942PIWcR-i?M$@d4H zqNlo58Ftmn$7+RHt2ENz=!1U%%8WP~#K-DtYSmEcOf0D@kJx91*)`7PwE2uF*H=YO zFPAgIs7B@G+4a}uR=n-rW;Jjh+rCj1$In!~7#Kv?0wwxm-TM&P<@ z$pj4hohdT&+SwyT9c-u}Fy>6OXo^S7sHOMT@hmTyRS za%a9tVUj5QapHI?d-7DO6`!epOf(Z`ch}vr7&gjN#KG9^_8-#5b~n6d_)pH@IB)Z$Fhu0$Vc-V%xrV%EhWmvkuD85tMfq*` z#hB<{chP5}6SrdS9Ui9X1rGQ7b&`1-r0oIY6yH?v=tQY3G7liN=5JyVE!s*s$l9Gl z2|hnlQ&Z(}nhdY>FDsNn#6!iT9ywH4gyykW7-!q@KF++zcf?}3TMFhWiams@2`QW7 zWl)8QhPaSGi;J9oOB;tkQc(09K0d~XN7FImwIB>a7O6E+fs;R}5ctGhup|m;7}jun z#~GP3!H}IBKu}A2v6`{P^C`rN0tO z7_VwgJmNW|N4;ED68pn25~+LkLHZDK_{v~@uozQx8@p9`?#p6w-jHF$xP#^mD*`l$ zJ5U?!5=#Cx*3E1_F&%`UkbgxR-6oTp3AXJzV_llsj!QGq$`FI#V2H%L%dC(gR(q8^ zl>o1#EFOe;lm%X{_Z!NR@;EvwGc>xe&=#yTWDMAGh@qDal9GuDfv`w3pGNOjom{zm zZ?~edkDr`=D+wU3P|VT(B(6y_*Mg{rH315kdAE$!=*M%-i&8=!+F9KmUzyfj!7U#% zCSkAF14zchfv##EIAVFFmN}L0%VUKJ(9=%Bn;poQA(z^xrzF(9kX+Jc8Hl{p0#tzR z&Ev4JRYfCS+GeHz$4X_IuW_Hif+7RHAX-#Fc!A`_egW*Fy+!iij}5dWF{gFb^Zy5{y8CA8eSI?;`6Ke`Vpt)I|CoZB(r-z2K{FH zyrOzeh~X`bn_2q#kBEyzC&wfrd$Y)9h|Jg>Ie+4Ey0KbxG&Ms4$PpcNol=3ME>n|+ zacEg|BtQ($3?5S(MG}*)NZ@M%haG1IAF<5`ut#pro~3SDM4j(7&Y6~*G1UPtNB?!` z9mx-EB#&5VNkg7k-rAXR!zb3SvLH+21;+iUG(slj1}A=k?qyfzcA)UrQg0F92(3*Z zN@tE~`)Ncz=4s4v*1^K-G``kqbwkYv9uPvE>JM=1zQ0RGi#%+~k>)$cO=~|nP>!mF ze0)PiicVw*lGn-AX12$;R*PLY`(CT!|`?ewxRkI< zFvT5R>xk=P;}#tlc?dS$f!d|SnMmoWq_U=0rA&EWXfhh_ZW0$3Q~dHQFDmuUSOW z<^jjKxMU8Ct{slL>Y_6Om#d!#XkI$4d-{{8X6yb!nH=RMdvEk-s6!5^X*wHZj98-> z3}KNHX)O)j%6qlim@q+(B>!X)x96fh!w~X%SI-=JDAA)pO+06vTHJO7q4DZPNCX==~BhO%XU?QSoIbP>vGK%QETaSqIwnzEJ-`l4B~6Rp)~ts)bh*Ku6)$ zf~gg9C=xnBNk<)Sq=iS!mCnH~0QP>{dB91^;gt|ZavS>E+2m*?#|I;lKV@R4$}}Z& z9l)na0`zjsTRD!E+#)`RGzXId`01D9>N;ixf=BZB9J@}* zE{0i`HWjLN`v4vW*W7_RE^(&@zUUtcV3p`|zg%}dQeGK)aef{L=}=l*ZW5@GG+)j^ zI4EQR#BdHAK#i(u(SMO)eJITT`LE{38_w{BU}K5y2_!5!M8O8==R^RRpe`^s2Fz! zOp|9=NZymR_bPzCgOWqH+$F_GeIA(fYOKF6Lw`Q%0wLZ8;u2CE99_uLf}g$i za>hnRh$QJ?iTo-4BB3|4kCjFYOGYog{usuBHFO+0R0%A9S~xI;B*bWy0M zqPx#Yki59yIP5o(C4KB|F5wKTu7sTRQUYHzzeu5?YMHVd0rQjUONi!2pBY=pEX>Wo z!T7$WM4zS}AEZcCg}_O4;o-MEsLFH8e#rontdmh49^~FTqf)Hka4i#}hCQcNI9gQb z-D#T)@nX%D0-Us3k!oX+&Mp~ftFs)+a(~ngZR4mB&>C)Xo+ zKAZ5C5rDn^DKF#1@DpLDS-*Nwjt9*!EUb*gv@ZP7g^ft`Ud$p!=I-eG66NpBsr>=f zJ0RKU_y&h0%}$l@g!sy6`AVe_IRmiJaFdPZ=svR~P-s8*u$Y2KtoZ%>pEiVVi3>+n zV`i$$8%H}Rb+H_AOOq94hcH4=P=~Fvm@=pr@woZiy_R?26$4oW3ASU-ewRyA|j%h znm}g;t-*PmMXy56{cW+QiLEYYsH~`FY&R@%R@`ZjsUKbLh?h41 zRa|mciL(uf#HeDNNxG^F(+``P1Ur1lNQQk+;Y|MCw8ym968qWesudoEzYKalqFe2& zrotP>>r{tr1nBwZ!3LfzlnvtF%Ob&o18?TksLUd=gT=;1_l{?UL8iP5^P9|u+$jWT7Z`rz| zMJM0r@kI`%g6}T8_v<(gq(?KHs$h{yA{ti$^kgea1?V3z-zR`uHd(!O*(Fc`C6ONd z);s^D0Uv8IZ%5TU7V33U@-Og}>fT+V8sb|UcS~-!()W-UrM*XN5biqz zr;=b7E$+CLm4;n8JJLJY7jXL5_P17c;7AkHHxaU@hYG7EaAEiWK#fk$2(l<#oe2GG zpO`F-UEAPJ)uzzj=n##J>H5byX*Q`l)6nau7Gpfa$D+I-W(J#{gE$>D(}cZ0q?9N( z#r>}%%N>JQsk`swP7+w9I}9umepMD09&bQtW^}+_@zdX{y(`poabn>=tEeSneSgqJ zRkO3#esNDd?VHZJ5p&|KLXfkCF_0A&Ut*>b8*H0a%q8{$H%gN*-WhA5y|srhOSH!w ztp{9&o$T&m7wjgT-3nCzBegf1b?Hi*+7g~8EpE48NEMo}H7UcB!`n*lUcsgQbqoav%`1EXbYiRQLAn> zwI`GfmeFyEa~ACgbBLZe^uawl^YkE}i@U~db4kHR0DfCV=d8dtazgg4V{Wio<$GuH z?mXBg8GZ(tYmX$nerPN{#a6FAQF6VFrr!~DC_@6z=E%PFdO%&o`GmWZ_?`e}ad{Zw z-OTkL`A`z(_Qmn7rG#Gr-cX&znf&!p7y2;ylI;Y?W>N#wew5Zx@yu%f@LK+o z0*f@TbaYF#$eis!>I$%NcJJ5KXW55h9&M|h*Y4efSK0johxtJtO@1Lv*6v#x3uyp@ z7XY5qQtFP}-qfnmC;XNoRNEmC)|<6#UgCS?^n2GZL+I~Qt5HSv-%Nf0S?3S?_Axj{ zY3XyQ-G$VWf*mM{!JA!2bgbt+{>F_1s*tfvVHoH9Vr66Pq0>MoO*Ai*l&_1C7i+D~ z5PSe_C=ODa%UG)%f0uUKcm!RrMPr7V4gCznw1^yHk%?N0O_xix#@rDha_|;AJ6DHI z3KQ?=cXIEy_`69SNfBa6iV=)$-2}-T<`iM6qlVm6z8Lnc`g~ypyVc3d5FQ*jK4^Pt zIyxcF^5!alpJlFg`o=_$HO2zMh;iaKySqvf`||7yWJZ#gn8o3612>plgG743Uh$c~ zB!g8+foKxG{CF&_YXbYOI^*mqbo3!9se0|$+YfTkHd(5!so}Z_7juZFl9yVLDlfew zM+^IMGkZZ;&6csS*58*Vvppol0@0G4QA-^9#{i*zM#^5m);*WvFTv0JoPD-0HZfq5Gqxl&h zf1MaM_}vf4h>u&#mR*XwG9fsle0fN~xPpYUz(*VT<|dIpl&;8sQM#W(wxN<*PhR@? zruy>z&{seFVzNr1?mt5{W;+~J08_f}7+}Kw*Utv!-v_|~1Dm4yXRairC`*g(i zBhj%~JsRN1`_%h9=hMZ2R^%suZq7M8{D<{Oi$iJU%Oq9T*BenrUb_G~6KsEHg^?%B zUEu=n!x_nVoAa)lITO=0cSs$!ku|bF*9S8zH81wT$wMfT43PH@*v{L{K=HUDFe|8|Q(F?RpoG2)a>cwM;GW{<0V0mC242w7dczcM{7EPZet(UqoU zAQ{NSw6``?y{(v2GAs`Xt^+un6ZFu(r)ibxvl)Nfq}n28tNr}fw>hz&SGD+9Ms75_ z6j=D;S&X7_lz(@fz`*|?{1m=GN)E{A(kN*gA%nVfFlUm1s^3<@WS%iqxK92e%q+TS7f3(L>VW;zhg7NQR{OE(7LZJ1 zSn1-4Cs#JtrhP=sg^pu`w?r(4C8u3^J9{Sw;e~B8$oHm-kn6%ioVo_9$fRqfylcec ze)b<)R}^j@dFAF;#7Renk#2kB0UDE`X=L61^t@n$+hp(R_dYNmzwL4sW@TQa27g*u zwfow3W9PN72fom1scYAx7Tx=%9~c6-!gZLpa?-TnWs(mC8FhVZ+{BnsB91*j(5qAk zh{j0QACJ1PFDDho;DizmgeE1RE^Tr1~P$5QtXEW!YZm8HZag zSg9a!TeF*q&5Hn8XSXsci>3slJ<3d#5(f2^^y1lWJQ|D7iyow4JG_I&-o?3cV19?o z@j#cwhJ#8Y!*v+FkEK(S>ByS^Z+95mFmB*I4qzB8EsLx1bnXkZ4=4u~m2jyOu9L0L zHatD%alGbx8lT-@8-ITGr}u~a&$~R>3%+ah=wg|#JhU-Mw1I_5+NKUbqak-eMA65+ z#^NRu!@1Y&?b?~(2fG*Xr+Z{=}S|;>0JIo)<-7iJc zS!z27Ax}!WgV7rToS1X*@numK&H#Tb?#WW;e^uZ+TpiK6-!#GJ^Nb^Y>V7%^TI9B$ zw(t-H*^Ry89#PtRxuiShuyXvFPm+6r|d`S-N}M!J*YS8}XeZ65DpKwV=Qz$q->= z0Y6p>ouDzN*Q%tz!#?}N>|)&Yo4zuzSdXtxWh5_$G?HI*dZx)T{fTVXQbpO%;BTvu zA$O=@QC$%jG2nAGR+)`cir70lNt&gme_28MgCzieg+fD$TV*J#sIk5>gu7Nch8tvgMCI?fsR~?;v1j|%-)!x|1TUyKJ)aN- zv1^VHg!_OIk4&CuCnLfSDh6a*;3Xi(EzLM{{FS;JRP3E zmMB??FuIy7JWJ_EGX0`oy)nhzK5*{ z-5x2N2P(l@j_B?7dqvTcckGmNTivUr;l=0Nq#H4S?x-a!nNr)cc**O>Dj>AUF!Fx> zaKaNzTkSj~Mj|byBS|1QqWtHDH8#q|Gr|tY_|M&!shj`a!7)V?S}LQft+B9x->L|o za5BXXFe7k4=E8?_XOG)>17_*loPC|#I$b-T9|{EW=dxy}d!<2%mKbQ?lusro4R zqO&v#-COFD7xDecv-4<$&I_kLmuqLbAN1VA_T7vx%HO8BUE)~?){&tjLbkps@C^e- zB7OE%1+;gxEovnZ){I2RF3E+9g|w8>rg(mO$W&_K{sVs#RFN4gmaA2w{Lmt8BRxSSM-|)3&GvXy*bXI3Ve~^wA6?EInFz3yyNM$>Lv6UUqdjP@cYdJ`rSu`TI!J3 z`td*zH5xhNyqG3D;3F$LarGr1>n>Jg^ZlZzZCRzp#@_jH1^L!_DRW&*pM#v3s4JWC z8Ev+y%FN2}vjzTxYj(oqoDWkpKGwG#XkV(WLm^>qD8y(d$33mp&%|^xs;3W^ZwW8i z3fMC5Kmp6`byhOKri%A_L|LM;f)-8wkvh*Fes8K2Rf*?+l3(I)-3^p6f;*w;=BHN! z(w>!A$7DTBjgXwBLuB?09LhDf>v<(C?-qbo%}w~_)9?GUF!+T8LgmX$q#@;mEuqfF z>+IL0SUa1rEP#|U=%Or6-EXGfP0Vg1(fMAk$#qbwz;u+=`ewxN865=|LzKFu!98`= z)VUIYEC>d+laF<%bXD(y)9G8zO#SX{h6tsp<#gMkqBtK;u*tL^6lk{2zvNMlZnxl# z7QnXrW8!uXp>ybpIaRYHd_8Z_e8`qy_w;adt0W3NHo1?VUT{>2 z5oK|eupC+l=r140?K72cM2U@LqYB#BFNDMP5ygARO&jn<8Ym_3)5}4}a7HKXz`+}f zt_e^IOrMv%PiYdfP-o!+ml^N>EIBG>^t6Lss=BFG!3$Ge3?XXPHp8!&R~Pz|;M)Vj zPVaeGUm0nlj3U+S@HnL8c-Z$Je6j)PvXiu0`!DFbmdRCNUMI|up*}J0;45jVr3V$% zmF8uqs@wv+bOp()eif!Siwfbbo`6`c_E_M{$BRW<#vO4jFJR7mxP)o3R2*^5uC{y`(;r4-yHXPD_%vR*k$pxO|X0~ z8M+CN)-;pwk-Y@QD*3VVF+Xe(Bh8_J4nnE*nMA>Ly}0a7IdsGpJk2K42zt&lGLzk3 zFGvQ@JSXM#Jd-IoZUQ*YkrTvQVocc@=sjx4xzy|#O%8mA#EqB#IFBB_NN0}0A%*zq zey*QKA)yw~_wGWWN&ERL4Bp=ClBHZXEZT3@Y5Q4`S!U5qRnE@!m8r>vR%*WgV*o z_M^;9mnti4Jnfj6LrF!JaFa#F&XBUq-t8!+d+GxRA5U@t|Ac{~eN2|46RGps^R=J7 z_xskA@~W>z8;p*ptG!5E99<#Sd{F)K)UbmRr(Z6+amTWNO}3n5ee@*U``^b(yX!6p zSyk;{d0|v?6}I&F=2$EUZU8j(B2S20?LGY?;k9r#K>=c*{bH^-A7t#7P}|X@Hld}- z7g=wbn0KTB2>QyDo`sO>QlwP)X z?%o0DUnA}VE1CDBRBgi?mQ((B1}1>ge2-+?8T@whGY{f?WF4imAFk{*zV|XJL35?2 zuyw7I=10!|e16vQwa)uDH&*nE_#(MukG<=v=P0qyw5t%#Rr=)5kE@5SSo1|TY!8>~ z>r>J*qfVz&RRMv(t;Bb&6BXmpQs%)A9@?q9g>mK`qPMhuOq__<-WA~bHWH9Rws=Mc zX0NOHkB>@dDqYcz4dZzg0btA#0AnA2vmQr(GyU`|NZaw!P<62pWO3o_n<9NG1?z`j zPSs#{sSGKwTcVALe7N@Y5^2Iz54rl-s{fM2-WD zqThjN&k9J;<8ST|I1yl?R*y3>IHq~f=mHO@Cu$bs9GgD^y^|eG-f^%`0 zn{Fa$4p>{lx2l2MN#&4)vSZ|^ykkEc+z|H}=l=G*Tm{K{3UzwS^a9F$l(*5!eXhre zmv?VFBCc7ay4xv0h?SF}$uI|ha@&6o7uD=NnEfK`;_l&4L(gjfluPi~<1}eF`+^^u zyT{$Y7z#9i2q6G1Yktkw)z62xg|^UC&?kg3EQv@ZMAkalZI+?R|gyG}F0|8zgjaiJ|%(K|Ad#zo<<}$d1faDf7SwhwL^fAe8iW&iWeC zR5R~I=*Hfi*g2sw57CCgDnFnt$gOQ=y7^^E+{bZZgY+c7=;{vk`U*+L@bGJw7kB^Y z1~`$qX5E8k)PB>|gS^~e*veR3-Kcol3B;ldqHASv^{q1N?c;eGo5)Ifp0jMX%fg<&Lj3}-Lq?o3$sAX=T82-QpL^~NXs?BFmoCxA1 z8qRx5Wnr#V9DuDY4scBoe@x`6LQ07?_P7?7;BA%Z>J=Fkb7hYiH4UUVWoibg3k@h* zSc_LH{i+m6e(T@&hi#A1__A|=LwW2`yX-c^C@wI;)FrBG4PM!EU6p7L9!N+k43<7muHvSHI)sFl9;0~Dt5qV+n@T^ zUiPRZ%enPej_8VgSKRpTI6hVljx#q4Yaqd#!AM>Wk3lSt*z@OZ&g?=_UHeChB8Bob z&5~yHO6}}x1ZZdfZ27xa4EJ}k?M){)C>YqpgfcH&&-M>4F*``qeq)fCX%swt-4Hfa zoEUq!qP{+AQO@`d>w9jP$kkjyvf>U@)0z;O&6sOKneq&niyZpumOaY+q1rym@N^V-ae^Iw8zjf+wjH**TT65hp=u$NA}roAB36_~{veweP^;+@(}p zy^L1pvH$p>SrIZdnpx)(OEP8Un5HY9xV~Ac%3asAnfLYwYRy_5?M55!^s>IqS zi5qMtwbN1HKR-v9>_;dKkjgUUt#)=+2aeSGK8*Q~#pnspF$m7gz+@`KBqTg~GR5g; zk|}qk(17!w58R9=l(?kAJDE|P__0+fC)el3E5Gg+&HwyM0{uJAn`&lB8NbqX{T^=Z z)U2B=p9O5iEd}lNA17+a$C@9htB|SxB~;R9%v&Rw;(TpC{^#oW=_;0qHwdnGtT(QI zrO;&LKNd!jz^kmlo7LRARE-cs_GrztZPlNTNpk{-NsRzte66Hfk&3%Db`k@ntgX@p zfKkH8fbQ{r7JK{LJ9Jh$U8BU8L%TIUyn2o*_Q`4+DhwFb>`_FunvOy?E;FcM^(Okt zADyojy5r4>4C5vIMBW_zT~A3h4xl0uhMbLVeW>6>194cU1~x^1?Z2H<4Z!Q!vY>zu zxt&|*UA{YJg#RIkW`Q26KXG0FOH(4j0A69VX{#GO>`8NU5z9o!SO&D^z`9jto~z?= z*%e7qzf}fAOcw6W>S7{(G>6Vr5lUC6$Lq($v4k&v1PlZR42u1=rk3LkT{|jxXE&Ca zBuM^|skngu>??-1&C*Yq8*QjrFuzv`q?fB?>aLu0BOJE;wFux$KQ`f=B4W4+xRnH~ z+513PtKq2j3r;w*j+p5dHUfP**xh|e?_3aqJtR-1y*(9YKuo&aZg_G*w0TId$*-@N z5<49>8sQa`L6b${Xqi<1UdyglHzUQtpRd$iE{{vKzBbt$O|)c@(3I*+q;>+*Z*at@A@*25X6fj&q$aO5|(6!X%#|K+bn6 zmb^J53~=EyXJZ=L1n5-GC^euG!(%~sGtXyRElpWnEgb2)Gr4b!B8~#)*Dg0l#jU0C z%6s4+s15AMyI~n-;dq_N&6_t50B{aLw?zCbG<64jk=eoCXGF)Jddg(>xda8QozTAU zxuC^9gE7i%2;0Y5pP|0!B$(-DJr?~N-{s%h=Z2NGkLqmNIq!4G=rLsit!qYH%Phgh zZ5n^m`g3kk>+_BzBaYUY$^yrraezyTy;d(45;G6@ub!3uKC{kpV;DnczNR1x`LfE7 zpf^fn){~4gqe87}Mym(w&m}yIRxoEB5}tONeMInTG(X+$ zwL^^3F}?g=QV~Es07|j1fa35r$32v zIfo0Izk?iNi!|-qQjy50h0Zv`Zwh|Eac6KQ)xV60*Rk{~TkzD6q^y<-q1-u<=wBk` zBXwb3$@|PUD8;6@I#Nd_vJPb2AoIhPWBd+eY*NJgSSI17Fm$Ma`IH-vGT@)JD?GLz z7X4!d7_;>YfDx*VWr%lXzg{}X7wM{;GrwbdlB355 zI;lJ^HOLKWBHrU?Ims?iEdyp8T2F0uat&_Ax59eCPM$6`fjeh>6n>mefSMvq7ir(&;R?fB^KhQsLoaJgSWn%gj#$W z=6%=i)II<>wrkl#IAU&5Kk?jowXYJhocR;Q+ok3>gjRi%2DlPSB#L zSmxH{D?q#;IK+~2wsZlq29LJo_3(a5nOKpvTs-?(M0u9JGPv!(Hu5&iVYk!c_<>6P z8ebRe^rZC=te4CVRVE94Uj-SQqy!6+%P>B=q0Y-L2hEbsPe*;-1(zBcDSxZKTgekB z^bB&r^s=ci)__F{2#|nrVHo1LKTrd`u|&39ZsYl1x`5`l+WaT$Pe`DZ;q)&5H|Y=8 zno4_>_?C0;FPZOQH8wW>1jb66*K`T4$LPV$krW6JJ}P|nZg|J1`tyX&QnC5#s`KV! zVOIAQ`Jh)W%MaP8y{HKo(0;B9%!z%l0iGDudJ2TLDK%mg$SQdOXkCuHDk8AgyVL14 z+Ao@!=x2t1gRC`mrbD9Z>|z!9*tz${fm*0k9}Q(6Z=p8hf+SJO!-#^Tc#PpO{!jJW z*VEQ$$MJ^6r!DZ6Ivl%h;Ddkv`?rf+PNU}9$%)R#S8J^&S7}+y`q>uwpznb=4r8`~ z4d|ysEVYyoVM75mo|=6e%kNe8Y$ zS$InW1~3ZKuB7mIKu&`}OPr1tP7C`3j)oo| z=yBhO=EsS5sK^Fw_qF^G4Vap^qCRd+Jm_ow9-Qt96J zoL?HMUlUG4Z}W7u!7zDXNEnRoQ9!jCj8eE~_D4Sl0+g4AnXU4h0s}NGVaEnM{RnZtE`lA=_lxR3F_4K?+#n z)|XBed?zmh85rFU!pbIAAfTb`J|QC$bpslIP5C4J)>H* zl9s(xf5gG&)b9f4((S!%KCx7k-k~Q*K>RUN>$L1r2-Mb1)A=Re5RSKFKz zeXj$%(+yw@f5xnrX`P)i_opq#LC+V-rK9DMLx31;$5RuPdkgtU$-7}rGtZc()~=~` z<=k#~x}Jr(-(_$s%>(P%2LSPiDX-}Kkm(|yJsYlVCbReAlbGOsX?vqM(dj||gZ#e_ zr1TBxnJVr(W9EJ^&88?yWy_P824dm8DWI*_1ARwJVMkEz3#?AWn1qO&mT3SCLA8fm~RJgcMH(+4j@1faJ|hfQmEt5 z^#YSEHS=@5E&AK-s5RTvv8S)Zq{Ehm^_VH`o?fbygAEGMFUtcABB1I~gBX4fhF$~6 z$JwYFcz_%#sY0S~cBBf?%w;N{-ZkkjsqGep1dzbFn6+w5PG2pF6DUka2oD}tDJ zoiXy2jw_WwV1sXbxa0g%FjpozO}^iL~oDAWZj2)gC$!G91^ckVpkE| z4f}NQL)B>?f87;P`kKEn-f~&|RiYB^#^7{XWIZoT`v)v@0jv7bBzW!jMe{qg%J=h$ z^GD5W(MySk2is4BjQ#jnz|6-_u~#|Q>ui**T!62Z&@(>Bi90!c*`W5P{2j15yZ}%Z z1LP0zbuWXGe}Xf>P+X6kJHS~TCYjVMQJVCkC!M167C(+u2e{15Sx$!b%n_s6a{~q9 z_OVKz*&kH#2y$HXp}Mh$^3JhjC-(?XM!!drXV1oW$rKAOj<_3$uXdNA1(wZcZnVCf zgPx>ZBLJbWXQsGm6zIO#_o3PH@T1-(^ee!0@Eymhj>yQ4>VQ%09ej~Sv(K}V()PUd z>oXIP-Kf(?gviS5{#H3AX+4Yh?<`%9#O%cvsr9*CjG+3(mNKE<9QV!e{;P;I5C;IH z!9kInG!2~fBQBiCgbyU!n!L7F`%{I!cb*+>I4TOjTP=(sIMiWWT59mL(5>h?=&3$B z7Wl#o;ClmL1VGb-e}j1T3xF%9lnMKQE?aMu9pv|YvUxABXB!HA?C|LXnz0yvNKDtp z3A)8J0A2eQ&T0m3HNy?r?LSfreaf8RP!WinPVZVyY6CrQ;DZr0qSLQ3hyau?mj&qW zD=UA(yx#(gY?Nux8mbA>;iUy^s3?o2R}c4TI=U;Xm)2`+_E7Ooe1}tUO@w}t(W4$3 z4jg%=dvd1Iz=W&tmJ3LgVgHM`cYkEEkN?Lf9Z9M?DMDx6D(i%tPnAwW897c-jzc+{ zZAwzf;jUYXoXdF#8^*RuQN)I3m~BaH%r=Y-v-w^+-0#ol{rLmFesS1byI!x?^Z0l^ zp3m#B*4xBWpdIgm2fD|;0nR+?GfB{@;$Oik8kh1o&#HleMebrm?~0jJQ}8mNm{sR# z^-Q~@GzF>Z1t7bjFHa?wj1RwrV1OdLo^*$d!CUsvY@V-erU-`jx2w#KQ5>OJVO#_s z4jmROpo0))LLtAAPiUbLxma5Dt%qh!p9yHEnwtUsxvg7xh)u|)Z;DcnvW>SFNvSaZbcOHmmOJPV&7G;Y1^{zL9tN+7QHsVhxm?Va{hUeFuwFbpzNlFQsLu z*-90JRD@a*=?x(74OyogwGiFfesig}5^5?083!;4cs(_E=0@YfxD?-^qKCh4Eng+# zG-O)%eVn)3?uFYHk241}dYTf}dWum{&3`)_^Lb}!s(@oNV?&(E*z zN-(a_{a(|(lUNdYayJx~jlGJJ0=auB2Cd7&9&COocwPv+G`Ev}roVbcS43QkFLG-_ ze!@wR10d=Ef<<4=25^nYVH3$@g6UxY{2(44F-!t-TBgbLEGcD&{*$B&<-Uy2rR&@o zV4%p*n^yJq<lG`3IcvzxEf*2K1CB{5%St>`Vkfkd-73 zyLIc$Q$dtuma4Y^SQJr1QqDraU(=WT8)ti6G*xmF0KoyUvEw=Y<0lzF+~?I}_8Wvb z0!&6&zBfhkk}zlqdy9yiewq`2&`wb^eC7t&(IgX5I&v&s1=abgnP*p9T)OP4O=e4M zFec9odG&iTPyB|O-bVfsU=x6r8MtgNTKuop&BlXc@}}k&%ZLkQEjh=>s^HnOGn09W zh~n9)a<@j^%)S55e=9CcnMzyFAXGWf9Ah|^;w=@UmiVfu(UL1gDu@;!;{MRgi&7*v zAFdnh9*@)}E>2Hm!+D?g1GE{yt${$lS7ZfvqmhG9-@&>!tA$n9)dW{bF<&8k5j}zl zT!6AMxAoBKpy-}VXSJSX1Pkp_r=9tyS>g9uGyp42`LUdItIqOWm>*m*c) z{sk$faEeuIrePDS-kD>XgBM~dVM_S`99tOCvqSIyUe)q-G8{Ix;>PTvj2^oL;W1MU zuCA^-#%pk;3)zLbFeNrZ^U}*t?o*E(YYni=nNt5UB6jQd9Hu|V-L;C1lCn^YdXz@Y z~Zbi7+LJpPcX{_2a@!? z@dW&%TW@g+bG9NvNBXi=9bP^a)g%$p5=<%zbCiaVeh7? zJTcZ9Xf^*|o2;ChK*jUIk{0F?P;&{YsFnninY@&I?ZStoK_~)hue!|(?qUt4Wj)dwiVI^KT;LsX6oTR5~QaH<@dJAq20yNXQ zK)3LZN{+R1UCfz6y&C|W*MH#8P_`D!RBAFlH%_+20XZ36~l0Cu7PP}g!}`m3B} zn2O2}YMM+vY++3fC1?bO8NIm~*+Xgql&dD{H~@J7vW$hqs&gYyqKF#1-!;Ggj%^K_ zD1sBI^*IIAP01(`U+PxCbw|un&?&4>|LjRhDM}<|={^O>|8D9#najF$qu@^`@}GNA zBDV!g6opvaWE{upYADsV9zyBW^ff|slrtuJMZ)8M7s z1u*w83Z#E}`nG24UwI)tZ)$FKWH4s<1v44croyS^D3%2B>du9E!O24Q4U3c4*Jz zApf31>!bnHJa}oJct}G!bO!DB?0)GQ(~MdYf)|n`I@q%3r@?!B@+Xk^cB@Q?J?|2Z zeydeYHih41rJ*)HcV?&zD)AdeI03pdghl?aLcZz>NAg(6z`d;}0+bej2$<&dz+}Q? zFq(r#n?x+%Lr;}n7I!D}y1|^CQLx>Jr-u0Iuw-#LVw#qy+>F7C@O-s-@C28-Dpi#Q|&)`dZQ#?8ShgZQV+=0gsjG#+3*p=yP|8o0DQKVLCY8g85>Q$mF-_f zo*bxs)_U|4SP3wA@8{l~ST5DQvYe!B&29a+$6BPR)p^A_ZkSU^G*QI>r!FeRB^ak~ zdc}+pbo6IMFr1vzkV_Jgx5oS@tLQWiti`&V#)tqmM{RYFjl5Kn%plA5p(P6qERcK4j-8a9Ul%@23rhW;0R#u%w&lTqBM-Se29CP zHos-cECg`U)&^u(7!VXlI8de!D*t;EKO{%&Iy~H8<87m>-LoS<;tg;9&}#NjoW`OD zN9xiT&OIx#`>pommq5Vqbc2qgXRLH81r@{A`LgUv2wiZfusZ$5ux`?>=<+W0o-Yx; zK)XXQU=M4@n*rl!)CgF~GuU6nMF3p=wU(*|Ku(rTt9YiLgw$;W3M#^vETf_>F_IO5 z)Xcmm<~3GQGT3g|tAeKXHi44@8R)RC(Zi2fqZg)^3y{5N7riHvKeHcyokDX5Wm|4S z&fLQspIRHcC?tR*>Y6-j)j%0#{<{{voOC=){s-`$VP36z{xZ=gfd*h95IbQZITg!p zcMtCmS2^R52qM@)ZtKxT^5tQ*)Zp5M9gltNv>KeDYSLAd_j-EuCL>qX8;tD3PFtiFUjeeQ$JAXUg>~f!3z4dd8Z1^w2B1hr=iO z{&7P|X3;HuLDJEnaIEQaaW4klQoXuBxST0WAtj%47C(Oy$^R7coomV0l0Tf_Y(g@_ zX)ydwBa42(9z~_!`{C=OIs|Xhyc_xReW0rVZ1d|UQ0gDY3d*(mHPr@q!Rxn3gFtM% zUxS~3Kfd_AJ0MW>ZzetnWVZiTu^o^Y^*^IW{(rjUk||sy^rVILM&*=AW$Q;C9n`LW z@NeSTvu8J5zISiqqx=IFn;w}ZndP34dk`C$8F=e#kHQ1Xh=W-hFGby%Q`m6HFnRdo zy6_#X?@nTCU_pFO2-l4zn2)KLAJ>bxIbUMusowhBzXQ`N+N1E10nwovC3C+9nUK!@ z-bmov&R^y2-+wUsZxZbCAO7#xcAoxkua{q3_gnv$e+Gg6{%>0B@^5GV$kg&Lpxj^d zRuJgzq`ryCxeY=^yd_jV42jHsnU(c=$?!of(?C(hAXyh%$;H0mnuml&{tE}+&(29q zOuUrk&9w@LhK=-p>P+cW0)Zf1?|+m2-`c*&dTA3#2&Wkq?!S2#i$0=F>qV1;wn#1~#EAr{$nm5>fKK)!c zHXJPSUU(Nkj93FYV)ow()~5l*=boM`D`qx}f~-$$AVcuwlMv6K2-EGMKqpHLzQGc( zV?Ou(NZMQ5x?rf2>pnA$R1m`7vyp<}M94=#EUw&P0G)fF*uww0vJS0HzjlS2zv00? z%JUrei>vKuG7H}ar$_V+^!3%oK==+UCzIon&Weig29N2vH8zZW44vSs&A=nN)Agza z1#opxZ0Y}85Az%d6MPKK6n{p-3%}BA-~is*stQ4F5Ju_a%10o41F)^34?&V3;z4o( zrN7_F09P2O7w_JMzZ$V&BVuXHckXC>pp_}VkM11)pJCCZ&G7=YdD`f;Q6)q6&<1hOB%WdyoEExWDKY15TPc2j8th>cR)ZHlzjMknQ;Hg9~nsVG7-Rj&| zvfJg$8dwd~!-EucdIYT6E&?_4cs!e`k6}gJj@i#WZyQel^B2n@K0_5c<24ehU>~PU zbqv|hni!Il%*?^hiOyq#a2hqkYDB^0fN=?e1eY-vmGW+g^m~q?4^=oIkG>iUpV05! zyb`C|PvTq^hKqX4fR^Lrb4+;<{?u7D9&6X7D)_(=HrkdJ+;!;FA1^^M`xEpMP0eJi)k z1fpPXxgu08g258uGVtsMa^F>2d9B(iwD9GRr1`Or-NBAv&mWQ)hQRCZ=bZ%TJN#XC zKH_p8?Vte?j-LH~ri%azaz{y(EF7b-ZKEZrTpxF>1^o^kxgiAf%G?GJ-s1nd^}yF? zg`>c1r8msVZ@5qNx)wRvBiuw)t-skC#;$VPLYy083HVB3Q> zsrc(qnKQ*fuJIkdGUH^~hQO!E?Fc(_mPJmb0KJ5d2q`a;!6AFIz2SJcTCUjOc%P(1 zCAV>?p|gZ{L&1e!Zr3|y{;tB4lF%eD0tD_~iNO?%`wgUAQ}^8xprzvz&aj!@T;DJuq6 zL!*SE4HW@YN0|S?<4W4NJhWCJCD1^e{jl*VW<@CE@c{Yq(x8{Zbye41SQr}nHEIY6 zMr)(p*3E1goj-7(cXs@AEMpXV-S(}UrWqA~gY@R^RS}_%_Vr}j*u8HB8};{^QNzMq zx;RGEDDziIAjxfQ)tm-l%=jYvYK<;`yDE+|AByViyrL3yxW}^2Q#Mq!ik0O^`?x2_ zL^f9U|Ii`Hda{zA<2n%R!p-;_r>}W*g@3(FcUaMA?(800k=y3*wru)J>%fQHHEf}Z zV@V7}lL%T}lxccII+jpk}mhOM!l zb$Qyd+cDmxOvjq}a<;lmY#$JVv%edi)8%LFeQLws7qR5486`l$_A@2Tvp+tjp!Y(T zK3$%kAZhOt*jL z)q_?vEoT>}J5EkIp=WabK~z@K8&q@s5h1F$vNrp7s4H>mxikbhY@NO`knN*%LhM|S zaH3Wd4-#9EZrApHqWZ@60XcHwGsuBxJ9D=~)Abil)l)`}Vu2kLUw?!?L2WmD1{?k< z%nTXLM&-{Ff;FExsHT66B+~gG$4yWE6Eki4cuf*`vG3yadn?B$E+~B3{i$E~`L=jG zA~sOgDZX7B^H96&17_EgQUfLJ#^$=@`uE~$v+nyWgTkh7Rb?``(kYmSV?@2b|G_V? zgCBrKc4_xn_novv5dJ`MohX5p^%kZNjbPt+Rt<9F+^1fH54pA)|y*CyImfAlz)k z;h7P2P%+>D6e**6ruekg1NgO_h~bi)Y@1OcVwQXK*PeM?oi#L5MvFLr)(`Y!oqLpX z8)WiJyOgvb+X*@afC1s!P*SVqWpD$gWm?2B+u|(cEftju)OUf8Ky;=~Zn-rwEmhwy z*2JXefEksTFqjwx-t6kDO00eM^_t;DAMHceoBisH5s!gwLtw7QR-k}c#AN@KY~z`4 z7vP_n5^^(P6^j@79bfbLd;6?ZcI&Sj+jhcjEg6;>zUl@6(eAH`PNlDY5a>UG8w9xx zy-aRj?G>mxV&^Q2Z_Ajq-jr*a|5mBjXWbtzK+|{zBaBeTl!d!mq(&0`t) zswixi$fJq(JSrK}^-EBXxNiB@M{e|6~9+O1usX2DK!ZTG-^q@pz_VD9dA@d`RLsf3BBA2e^b8R`^Rvrnua z#3Lu_pH7n=#R3tOhwaAPx}n|Y+@&Ocgkp1M@|)IXXS3hB;`Y5_m-^**YMXN#(S6(l zF4U`(@d&0$^L9D}k%bB1)> zNJCAXf_#|o8^2VwlFe(FREt9lneE{l%xwf@MgJ|&=h=yCR8ITkiV8PB+*^b)MMe|cWIj+;)pGi<4l-*oy1H#rcPvAg&nQ}>(@h>{R7<4 zSna37$ElrfDk^!taAZ5~1UsX?HArLy4y%VoK%M*h83Z zUyfJi!ELE$?_ftairh(Xea%oqz{Rd!B@^uhc#2u$#pI(Vg1Emv3*#1b(tpWdta<+U zx%g4Sq_bt-^9#FN7$W7Qe64=ZB3$F4a(Q}Dp$x9LjUL#p@No97`BOnbSZ`T{6JIrt z)93hlT-1~-!vk8+IdJZU4ZKMRz44t_TAmosk-@od4Te6SOo9ic6p=1b;F$3$HI@7N zW{!?>v}2}0Y?Fi0Q@j7WyHnx|-oBHA{7J*!xy8n!r-I>8j`<&H$)B`R`Xc8+H|ti; zZSm+zcc;HEsFx8-)pqq;-)a%F0>?Nd;-u zuUi1^Oo4sm`CBT2-sbK!OZnli;r@8D!xCN5_##2PhV!{MQZn00+5mp?5X{?eLqwVQ z#Y(@owz+8iJ`T{Crz=8eEiP%FL7??o0Q+l$LudX(+XWS;Xe2k zzdx`b5Xi1z{aHz`=hF@j&A+O-uLAAKYX z-Yn1&qe6`@?(4qLZ$$?8@!eK}cCH0bKN~`j_JNnZ1}xx`#@5m)tNn4ZGpPBg{$Amx z-=BD-Pi7$w{`B)y*DYd%A}H1acqqU<`>2D1$rP+{O7H0uyLe=+xoX7kw{N>)Wz{;C z!RojDI6tpXG+Ye=x*%rEysebY^JbQVlUgniZmFY47lUI>W&;*=1;76h_(le7{vE)_ ztc>p-wwc(E&z&?pVTKwJ^+s3!rj68&CvmrE zOM*ojSQpbvf6gV=eUF)9tqyYYj+tyV;r`kQEp%6foa3t0JlLq9^UaFxtk1$Hp@=g* zk}Io1vPY;X-CE*mn3qg6TXMA)eDv=RFkDLM;)bwl_&-fjP1Vod$l&}m_@!%$w4DC) z$#nxI7nvDr8#_;jBxj$7(9&UXf8YW@C-(djop;wVou8&?VSE(Pi|FI-^OD&gUi|Aj zSA?#rg4F%9ihX(C;ulpv7lv+p-7f0nP`W4sCG0wl+iT|XuEB6dU!+bu2i-gd#2!3Z zXSuv_-T!a_(Ay)sfjRj#rh`URUs~#rup6&{=Ns^{_u4c?U^RCY8iwN4V@jgd0>Q2u z81l6rAiWUhDv)1l>lBX+3l4L_b%1PO$(MM?Htz|{RI?g5{2e-oh8;6=Y# zY%AoxbFe>Rl$*ds_tU|8rV#w$C~%9TX>ICD9yD~n85F^(@6rv3tr%XXuPIt0p4=6b zjZ5ncZ<RsPE{K_1&S|-r4a%(8M zmr=o3EeV!11+}mISHW1m^f5mowgNkC{r{|rRdjCnTg^Fnj^h$7`j~jpJeW}w z){;9_q4RGoYz=|YU(ru>2cq&sG!&(&8_J!XeQ&ASq8=oze%N`7LUtnkxME6Q+;`Xg zu`0^FGQ%Vs82R%4{{K3qe+Cyg(%Su~3AY|!9S)!pdsQQBoaId{?^S%JmrVgCgYObs zp}gBdNLL`&wKT8jP18$$q6;Wy2Xu4fP{F}8Q!6*s1%xSNd$jI?QZ7oQ*xb44e;~nS zN%+Ty?hD}w4O4qeiV=0aP7X)lO%&!SH%@)433!0vjfy-5!>hw-iiBEfI!FRV?Fz^W zgwkeH$G@kJvyGw~_WUq1jwv0$M9q!h*`zd~3+u`Stq+9sI*pezMrNk{jcajR|D2Y` zbF~SL(d5L$cW!8JD^Sf!4~J%4ZQ2? z(Xx_-oPfOt^^&8V^N`3VdsB$s``eB-&AmxG;ls#`kXo{KpQC<6&i-{vD_KcP%eDOA z-p4QJt_!fEr)3r{Xh0R5wRU1tXR$h^P3t!c;*)^*`5(ZKM#4%~(y0Qb$=cGwy5TYn zXLb}Ydn#=*ef@8xd|PE;U~mb*E)64gsP7QMmF?b*F7%G*WM;i&y_B)Mel2`#y5HTU zpDp5rc7OdaF`_?kH4%D5Uk-5R-MDtS=Ii(TU<&bxq^0iL$D|(uZ`U&dGb&;ahoVfE z&EEHRiZm`%z&PieroQl^QqmP|BRk`5+SGSGR(-%=$xDxySsS!S>Sz&9{}7k1S~zOK zt&H*5)t{x0o{73b;;nJK?SCMZ53rYqv-kW)%bS5Fy_4%*VhO~E)YaEia8`HhPRE)6 zep4Fz<;F_(;o94dA0vEDQp)n)-Wq{hp_?vl?6zc z?-jq14W%dUqS>F^(R0OAhZ+e5+}rdzlTdm@(wo)@e~onaQ}YX?_*LEGUu}yFPx-{E zd;W${Vy}&2VMKB7HYK*~&At^g08rMs`nnl-Q?6ootcmH)knJ+j^2xfLPg@yE5$GS- z1dFB8GzMf49-!%is@-2dP@^{+MC@roG~6G^0WvjhAYvg*=>i6Cw$pqQ=A)v%31$o| zSOkODAX>OYW?)QA^sPO$VsnZP$}1T?>ppswUU7hMs%xk{N9+ zxgB`K!5*`q&%6oW>G=pnhX)l?vnhulfl#``?EJFCqq8^U_}K zbu3|CTtvZ4osirEPQ;Lx6Bn}zA=CZj3#ukSrmpnW#PnHe__P7Qe7H&ze4m&&L=SwL)3dD?fUhfg66y%ZjWseix3^8FC(6XsGzABJ!{;D7 z%=1GAf3R?us7;-&c3W~=BOSuB!}#|YiEmid1|e1eP=4-h%>bq0VyjoBvP`sMz@YK= z;;^oosWyJ4o?I|=vG4pa$(|DF;}cM#6w`y4A4pJ!tLK(|kCA?ii`3k?j~q7g1BVAjYCHLb~d!cAxvX}JB`$lybrhc$sjYs}W? zpXFM?g0bA{M=PG-=@PHo0S0OZ6H}i3J;*ey-+aK!Gth6BzJY;y2C%*x{+@=$;aOn+ zuqerN(*$qgb?Z?ddX8AsCc!mT-ImNT)NQEfO7V&o0CvYgf$D~9 zzb3b+!kb|d7PX;UheDzw{3qI`*rZ3zLaG#C@Jkg!+Y? z%Bk%uX51T2UH}BQQEkgjq32gOc~^^MJ0Ch2M#`B}v^u>0;$ z1-;B&tdU4g6NL+2bQxS~{%d;0=coKk4gf+-E>2XzgFJqw5V2j?EzhG}vRb8g8;eVA zcAXX?7DnnN5lq`{$JxNlzph&Gi*{U-v+%O)2YXlmEUOro*ao6|};))olc} zx>i2xlLC&O$Em8pT*297D267LO{DYZx!-rZ;@Vh$`B&YB9^9yvrK$R)H%~dVItMbx zxxCS3$4}UxOTP9jF+OL06gRBUpv%<4`!rBaN67Kxm zMzx@R6Wcos&(ex)Ah(MGBK`c89{6pQODW@d;XYN$jqx+m_fBN#A-uCx!~D&PnGtFk z!U0Uxu%X^n{4&Fr_FO8vbNiDpjD9`b=)qL?nKXgg9y98jbo|Soat7k}XljuX0$`nZJWJX$gHJj97TXty=DY=L>7Rf^RHx^P#Ul)1fZp9rJO zcFlJmHO}B@WErlQF&O}o$mKjbYBY|n*u!!!?sZrcGK1EviiWNA6YXTZw9)ZQ>t*L7+8Z&s=iOFLoP3dOBfEXMs)3)`89 zf{;|qN{xGAabZomWC5gmybto){i`lhp>i-pP`nT=JRb?P8+r2oY4XnxDdf;NApGYY zUI%M2B*Kv{Nw#k98LI0O#QoB?vdcR&B`>u~U@npHOQGv|#;>-rh42XyE1nYF`!#+K z-<=Xn~ki^S{?)>!(RXNwu*Pka}zh&FVn>(1h>DU#as%>M@toZXd zt9IULy{3cmdS&Ia^my{QJ8oD-xzCxl+XnZZorzC3s0!mrE~Mp2hhcb{RF=Em#MzS!h>#= zsc-GC-JH<Bne+|Q^x&M7u zfl>)_qudIF)T_zu{+e7TrwWvA%Px1h*juw#BW}PRCRV`hltHS66sN?7z}UdJ+Ul@6 z7^ReddL<3ncs?Oc3%AjrUq~zcww`{*iR~}z%8aTd;VPh6y2EZ+aRC=@K_dgk`oYfr zM*pH6!VhL~!)inFEgZzV+eUIU)FCFQ=+EzvmE!I=;5kGq{P92RbXol@Lb1O^O0q;{v*k! zt=?;WU$qvi=KhhK0&?S0fXAMgzo@uDFO62sRgTxh0y8pS-j^O!HF^?(ufwl?UN0Lrh*dFARdQKnw$Z?%9j0y1k7k^@-N-l94-ofxIJ@{kfeu51I z%G~QWoJ(4W77CeW*qrC zm^dl|yfHc8f2ZN7>(&rnFtj2##aa;a;yKBW zI(p3bAg^7Em%9S--~ih#X1>c{90s|&!~q`~K@xh;@+0Q@T8b|tfSLnwN*VuR=U0`c zF%!bvn+3E*a$RS0r}*Z4RE}iSoXJn=8=ewXnhx$ zVTmn(I*YWL{045~fWlc&(k%suY1uUPK=I%5WP0gXzTZSs?ngS&U{HJYjdwW`Mk$MU zDVA3;TXv*=oSgV~)SZ_u7}b3Fyr=9%8ta|jmf{8X&|uk9u%huOVVE^}vur$Hfo#}v&1f~B%1ZU1?4yCzpO?>=wIVU0WsHNUhfwm=}{PV##WkYGV8xCzF8B&Gf z@%zRQ%LN>hcEYUA-5CMq{{L4PK#!Cc9>&d zNA~D6cvgecy}+h7h6*ZBz5kdoy(=pl7vmJ@?andjKy|cx@%%_bK`}gD=q7{3!uV?p zV<^0z+W;6D|IT2;HCrJ3R-{AYiS#kPJv<=8Fb0@{v9Q(_R34V*xhh7SC?#&|<*$bt z!jiJa!ewXPv{qs@eA~y6R}(k`l;Dm?zkqP-B7!~wzt zs7i7EB_F-Bc~>^M{e|fdkF#P*inX`&Qz7%JWd^wN@i5!bo#^I?)x8q06c^TK1~WRa zMt8wrNjEo)`l^m~O5j@#auYbK*BLZzP%FZz)TfOR($lVY+&IcgZc4u}$dfsZ^8OIB zyzYJ$p8ZjE#muJLf*u#U6@+00?&oI$BQXM;toHdOjzzC|1b#65lphfl{= zIMbRQ!EHQ<;~Nz};f;a1dBlC4=&SZN5l7Gk*lNcR`>uI;CnhkH|1<|i8;`=U z0^chPA;7jDfBILUeu$-?SoFe#RHqvQ{Qrdzr(PZNVfGPJgA1_kg4r%hS;89V^wOH| zL(Y8}R5vxaz}xb#T;`&^ApJo0yoWT?wE{i$v{BGb9P%hD*I@XUhSySJ!s}ak4wYpp)rT&ImYwxn&2}2`b6R^-RMZ*W4YZaP<_s5v z1jRL`RvjtuE;!E_LDozoBYgFKUAHIMUP*vc87UjCigkICbOEQ2{+$ewun_4 z*F75^cN6b=Ev`LPIK#ty;>cZ>{gz6UpFoOD4gyBdpLYPr=M5O@R~Up}2b|_+)AeR^ zu3jjBeznqA<^>u6VjG$?k{ZkBnFg@3pCS`dVCyj^8o=;*%FHg-_r>}FF9atjv(5oE zR50)Vtp2M)9#dpBF$D(3Qz+~IS@`q5XVsu@ za{=;PfJc${P_?t~QAc(72cVt}cSEE-&|O*SB7y3eMKt&2$AVolO)3$!E|}8T!$Dcm zZgpDRR_+vA|Dn=N3c{fVQz783Q@u1_+7PBbv%X(^f`4^Jf5C`e7R4els0$NK@Kbqt z*x|C1+<_fl1dB(Rf4Bga_F);H;Qgn!RB@z%$9?M3r2XWuDa;4$3`P62K%y0FNx+W& z+jSOS^<_-T6$mGAc7YjBSHf1ws&wj;}Q8VVwA;B+&0&nP%rY6 zb=q8FavBJqw+&rDnQqc>7xPuk#gfsbPE=wUbI4P*;F0+{vZYHIiD(Z)6nUeqbeW||x>+-6y$Bma7I~(2~k0%a1J42(iIoXzy zFR;#wlJA~V$kL=CoyF%H`I)pj!>-!44{H@4)}~ zx$!^T{S~ql={*MfC6zpI<73M&I;HKsyzmsJ!1X- zso?Z5YE0K>dJzW>U`W+Gb2md{s{svB$fN8YAWM7PP;kgh*)V+se*~j8wY=|z4Q==} zI>aJDUt7x!%~dOA?mvSX%57+Ck;}f0Ps}awVipt2~xI(xIkn23EywJxw(N^x2bFyg8RM#|35=3n3|$rvaONEIo~` z2cY5#CFlsBSob1G*u~P_8~TILhFL6*XY;>s42+CdiGI4_Hs+C}SRAltNDXN9yI} zp1t!NT+T6oBbADaZ?^o^C~j)w*p_yY9c|S3i3Tzz~n$LmRVe_yqg6 zkk{ugoBP;R6~b$aA%qL&B|LLr;$hN+lW!tVl_u@es|||Ug#m{iqGflcz?8GCL=P$+ z^0PbNn}==u)R>MtxdsxIXUVc5-KN2+iuT`>qakSGE~++Nxov?r6u^FAx3Lob;YRZf zwZ`n(riP0G`%SAbH?Eax4YXr>T3ojti|A9f2t`i&Y&ZzNwo6zu5Z`c-bb*2C9;|#6 zx?%>jToDa^fUNBjW5}&JeF_7tG`<^j&t3t*iqmCO^xo94kK2-)GZBwpyj691QF#i$ zK!cjNs;F$Dky-X~{bx>)y)-i7wefeuRqVCbrK9DiZ_yul0;8yoJOAa(a=B*xG}mB1 zH*v)_$mDpAz&Y0OTG{@x@}s;0LEe$8;)1-8AM!I90TV}J~ z5?;e+p{2+JTOQwet4IInsB15#Vff_tnt>o3^+ctZ2e^3<7#Xndq}dW9CvXNyL>UJV zMat6}U;2a8SHj@KfvDbijn^PI!s77`OK*Iv-4A(F-JVFAzaD34&AGE4m7MUSu}evR zN_DYUfrzmhQhNIEYZ*}Rrrb5p#ST}mmgfn3Xo{J{G)TG@!&206AlS&&RHI&5?#xj$ z;(Q6(p%kk~&48k^nqR%UaQ-RjM9{g)yk`QS{QsNx9hP)CwnEE-*6jqzEI0#JD4q>v z+lKwUI>9-*wVvk}q4;>(^-Qe#x5X2OPIlba3MVB;lNsN3pOeliKf5c$w5jyjNWQub zb5T62q9=)hdMevXl;3sIl(208c$gyN+(C2E2jxt#;^g~EUH>8^PNVmUd)D6T-8F+i zhlaC)&F^?77I$CA%oM2N(D*kkueIarS}V)1(fPeIlGM7bk#sw6YtNaS-jyl0=(Fo( z3XIZ$^)GY=p;xbAt1K7tRB|z67*jvSxPhwP63JnVwOpZKAK!o8`5twu1M0LvPEh7> zO3^u1+c|_VW>U$Vu5$()Zb$l>$jKk>!=84Km`}!ahJO{we*oYBhMu)(*oEi-)Leg; zYs}o5RdQ*VPK4+4WXy&YGhL@*ldpv&f)GbwNqUt(W+4$l$seJzi;*0oWoW!$Vp!Vk z^qM3;zkXHfQLSlAcK9b`%rGlm3)Q=GH5)9sDM&YEEWBc!ew0`x$Q@agk%2G?(2vi@ z&V)5d96MZw{RgFDKlH0VRcBjj@hUdJ(Yg?h6$=`v+I>vJvjDL=E@5EEGx%k8_L&A= z!kW0cnZNJ%Aah2KX_5D=Ey2wy027Xw@=(W9=E5XX+F7bo${Q#tCOB`}I|2Jh5mtuQ zo!o`~p#u;jj(n9__*&CJ-LN!G`U0gfTCzsLhhnBQZ;A;o-$#0|WYJ6N3*}i8_V9;) zhRK1#mWzVs_gap`=gY0H8SW;lkefqeK(UNHW+@)t3n$K-aW|}bdz{%<33<_|7me~5$*2WMM;&J28OnMV2Vtk+{SD>@sCad8qW5POQ%%|s#c31&M@9yL zYU4o0o8u$o-q7T##QIyl@H?}6O*G}Udd5hXsfWje@$~oSV-VJ0rF+<^UUH%wF1ui3 zZi0OhaWRLgT^ABfZpv_DRb0ZLTwr6|NT6KOy;T~RhR5h2pTvmuz8_*vW-*=7VK4uu zrVio)BqAU0=~{}`BvK)sYE73ma*M^&{TkU92@84gKhl^`8>xg?zV9a~8EmR9`WQb# zlJZ5N_5B_)zXtIAQ)^yIsmT(Wx(D^Va!FV%5LlF_B}#Wn&WMg6~GE zxQOIB4kqGHzbdfvK5GDC+Xbfaj?#^8@2(YZJ4}J=*xL<$$X(6OPv4xu6EhZ_a0>V@ z(2tkKHHcp`ZUe4`pNMT zO?K+lx{Ldi!wawh;QYx+0GpbX)I%gOn|$ZF0DCkDSt_DhvJUmmgF|YgvYRHmM*Z=v z+e+`{%(o!_9nSmHUi51__w0Ce@NJC5M)ygNM13DCnY%_4QF6wSGQYsBPa^?z^dN@e z*nHUBXyqZ%;z^eK$+e}=gAAnH2#2T5l(*F({|7T?Sx@(DeN`qWjEA~i8YP5ntX&;) z5h&}Qwd7(`Bqcc+7q&w614g#3Ha1fZ+sz)qpPWL87H+>}0dwm0QH#dMjib(acpFIL z&iv`b_$4_ev=;`}JY(!rqa|SMBf4jCKEkHRoY4&C@AFQOl;~Ef$PpGB{l}y&2Et`x z`+znwqL5yrV(jDdHz>wAfNW$)XvPSt zLWuoKA&P)Z{QM%H25ipyf0j!>(`cy6xw7hf1wF5?@`6B^F5%$NdGfkEJDrNw0kp#Y0scCDZDh8K~BZGhWa@dldX#^nT_~{rx%k*uKm3~<}NNc-Pq!H z$@F=Lnp?n71?Z?DG6^V)4|2JDwO6|e+^B9W7SNC7g$UatVudDEbNK;W2S24O1j=#X z9;b=zev2w`ILHZrI;Qt_duZY{nd(F%E6+NOSEB2bs>ndUp*G`^oE|Kn5H@Q<4cF_{ z4XImh6m$nSg8OHkom|md-utgx0Y~-O=>_+)4ui{InU+aD3D1d(LR>`iS6%wg=2(lG zeBcHq&?AN7&+2jTz?jw&e?7mD7l***qn$T^+h!y-T!gk`E8z`ahTaVsicgq?%(HO( zg)TW18Hia5Q5x**oUk|)5oGLiC$p2%K>wR!Dt^Bxrw&BRY0lkdG~AbxuG(Ndi?8MEEk!(mB+~0j`3S> z1Qa*mnzCmS?E;<{@7I0Ivz5;S-BdLuN-l6IJu zlTQkV?mv(ZXWmaE)0xWP~7JJ zsO!R}bV2ina*^{`lw&UNXc^9og=LzzrNEpqf2DPUp=O`|_srvuvX3&2ze$b$qj%*7 z1+2~YIQQ;3@@U1kvuR$~_Yg~^1e3to()-ejXWWD*n}9s@|Ujs&ZvY?jB;WSdA>z^z}gS@c|B(chG7Q05v=N_jsA zJfd$dE5Q-zzN;4N%fBGnQPRewd#}1H=t-nn*dZJ?(Ur09yBR$YfKVi&c9P~Us@z`4 zeiMgSsu^!guU7u6THJ*Wj3{xYsHgmY)qQ9_h-VdLhzyLL*u#C!Qk#ZOcOae_W}^n++|$`Lz{ zM}(GMY2Kb6(#c?P_f~sqKv^d9uB=F8&IkE6Zz8Xo&YS^QFH!Q~rDYNFsd16|Xa!&~=gYsIypp{;*Yy86~w8kq5Jc z-C(@CDm!+${}Cr&D;Y>l*vEfc@g{Ce{nyxAXy)M0i|$-n`j8jH>*vB`?FhBs$7F9$ z)KO6ijT}b(wx2wn-@;0`(jZV1>POAPIy;0S#z(y(rh~8n${}i_*p0WQogEIjJJ;&r z`S9j`U8RePLC7>j?dD0!hrYxuwcTzOo>aR~xByh0YW$MZs3~ZuQMd)Ie~ar$Sdmk4 zfTo4ld;wxu2ptLT;EGAsD=IYUVS)W?N{_lBxQQLQ8*4Gxc9&hH5wBu4FkOBN1vLhO zN87z;e7^k42UfW75!2yexqKultqA_fi@4Y8i;tz7+ZS{*n>bdv>Iw(q z#4a;y)YM|k_#nd4^QNEKW|=4?B2EdZ51>9aeJTRD55sqQ9beg*VNVS&LxI#5^`*H_ zem`++ivA~FNBWp;WOx%oSC2lCuP9CF425?V>s9NT`3Bl~LtDQ1Tm%(+Fv5$Rjin!3 zUJ@Dk>0F4d1ku2XY7R2RE;>(Z91>7_z4iOnJ!VjHgqND^1@3~Xf=V-Txk7Hm1#SC~ z(rtfL6??qC+d_q(DZ~?jgeM!;^kQ7O)7vpH%yzoOt1UrvDV8g)LOF9JQ~CsrcgcKa z)-g26Mx@@U?J2zX7JVlIms@=Zvujypl%Zz>C^e@jj{Iu1?S-#5psLrwo2^MDNzbV@ z(?mP8?-O03t?fgFL^i(~7`FfIjqEIiQC;i8tedwuM*fQdQxF!9T)BNWiiNp%GjE1oOQYgr(1zs_F@6 zQ?aC3m-!~BsotQM^6idBAW<4< zext-lb*u^LrH}kiTUo_qbpz}s_Va`G=ku6_OISOnA1I+!yBpa<8@ z(tf!QIhu~>zNQ>KYZUi9PU;fVA#;hHoA3*~X!54oooZ04QFu{N{6O>!WOLMN?iLv$ z-yPSnx=DFU6#{yNyHRM*HSgDz!BV0K&DNhL2J79RZ=U`V(*!qtj19a`4sSg85ah=w zTSy-cJ@l@|R1*cPnOuE5C*Yt{Pc2+xNI}&}&pY+i>;(Z6{$;4Y22*aC-xt6cuu+w( zdcz3q9eet-I_o556(&&9+pM9(9f({vLHBzH>{{}< zBy{DrMH95&QZL+P+VmWC0j>0BHS=_2eDg8pgAHexEPHkWlTXXL?JR7nP$Y-%=59Je zEpE0w8cQGvk6DxoN3yQ>xrI`Fl)X>$F}dZH@EYP5bG15j8e*3*)eP5Rf<$*nE8x|! ziZStKBaP54^+6bdslx1rSE2qu2W)$OL&x2A#0 zjFae3;g+=Fkiq^bYR$n@X&7Yf8DxLQj|3FhjFfD-dSqdBOLg!w7j}969w>;b|SvaRdC3Al$vkZJv0YspVH$M#_!BB|!ee(xMuRoqYO{WU;=rSO8%#la6)zYE;@|4M zam4on-T1GcL0w$KHnl$rNP9qzr7k_i{U@*Z!r{L5nN2EsmEnjHio@Oin*V9@8d)x4 zyh`*r=X|2sD|`ZzU1Dpq0!_<@u8w_0ynaQUAy$MdN_mC!L(!2vuEzfwaFv6P`&Ey4 z-n%Ayx1P4?{24t|fs#Hx=V_2U7Fj=H5P!ls8m^W*0Td`vzS(Z?O5FbT&J(Nb#H1C) zM+&}rX%vB~c%muNh*>B1*y2{Hhv!&vtm6~p-S^9)FID*I9p)G*Cx%`ft~OT@%oTNT zFGkyhX%+sKL>%iHeU=?z``JCd;~u-I*~SvrDSHJX2R^1Xep()8t0q-h@db%{}hAZ9nb+{kUUZm>DR1UT`H+1L{Y_(KzxX5b z#r~Wp#o=jUf+L%{6B&Z^c9~mJe1H_b^Rh5K-~pO2Y_53t()TF!kl-dIn>D?Yi)8XC z#a_ot+}ic69(r#0kmBF=hRi%zoUXAc);zg5*FVWx-2pyQxzaJcyz>GYizqXz)u^d1 zM#ts(I16|4+*lgSn#qSu4~4>cb`~G**l`u&&$`7Xk97f_=}2@j<-B%9Bf3=RnaS2* zKBH1PQ(Wc6c5K`GLQ>M-MdeM{H!@oKog!0Pg7skHg9v!Mwzy;cYy#aAFC_kgqE9*8 z>8O_X7ZM^`9_An7lCw!??H)$oA!Q~Fm>;xFkWvFJ z2ND97o+9!dclID1DqWkv?x#z57z3GP@MxQ6$hRM%l0Eq&e6Fu}W#zU@wZ+|k@!}PK z9dx^$r<*eCc+;fx6!AUPsl0MspT^c?g+Cm7E80AbVvf~+n)f@9BuRO!&?u%fSzfMj z_nca1YVU1C(6~0a&nMG&hcsdpUfxg6zg&{!@Fc27{xC1?z`-scdSVM`58pGEWo|gO-kQ zu3GD<`RzY#4muXs&6GH>`P(@so5j)T$%)raru?01NKo{Ln)K?7GNL7&CuUP*nW`GR z;85P?h~v0ji#s#~TW)eF^VVr~K9N`36Q^$#^TKf}e7qvF`swS8IVIMEsXx~B&A8i_ zrFt2=%Nn(b2W^0#uunErniN$&ZM(JrhIHsImCpb{5wp{UKxb$Wa=CjQhGPoK94oke6gJ_M!e{Mq-__ht>s;&YpFf) znl{WNCYJ3@>b_Ek0*L`{o3r}9R<$Qi~PZN^6zYGYg;+{GEyi2^e=Qceglgk}qPetIdcwt`tLKHAKF2?c7rO zCxqS$R@~9sLmWP+kW7)R+q-mA9$tXZ?fx4yDH#^!g6ev)LIyX=?k1udWUw_2(3LlM znGIu7>CP83ujh6dg9F=NY8LrZVk3p33s0)Q&F!hwg`WIBU38sa^O^a*v!qNTz7ivTY+)erC(jX9h>cRgwcrGeMfz#P9>ZJg-O7F}}LR4~c z)E?KPM^STM7i4nG&CPu{`vW{j8eSAVzAe2M*0t9l&^&1q4eIJT@y&cu<_MJZ`McFx z&~9=QCR3s&DID*=Kla9a5{*TO5c6DVMxU2#?HcDq*P5^hbm(vWaT2<1N&~O7wKU}I z@A>q}Am=zFxAE+*_9e#!l2uXClNGj6VRm}B(+%_rN>XiKK%}l*xdIR=h(h8M?hRYQ zXvr@oG3Fe@>ka?HFfZNf`7~vkKEr+MSWRsz@~R6Ti;!H20HreYa|wIZo~EUgD)J-) zuO4@VP3?9J(6`QirPJGHl%*{a#?0N08!zHJ!Oy6Sx3xd>9EKP;GL zlk}QuBQWCzs|t#v`@UX%37|0WvH!jTJ}yr@!+H?-^OLkdoN5sGkKybP@xPwF1v`@J z4#tjz)ozY^$P}IW#_1JTjU7E)hvCwfC*I}mpyMqs=jI*_$Dy#zDdMrQh=)!V`sL!+ zn^AW0b9cJr(L2lePw3u`iZTAB$0q~2g-6l`Iz_Inn}e%mU=ijHNg&>>YrGcD!bEt77DHW2s4~Z8IJX@`M^vG58_94$k?#nz@ePu`wxl~oR-p&&^fECyM-`mg}kqg$CsunLXC zp>zI${Pjx#!Qe>#TpyG?rJf>}Fox7|S|RmA_idy(XEUdq9qT@1S%}B!T36>1^nygW z=Y8}J1M;1huQ1#6oSC)Npqs^wHiJ@|7$hq&!HC05s8mANkUpagJSZ0(5P+MR;7SH{#4}i4_R49e7y26Q0yvZ>yRBn#jDEKr;|A0WhAx3b&YRQNTfURsmfM|6Fp5? zbUrXznUMD~8o){Uyw5v&P)#$TdY=h$lMNMT_DV;gSt($FChu7A~Lh>NIc)E1kDcb7zxuIml z7viC_GRZpI9gQg!x8rbjgJ18-R^^J3yboe^B?gcItUYDWoiJfbaE7Y<(HZ)9_zJL^ z4qUgL?&Gckl7n7E8YcV7!LFt&PBMih3{aA+%~vbY`l~T8FXeF&<;AOBlkIa89O6WG zTtr@RKT7^r?-dQDnp@P0`!#l9O3Tr|$tNrGVh{f=YoqFP`WVT3H!v{cTtIWw{^=P| zb}|0w1+OsiU_VE@lO&Yl0`fYK^diUolnn;8mP`@DPo)@4IqOpAQ)d}s?8OcVV9m9N z8IZu%z@*69&u?CCc+77;l!n2n`{!rOWtbw_HEu?rJgJ5a-`He>6!0UqXdoX7RRsMH zd{2wwt{aEb2UzO_8>3G==N9wZ4Uz_qEf?qf63ZrP0&iYeG5IeL*ALVe;lryw&O7?S zGvTmS;-xYsHV=rg#vG1bcCO-P(R{F1C zPF4tL^DhCt5xaRp(Lo}vYNYW1@Utdj zC9_d!;RcNy>t$=J2igwvF zkLN4#>>kAhtCpcfNAu2bRV>4w@lgIidTK*YsFN4`q-IhYV#@*Xn;nr({<|*K2rD@6 zda~-^71y#)6he5!lbtj~+=>o$rf7=WXR@Le&DW@Na_C8VJ|H1<2-}U71*i^TS%*%q z&Lq0?2ngL|Cl*(I3q`F19OOE-ejoG>FSbf}rOE0ACJmA8-{$48{}&pyk~3}1ii-#+ zWV5&`U903fM;Vzj976&%7SUcHXpkIqG!BP zj7->wx!W;Tg-YV+Zsd+HkU^m@PMN=C9A&58Txhsc8<*QgoQRA*p%jml4(^2g>vAI8 z-iCxc$vXe%Rnf<}_u!PVgjn<%OPf8&FLbM&EW5hLg-tE}Bp%9XhcEnu(eX#7)ARMR zG-72e`|%W9oAY9Hb7^HzU5-Lx-mO>WBATs^jllr!5E-X9MRlq=9=-oHqrQu|!FJMH zF{a$ZexN3-^{v&d-Op)}70c%$#zcRqF-0y$pY!(ec1(gwqr&VSzu&76x z(P?5v+WUEW5RTZCKL*6KCRrgrSReq5!Xt}>x)VqCv#O7$pL!mrb?c_GAeI9+{qiEk zDpJmPcS!h=`nb>}ouUDwt;Dl=hVax`)@85QlP7N9@6u?!urcT|$@U7YVUMvX!u4=LtA9&Gb`T>My0 ze@xw>rqGBki^$l{sct;j6T2MP1r}w&E*91_RV9X+owVUn>QvDh(y5+T>DXb~8ocIp z@OWV_6juhP6{>RiUH&e%Kn*ySs?F#uuxVq7V5@(#@A6%> z5R4Af0w{z#%IPO%&x%`}^ZrBd=RUlRS-;!Y#UQj{eL^40KlN`x=q_D*6%r+&Lazqi zUnU>k(%~QeEBSIf(D>+?@fjDE-DsNn#5seujqBR}+md^^MgZYu-J@Do=9+M;C2;Ln zJ@Iv}o}?o1PaOW4nWT&7QxxJGQ1|3T zcJisK)FGm>U^u0_;~+M22(}s)Uk^1GH1?)y7Lq|vFB{)^*R4x4o4$fcfsN%M|4ePR ztaRCU(EZdZluaoRRoVzY5zu40@DBbO&ibWJ8xMDE`qwFG3K0WS)q#-%lVSNz#zNRa zJ&cf;uO9|glI9(V&hEU1{rT4rT+lK>0uD4Le%&s1juJ-JkuSBE%dBf`|Jh9;YyO7t z=E|B_Lva}%nSMT7fIOe7Y_l(?6Y$;_B#EA6+hWbA{cMyLtO)QUyl>z#RV7R>Q`qBC9&!!(PH}{Ucoc=z-01#%?t@9cB1URy@ zIcJH4ih#-&iLZaB?l`2-c>d_FOUIGCHP0db>|-16=v)2uh7#Y5(E6GMcA`^+szVz~ zS*a|_Yo_gK0ssd0NW8{J`C1{SmEM_;_8qVhBEZ-az%3MJR*84q*IiW|O_-|Uk!jGZ z`X_aW_X$$0%?Nf(J#4bbPH!MAJvmNPBeb*GllrM9Q5(-`u_;KcZSIS%DCRflMxPjK zXzldxThBISKx}E78`)A>zWAV`^g!p)8y^i*LneIQVQbG0To8R%UTLvV6=3Y$ciz(V z^5u1c<=blS%YAl)y8vqA#luN78&6{I$uvah$IMn(!p7hNSpeuL>RpteC_UMAvZt8; zu7Ao00sXpG`LF4ki6T2&8iocqa~;5BcOdxNcG4Wc%Ze6^X|F_(tL~t z*HtMl#D7DwHp|N1TN!Hrr|$as5}ZO#XzAS2$Lw&W-|h(N(#I6cZHBI~48?i#>}KN! z_W~Pl2P7D}&bi3TxR{ek-!IDIkksXDrn=fj?JGO|)ooaDS$QTpXo1C{zyPxG+hYev zcvo3*qor}$dqmU4u#}w(bhNHNeCo5~nwiND!j5pdnOw5yH*-o7G|8ca9V0n)r6-oH zt_ws}wl$~3n2pVkX6r0HPmq56i$LEpb%_0Ibq8vqzxK5Mf6@$|>`F-)@p)(a?M+}9 z^#__@ucw`4^^?1AI0RDRi6dkt_B9IU9McR=#Scq;dgs0*o&49=2DuB*Q0jYTd&IM? z#iAPukk(J2YJbiq7(}aRc1r9kY0rJaPfR5`ps27FB|xyj<}tnQTexY$1U7o$?4-w( z@2^$1orW^?F4ake$q|eBxqr;CqF_C)VO3g)6RJ%S5&-l%62EocUSHtKX^006eA8|e zPA&*Ko;4bqrx_;RzGsMx2}%MId{aJaa2nXlqO zC5VZ#VAOBB&7Q$NmIxGt z+e*ByHbNDm<(`)S;>FiP?%R!jV(MAj9l4(R^;$bBT{bkEu!Xv1aE4f~Er`gSY@2&J z`6Rg|+4@py>i3%txw_FlQZVF^#7;T=4M=A#nFIKTv_$ZAi9pE^%+sr+Elc(f{)PwL zO;i9=y-_F5Zt_WHXT%~`2XI9U6%v(zxW@#gB}we7`aeGhTMWDwhOp{?935N2alqez z?!zKK1RRwtF{S=@90u|~^qZeEe~2tEoZ(-HO`1!2d3pQi&j+m?_4N&mROQ(NXXD~@ zYDEj0!g|G=N5ZdPzfOD=;8Zg^yElk;LGZFA3}=J9Y=(H@<1nhO8|ZVbi9v{)623+G ziUBv{r~c`uW5;UIJ&$+SeeL9%lec69rJVd6)eJ!R1%xAQ#S(Z5zGjcYEK^lLPaXmt zRX}T(?~5Gd)JFSw-}<9=x62giM;Cq#Ut-@dL3iqtoT*n+#E{`3KkK;o>FbCE=9tCU z(&yJyfn4&!Ve6Ge#EN+ks8|%LF`Z=PISR8QBc~Jnbai!~D9i$UsF9vBDcXNPU1rU1 zi;auBCC-gBoT4-~MqN<|xV?1h*^_Zmusgucv(LW}y($a|PafVr)&wXu+xNh305s+n zaNS{-0Vj$p>9+njTlDC{iAdv76|j5LMr7f3=CZH|YhD-FK7bPP2t&TFP)BTi5gxLb-CR}3mw#Q?u-G>^zxdsouP-QW>?NhZ(}#j z{gO59&i_mH-VtF;&R_e2lqbHN5`%)I1^z4k`A=Gr##|qOwtNax_^;&N3@YU@uLi>7 z8EV9*0@DG7_`;>Wo+h- z`=wWR=8HnyK4mKTyU6NeI%8gXtoE)eo0Mb?0QS5%5c!TMp*YaVIcbfC_3~0(1y8ty z;Z_`A09mQJ6d6Zy?R|<#8k?uOthRXyKZ83LenTYAoqw4noI9>W7gOI3a}4sIo&b6R z%FPMduP_HHg@L}q!srM$GPCYTcpgT!AN`gH=u^P$5G(7QBA}anZOQp+W9o|5x(WP! ze2d`LcxB0nj#yuRlJ44z;?ul*vE^B%ZzU`_ByDubq6>d!Vu_X7>-#02;XQ!dKa8YOkr5IgD8bC4X97dK%>HC|<@GB0F!~wMM4TM*86yP;=qh8)W#xB}tDRwcY_;OzN!PQ0y->G0lXF<^L_t z0m{Fzlk`7}O=3GRF8Z1i0&ahBuvrY`r~QkhyWd)}&K=puS|@+*`O0c7<^X8^RdPrS z)KQ)W&@KblL={6MV;HMbt{^q+REsldKG%d_MpZ~wz3{1M1J3Y(1~pdm53W-MSBC=F z2SwhN;sHbPOq7k9@dmc-$!x^!=UMbHI8SV|%j)>x0QWQFYcWPWVPgY2qq(}?=iHHq zFN4x&KS5e-7B4?wS^a-W3j%@HcD6n>lhSLkoqWC*tI*;-vk)6iQ6_ucJOg9cN--x8 z$3G)h!*fil;CKWJ!^`^1`3Z^mCa$YgBfH>IWeJZIiJ-C?MFH=MpmkMv26kJKIWlAgX<_aTAb zf!RV(AzP0%RD+Xu98r`43S}z)EBaIO^C2%e=i)htJ*X;Q*d@R#jRA6J=p@mxG#qvi zEKjyT3e)60#g$5YXK)kZAH%iAg_mIn`J(3pMlR@+9E1#Vd*}BTPzNo7`nHo^d%gs~ z?YU{oSfp`~WUgAG47s_q?8p2HQgLStGK>0TM~?WRl_b=rPQrTcqzP-#o^Q<~YJZ<> z>o$^JVz!rB+wu=RaC=lVC;9{G#vldak1F=oV8rfC;JXN0@_D-S70lRNJ~h2rr+2uv3D#@(5IEse#2m#4cof?|BROGX>fAUe zYh=U@Pxilw-)e7XXV)BMo4X>nX)ylm*|SC=(}hr4VF_yvgdw7-p1YqDxZ_Bu69n>f zaZ~=5niS7jXSs5pg7n)rTZhMHj9Q;%&NnNF#xWmV59+rajH`p+c)w?*7Vk)P`&``Q zVYRAEMJ#5mSgQ=JaCbz4I5gI9J<9Wt*aOeMy%i}r_FQc8aKDEXr7Mshj){C^pBwk{ zY=f#r|EF(!l6MjVdbHCE%ACYZVIRDg$M@$>sI>%Y{d4moO6;5OyiB1T*D0@eEoSWG z75>Wjc6cCXz#t5#h$Y=b=iA_HaM$M7i)*;k#mb2fYN7{RR=*HQ5>uoT4q5`yvDpS) z^jTXo&R{bi8$+LO)LP79@4V_`tehKqqpSzxBiwoJJx!f7nMO+D|iY zZdyaK-T!f;sxBzn8ohrgV3?&G?bIo!%U9J89$RO43}0t7HJ*IssH_NvyMG+5KA56M zPo+T*$@*p;sb**6(Jmbwk>JY@Pe>Z~iLiyH%xGv=*T5LIa6T@*C|hY()oLG*MgP{a z3=&`dPX;hxwQ}OEwRU#;y0`DF7%<8VmYYjYy9C;rX$;QGAAWV++Fx|kE^jY-tB=H_ z<04;v=ZsiZ5IXSfKj}pi&3Wz9>!WvG;~~m#cDP7hWAYPl1;=vtj*XMmq$6`85xs!= z(NsAp$#62|vk}f5rs@6bDYC~j+iuLl8y}V$?b6!CBYSnmn=8!>ys7>QS?~H!nyh8B zn)D|An&PfcA-6zpyQqfaf-A}LY5J{OGxN1w@-I`#q!ln6v3=^`qkBdpHJi?n;G}M+ zZ~on_KW`nIcVMW7L--i!@ zcy9i&G2*{pAaMJ)FXkMb7vJUm{3GY^|LYLfY6+~^_9`L&mp|_Wb_^9YsRnmh0cwNN h_W!$|{RCu&;VOxG2&$`IcVWXh= - - - - The LiveKit icon, the name of the repository and some sample code in the background. - - + + + + + The LiveKit icon, the name of the repository and some sample code in the background. + + + # LiveKit: Real-time video, audio and data for developers @@ -297,9 +299,10 @@ LiveKit server is licensed under Apache License v2.0.
- - - + + + +
LiveKit Ecosystem
Client SDKsComponents · JavaScript · iOS/macOS · Android · Flutter · React Native · Rust · Python · Unity (web) · Unity (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin · PHP (community) · Python (community)
ServicesLivekit server · Egress · Ingress
Real-time SDKsReact Components · JavaScript · iOS/macOS · Android · Flutter · React Native · Rust · Python · Unity (web) · Unity (beta)
Server APIsNode.js · Golang · Ruby · Java/Kotlin · Python · Rust · PHP (community)
Agents FrameworksPython · Playground
ServicesLivekit server · Egress · Ingress · SIP
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI
From 8c932da678a97d86c98aa50db936689d3fccec26 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 19 Jan 2024 17:06:09 +0530 Subject: [PATCH 082/114] Add ControllerNodeId and SelectionReason to StartSession. (#2396) * Add ControllerNodeId and SelectionReason to StartSession. Media node has that information and can log it in context. * Update deps * clean up and mage generate * clean up and fix test * clean up * clean up --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/rtcservice.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 501094499..36dfb175b 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/urfave/cli/v2 v2.27.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 + golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 48fed3d29..43692a9cd 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 064fc3101..baef4aa30 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -266,7 +266,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { done := make(chan struct{}) // function exits when websocket terminates, it'll close the event reading off of request sink and response source as well defer func() { - pLogger.Infow("finishing WS connection", + pLogger.Debugw("finishing WS connection", "connID", cr.ConnectionID, "closedByClient", closedByClient.Load(), ) @@ -306,7 +306,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { if signalStats != nil { signalStats.AddBytes(uint64(count), true) } - pLogger.Infow("new client WS connected", + pLogger.Debugw("new client WS connected", "connID", cr.ConnectionID, "reconnect", pi.Reconnect, "reconnectReason", pi.ReconnectReason, From cb42c6152c37bf78cbe7f2c3d419b51527eeae0a Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 21 Jan 2024 06:16:40 -0800 Subject: [PATCH 083/114] add psrpc redis keepalive (#2398) * add psrpc redis keepalive * deps --- go.mod | 2 +- go.sum | 2 + pkg/routing/interfaces.go | 20 +-- pkg/routing/messagechannel_test.go | 6 +- pkg/routing/redis.go | 211 ----------------------------- pkg/routing/redisrouter.go | 88 ++++-------- pkg/service/wire.go | 4 + pkg/service/wire_gen.go | 16 ++- 8 files changed, 51 insertions(+), 298 deletions(-) delete mode 100644 pkg/routing/redis.go diff --git a/go.mod b/go.mod index 36dfb175b..e111fbb21 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8 + github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 43692a9cd..cb4e8d092 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,8 @@ github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrw github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8 h1:E9s9KFCuKgYWYgaKz0ZmC7K3cPr8Iij77HbnwhQ4JZw= github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= +github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485 h1:X75uVI0+YA7QN28NaVniP4IjhbcDWlktZ3Ec+PHjoHA= +github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 0b4f31c41..d13d734a0 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -62,21 +63,6 @@ type ParticipantInit struct { SubscriberAllowPause *bool } -type NewParticipantCallback func( - ctx context.Context, - roomName livekit.RoomName, - pi ParticipantInit, - requestSource MessageSource, - responseSink MessageSink, -) error - -type RTCMessageCallback func( - ctx context.Context, - roomName livekit.RoomName, - identity livekit.ParticipantIdentity, - msg *livekit.RTCNodeMessage, -) - // Router allows multiple nodes to coordinate the participant session // //counterfeiter:generate . Router @@ -113,11 +99,11 @@ type MessageRouter interface { StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) } -func CreateRouter(rc redis.UniversalClient, node LocalNode, signalClient SignalClient) Router { +func CreateRouter(rc redis.UniversalClient, node LocalNode, signalClient SignalClient, kps rpc.KeepalivePubSub) Router { lr := NewLocalRouter(node, signalClient) if rc != nil { - return NewRedisRouter(lr, rc) + return NewRedisRouter(lr, rc, kps) } // local routing and store diff --git a/pkg/routing/messagechannel_test.go b/pkg/routing/messagechannel_test.go index 5d78c2104..bac68ef1e 100644 --- a/pkg/routing/messagechannel_test.go +++ b/pkg/routing/messagechannel_test.go @@ -39,12 +39,12 @@ func TestMessageChannel_WriteMessageClosed(t *testing.T) { go func() { defer wg.Done() for i := 0; i < 100; i++ { - _ = m.WriteMessage(&livekit.RTCNodeMessage{}) + _ = m.WriteMessage(&livekit.SignalRequest{}) } }() - _ = m.WriteMessage(&livekit.RTCNodeMessage{}) + _ = m.WriteMessage(&livekit.SignalRequest{}) m.Close() - _ = m.WriteMessage(&livekit.RTCNodeMessage{}) + _ = m.WriteMessage(&livekit.SignalRequest{}) wg.Wait() } diff --git a/pkg/routing/redis.go b/pkg/routing/redis.go deleted file mode 100644 index 5d81ce088..000000000 --- a/pkg/routing/redis.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "context" - - "github.com/redis/go-redis/v9" - "go.uber.org/atomic" - "google.golang.org/protobuf/proto" - - "github.com/livekit/protocol/livekit" -) - -const ( - // hash of node_id => Node proto - NodesKey = "nodes" - - // hash of room_name => node_id - NodeRoomKey = "room_node_map" -) - -var redisCtx = context.Background() - -// location of the participant's RTC connection, hash -func participantRTCKey(participantKey livekit.ParticipantKey) string { - return "participant_rtc:" + string(participantKey) -} - -// location of the participant's Signal connection, hash -func participantSignalKey(connectionID livekit.ConnectionID) string { - return "participant_signal:" + string(connectionID) -} - -func rtcNodeChannel(nodeID livekit.NodeID) string { - return "rtc_channel:" + string(nodeID) -} - -func signalNodeChannel(nodeID livekit.NodeID) string { - return "signal_channel:" + string(nodeID) -} - -func publishRTCMessage(rc redis.UniversalClient, nodeID livekit.NodeID, participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, msg proto.Message) error { - rm := &livekit.RTCNodeMessage{ - ParticipantKey: string(participantKey), - ParticipantKeyB62: string(participantKeyB62), - } - switch o := msg.(type) { - case *livekit.StartSession: - rm.Message = &livekit.RTCNodeMessage_StartSession{ - StartSession: o, - } - case *livekit.SignalRequest: - rm.Message = &livekit.RTCNodeMessage_Request{ - Request: o, - } - case *livekit.RTCNodeMessage: - rm = o - rm.ParticipantKey = string(participantKey) - rm.ParticipantKeyB62 = string(participantKeyB62) - default: - return ErrInvalidRouterMessage - } - data, err := proto.Marshal(rm) - if err != nil { - return err - } - - // logger.Debugw("publishing to rtc", "rtcChannel", rtcNodeChannel(nodeID), - // "message", rm.Message) - return rc.Publish(redisCtx, rtcNodeChannel(nodeID), data).Err() -} - -func publishSignalMessage(rc redis.UniversalClient, nodeID livekit.NodeID, connectionID livekit.ConnectionID, msg proto.Message) error { - rm := &livekit.SignalNodeMessage{ - ConnectionId: string(connectionID), - } - switch o := msg.(type) { - case *livekit.SignalResponse: - rm.Message = &livekit.SignalNodeMessage_Response{ - Response: o, - } - case *livekit.EndSession: - rm.Message = &livekit.SignalNodeMessage_EndSession{ - EndSession: o, - } - default: - return ErrInvalidRouterMessage - } - data, err := proto.Marshal(rm) - if err != nil { - return err - } - - // logger.Debugw("publishing to signal", "signalChannel", signalNodeChannel(nodeID), - // "message", rm.Message) - return rc.Publish(redisCtx, signalNodeChannel(nodeID), data).Err() -} - -type RTCNodeSink struct { - rc redis.UniversalClient - nodeID livekit.NodeID - connectionID livekit.ConnectionID - participantKey livekit.ParticipantKey - participantKeyB62 livekit.ParticipantKey - isClosed atomic.Bool - onClose func() -} - -func NewRTCNodeSink( - rc redis.UniversalClient, - nodeID livekit.NodeID, - connectionID livekit.ConnectionID, - participantKey livekit.ParticipantKey, - participantKeyB62 livekit.ParticipantKey, -) *RTCNodeSink { - return &RTCNodeSink{ - rc: rc, - nodeID: nodeID, - connectionID: connectionID, - participantKey: participantKey, - participantKeyB62: participantKeyB62, - } -} - -func (s *RTCNodeSink) WriteMessage(msg proto.Message) error { - if s.isClosed.Load() { - return ErrChannelClosed - } - return publishRTCMessage(s.rc, s.nodeID, s.participantKey, s.participantKeyB62, msg) -} - -func (s *RTCNodeSink) Close() { - if s.isClosed.Swap(true) { - return - } - if s.onClose != nil { - s.onClose() - } -} - -func (s *RTCNodeSink) IsClosed() bool { - return s.isClosed.Load() -} - -func (s *RTCNodeSink) OnClose(f func()) { - s.onClose = f -} - -func (s *RTCNodeSink) ConnectionID() livekit.ConnectionID { - return s.connectionID -} - -// ---------------------------------------------------------------------- - -type SignalNodeSink struct { - rc redis.UniversalClient - nodeID livekit.NodeID - connectionID livekit.ConnectionID - isClosed atomic.Bool - onClose func() -} - -func NewSignalNodeSink(rc redis.UniversalClient, nodeID livekit.NodeID, connectionID livekit.ConnectionID) *SignalNodeSink { - return &SignalNodeSink{ - rc: rc, - nodeID: nodeID, - connectionID: connectionID, - } -} - -func (s *SignalNodeSink) WriteMessage(msg proto.Message) error { - if s.isClosed.Load() { - return ErrChannelClosed - } - return publishSignalMessage(s.rc, s.nodeID, s.connectionID, msg) -} - -func (s *SignalNodeSink) Close() { - if s.isClosed.Swap(true) { - return - } - _ = publishSignalMessage(s.rc, s.nodeID, s.connectionID, &livekit.EndSession{}) - if s.onClose != nil { - s.onClose() - } -} - -func (s *SignalNodeSink) IsClosed() bool { - return s.isClosed.Load() -} - -func (s *SignalNodeSink) OnClose(f func()) { - s.onClose = f -} - -func (s *SignalNodeSink) ConnectionID() livekit.ConnectionID { - return s.connectionID -} diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index 743d63d1c..579671814 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" "github.com/livekit/livekit-server/pkg/routing/selector" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -38,6 +39,12 @@ const ( participantMappingTTL = 24 * time.Hour statsUpdateInterval = 2 * time.Second statsMaxDelaySeconds = 30 + + // hash of node_id => Node proto + NodesKey = "nodes" + + // hash of room_name => node_id + NodeRoomKey = "room_node_map" ) var _ Router = (*RedisRouter)(nil) @@ -49,20 +56,21 @@ type RedisRouter struct { *LocalRouter rc redis.UniversalClient + kps rpc.KeepalivePubSub ctx context.Context isStarted atomic.Bool nodeMu sync.RWMutex // previous stats for computing averages prevStats *livekit.NodeStats - pubsub *redis.PubSub cancel func() } -func NewRedisRouter(lr *LocalRouter, rc redis.UniversalClient) *RedisRouter { +func NewRedisRouter(lr *LocalRouter, rc redis.UniversalClient, kps rpc.KeepalivePubSub) *RedisRouter { rr := &RedisRouter{ LocalRouter: lr, rc: rc, + kps: kps, } rr.ctx, rr.cancel = context.WithCancel(context.Background()) return rr @@ -164,33 +172,17 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) } -func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *livekit.RTCNodeMessage) error { - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), "ephemeral", livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) - defer rtcSink.Close() - return r.writeRTCMessage(rtcSink, msg) -} - -func (r *LocalRouter) writeRTCMessage(sink MessageSink, msg *livekit.RTCNodeMessage) error { - msg.SenderTime = time.Now().Unix() - return sink.WriteMessage(msg) -} - func (r *RedisRouter) Start() error { if r.isStarted.Swap(true) { return nil } - workerStarted := make(chan struct{}) + workerStarted := make(chan error) go r.statsWorker() - go r.redisWorker(workerStarted) + go r.keepaliveWorker(workerStarted) // wait until worker is running - select { - case <-workerStarted: - return nil - case <-time.After(3 * time.Second): - return errors.New("Unable to start redis router") - } + return <-workerStarted } func (r *RedisRouter) Drain() { @@ -207,7 +199,6 @@ func (r *RedisRouter) Stop() { return } logger.Debugw("stopping RedisRouter") - _ = r.pubsub.Close() _ = r.UnregisterNode() r.cancel() } @@ -219,9 +210,8 @@ func (r *RedisRouter) statsWorker() { // update periodically select { case <-time.After(statsUpdateInterval): - _ = r.WriteNodeRTC(context.Background(), r.currentNode.Id, &livekit.RTCNodeMessage{ - Message: &livekit.RTCNodeMessage_KeepAlive{}, - }) + r.kps.PublishPing(r.ctx, livekit.NodeID(r.currentNode.Id), &rpc.KeepalivePing{Timestamp: time.Now().Unix()}) + r.nodeMu.RLock() stats := r.currentNode.Stats r.nodeMu.RUnlock() @@ -245,44 +235,17 @@ func (r *RedisRouter) statsWorker() { } } -// worker that consumes redis messages intended for this node -func (r *RedisRouter) redisWorker(startedChan chan struct{}) { - defer func() { - logger.Debugw("finishing redisWorker", "nodeID", r.currentNode.Id) - }() - logger.Debugw("starting redisWorker", "nodeID", r.currentNode.Id) - - rtcChannel := rtcNodeChannel(livekit.NodeID(r.currentNode.Id)) - r.pubsub = r.rc.Subscribe(r.ctx, rtcChannel) - - close(startedChan) - for msg := range r.pubsub.Channel() { - if msg == nil { - return - } - - if msg.Channel == rtcChannel { - rm := livekit.RTCNodeMessage{} - if err := proto.Unmarshal([]byte(msg.Payload), &rm); err != nil { - logger.Errorw("could not unmarshal RTC message on rtcchan", err) - prometheus.MessageCounter.WithLabelValues("rtc", "failure").Add(1) - continue - } - if err := r.handleRTCMessage(&rm); err != nil { - logger.Errorw("error processing RTC message", err) - prometheus.MessageCounter.WithLabelValues("rtc", "failure").Add(1) - continue - } - prometheus.MessageCounter.WithLabelValues("rtc", "success").Add(1) - } +func (r *RedisRouter) keepaliveWorker(startedChan chan error) { + pings, err := r.kps.SubscribePing(r.ctx, livekit.NodeID(r.currentNode.Id)) + if err != nil { + startedChan <- err + return } -} + close(startedChan) -func (r *RedisRouter) handleRTCMessage(rm *livekit.RTCNodeMessage) error { - switch rm.Message.(type) { - case *livekit.RTCNodeMessage_KeepAlive: - if time.Since(time.Unix(rm.SenderTime, 0)) > statsUpdateInterval { - logger.Infow("keep alive too old, skipping", "senderTime", rm.SenderTime) + for ping := range pings.Channel() { + if time.Since(time.Unix(ping.Timestamp, 0)) > statsUpdateInterval { + logger.Infow("keep alive too old, skipping", "timestamp", ping.Timestamp) break } @@ -294,7 +257,7 @@ func (r *RedisRouter) handleRTCMessage(rm *livekit.RTCNodeMessage) error { if err != nil { logger.Errorw("could not update node stats", err) r.nodeMu.Unlock() - return err + continue } r.currentNode.Stats = updated if computedAvg { @@ -307,5 +270,4 @@ func (r *RedisRouter) handleRTCMessage(rm *livekit.RTCNodeMessage) error { logger.Errorw("could not update node", err) } } - return nil } diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 93e3784b0..0a2501a1d 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -82,6 +82,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live getSignalRelayConfig, NewDefaultSignalServer, routing.NewSignalClient, + rpc.NewKeepalivePubSub, getPSRPCConfig, getPSRPCClientParams, rpc.NewTopicFormatter, @@ -103,7 +104,10 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi getNodeID, getMessageBus, getSignalRelayConfig, + getPSRPCConfig, + getPSRPCClientParams, routing.NewSignalClient, + rpc.NewKeepalivePubSub, routing.CreateRouter, ) diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 5b2a3aab6..e76fa56c7 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -50,7 +50,12 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - router := routing.CreateRouter(universalClient, currentNode, signalClient) + clientParams := getPSRPCClientParams(psrpcConfig, messageBus) + keepalivePubSub, err := rpc.NewKeepalivePubSub(clientParams) + if err != nil { + return nil, err + } + router := routing.CreateRouter(universalClient, currentNode, signalClient, keepalivePubSub) objectStore := createStore(universalClient) roomAllocator, err := NewRoomAllocator(conf, router, objectStore) if err != nil { @@ -60,7 +65,6 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - clientParams := getPSRPCClientParams(psrpcConfig, messageBus) egressClient, err := rpc.NewEgressClient(clientParams) if err != nil { return nil, err @@ -149,7 +153,13 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi if err != nil { return nil, err } - router := routing.CreateRouter(universalClient, currentNode, signalClient) + psrpcConfig := getPSRPCConfig(conf) + clientParams := getPSRPCClientParams(psrpcConfig, messageBus) + keepalivePubSub, err := rpc.NewKeepalivePubSub(clientParams) + if err != nil { + return nil, err + } + router := routing.CreateRouter(universalClient, currentNode, signalClient, keepalivePubSub) return router, nil } From 867325d1204df2a42e0300bc871000a6e135361b Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 22 Jan 2024 05:18:12 -0800 Subject: [PATCH 084/114] restore legacy room delete behavior (#2400) --- pkg/service/interfaces.go | 2 +- pkg/service/roomservice.go | 9 ++- .../servicefakes/fake_service_store.go | 76 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index 5dd0cb20b..ec8359e9d 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -35,7 +35,6 @@ type ObjectStore interface { UnlockRoom(ctx context.Context, roomName livekit.RoomName, uid string) error StoreRoom(ctx context.Context, room *livekit.Room, internal *livekit.RoomInternal) error - DeleteRoom(ctx context.Context, roomName livekit.RoomName) error StoreParticipant(ctx context.Context, roomName livekit.RoomName, participant *livekit.ParticipantInfo) error DeleteParticipant(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity) error @@ -44,6 +43,7 @@ type ObjectStore interface { //counterfeiter:generate . ServiceStore type ServiceStore interface { LoadRoom(ctx context.Context, roomName livekit.RoomName, includeInternal bool) (*livekit.Room, *livekit.RoomInternal, error) + DeleteRoom(ctx context.Context, roomName livekit.RoomName) error // ListRooms returns currently active rooms. if names is not nil, it'll filter and return // only rooms that match diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 003da7f19..d9bac7065 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/psrpc" ) // A rooms service that supports a single node @@ -166,7 +167,13 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, twirpAuthError(err) } - return s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + _, err := s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + if !errors.Is(err, psrpc.ErrNoResponse) { + return &livekit.DeleteRoomResponse{}, err + } + + err = s.roomStore.DeleteRoom(ctx, livekit.RoomName(req.Room)) + return &livekit.DeleteRoomResponse{}, err } func (s *RoomService) ListParticipants(ctx context.Context, req *livekit.ListParticipantsRequest) (*livekit.ListParticipantsResponse, error) { diff --git a/pkg/service/servicefakes/fake_service_store.go b/pkg/service/servicefakes/fake_service_store.go index 0ac442161..7ecf3e213 100644 --- a/pkg/service/servicefakes/fake_service_store.go +++ b/pkg/service/servicefakes/fake_service_store.go @@ -10,6 +10,18 @@ import ( ) type FakeServiceStore struct { + DeleteRoomStub func(context.Context, livekit.RoomName) error + deleteRoomMutex sync.RWMutex + deleteRoomArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomName + } + deleteRoomReturns struct { + result1 error + } + deleteRoomReturnsOnCall map[int]struct { + result1 error + } ListParticipantsStub func(context.Context, livekit.RoomName) ([]*livekit.ParticipantInfo, error) listParticipantsMutex sync.RWMutex listParticipantsArgsForCall []struct { @@ -74,6 +86,68 @@ type FakeServiceStore struct { invocationsMutex sync.RWMutex } +func (fake *FakeServiceStore) DeleteRoom(arg1 context.Context, arg2 livekit.RoomName) error { + fake.deleteRoomMutex.Lock() + ret, specificReturn := fake.deleteRoomReturnsOnCall[len(fake.deleteRoomArgsForCall)] + fake.deleteRoomArgsForCall = append(fake.deleteRoomArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomName + }{arg1, arg2}) + stub := fake.DeleteRoomStub + fakeReturns := fake.deleteRoomReturns + fake.recordInvocation("DeleteRoom", []interface{}{arg1, arg2}) + fake.deleteRoomMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeServiceStore) DeleteRoomCallCount() int { + fake.deleteRoomMutex.RLock() + defer fake.deleteRoomMutex.RUnlock() + return len(fake.deleteRoomArgsForCall) +} + +func (fake *FakeServiceStore) DeleteRoomCalls(stub func(context.Context, livekit.RoomName) error) { + fake.deleteRoomMutex.Lock() + defer fake.deleteRoomMutex.Unlock() + fake.DeleteRoomStub = stub +} + +func (fake *FakeServiceStore) DeleteRoomArgsForCall(i int) (context.Context, livekit.RoomName) { + fake.deleteRoomMutex.RLock() + defer fake.deleteRoomMutex.RUnlock() + argsForCall := fake.deleteRoomArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeServiceStore) DeleteRoomReturns(result1 error) { + fake.deleteRoomMutex.Lock() + defer fake.deleteRoomMutex.Unlock() + fake.DeleteRoomStub = nil + fake.deleteRoomReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeServiceStore) DeleteRoomReturnsOnCall(i int, result1 error) { + fake.deleteRoomMutex.Lock() + defer fake.deleteRoomMutex.Unlock() + fake.DeleteRoomStub = nil + if fake.deleteRoomReturnsOnCall == nil { + fake.deleteRoomReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteRoomReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeServiceStore) ListParticipants(arg1 context.Context, arg2 livekit.RoomName) ([]*livekit.ParticipantInfo, error) { fake.listParticipantsMutex.Lock() ret, specificReturn := fake.listParticipantsReturnsOnCall[len(fake.listParticipantsArgsForCall)] @@ -347,6 +421,8 @@ func (fake *FakeServiceStore) LoadRoomReturnsOnCall(i int, result1 *livekit.Room func (fake *FakeServiceStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.deleteRoomMutex.RLock() + defer fake.deleteRoomMutex.RUnlock() fake.listParticipantsMutex.RLock() defer fake.listParticipantsMutex.RUnlock() fake.listRoomsMutex.RLock() From f6608977f08a779584f9246553204912951167b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Fuente=20P=C3=A9rez?= Date: Tue, 23 Jan 2024 02:11:34 +0100 Subject: [PATCH 085/114] Fix race condition on Participant.updateState (#2401) The comparisson between the last and current ParticipantInfo_State wasn't atomic. This sometimes resulted in two calls to onStateChange method for the same participant state. In the end this was reflected in two ACTIVE events being generated for the same participant at exactly the same moment. The fix actually uses the atomic method Swap to properly protect the "compare and set" operation and avoid any race condition. --- pkg/rtc/participant.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index bfb3112ca..de62da35e 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1242,12 +1242,11 @@ func (p *ParticipantImpl) setupParticipantTrafficLoad() { func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { oldState := p.State() - if state == oldState { + if !(p.state.Swap(state) != state) { return } p.params.Logger.Debugw("updating participant state", "state", state.String()) - p.state.Store(state) p.dirty.Store(true) p.lock.RLock() From 89c7cec2ad957856295a7ff1c48291b5ebcf3a8b Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Wed, 24 Jan 2024 20:01:22 +0200 Subject: [PATCH 086/114] SIP: New protocol for creating participants. (#2404) --- go.mod | 4 +- go.sum | 10 ++-- pkg/service/interfaces.go | 5 -- pkg/service/redisstore.go | 38 +------------- pkg/service/sip.go | 105 +++++++++++--------------------------- 5 files changed, 38 insertions(+), 124 deletions(-) diff --git a/go.mod b/go.mod index e111fbb21..b13a7d01e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485 + github.com/livekit/protocol v1.9.5 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -102,6 +102,6 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect - google.golang.org/grpc v1.60.1 // indirect + google.golang.org/grpc v1.61.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index cb4e8d092..41f44c6e9 100644 --- a/go.sum +++ b/go.sum @@ -126,10 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8 h1:E9s9KFCuKgYWYgaKz0ZmC7K3cPr8Iij77HbnwhQ4JZw= -github.com/livekit/protocol v1.9.5-0.20240118112540-cf33ad3861d8/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= -github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485 h1:X75uVI0+YA7QN28NaVniP4IjhbcDWlktZ3Ec+PHjoHA= -github.com/livekit/protocol v1.9.5-0.20240121141201-9e82495c0485/go.mod h1:Qv55+z0kD0NYp/G0qAaFA4Mjalxt7tsOJwrvV3HymsA= +github.com/livekit/protocol v1.9.5 h1:/I6maM05euoUxrV6je16Qj5yCnCSPZ+nhHzm8akLCVk= +github.com/livekit/protocol v1.9.5/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -425,8 +423,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index ec8359e9d..73c7d4a67 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -88,9 +88,4 @@ type SIPStore interface { LoadSIPDispatchRule(ctx context.Context, sipDispatchRuleID string) (*livekit.SIPDispatchRuleInfo, error) ListSIPDispatchRule(ctx context.Context) ([]*livekit.SIPDispatchRuleInfo, error) DeleteSIPDispatchRule(ctx context.Context, info *livekit.SIPDispatchRuleInfo) error - - StoreSIPParticipant(ctx context.Context, info *livekit.SIPParticipantInfo) error - LoadSIPParticipant(ctx context.Context, sipParticipantID string) (*livekit.SIPParticipantInfo, error) - ListSIPParticipant(ctx context.Context) ([]*livekit.SIPParticipantInfo, error) - DeleteSIPParticipant(ctx context.Context, info *livekit.SIPParticipantInfo) error } diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index dfb4c3626..91b437f40 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -26,11 +26,12 @@ import ( "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" - "github.com/livekit/livekit-server/version" "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + + "github.com/livekit/livekit-server/version" ) const ( @@ -53,7 +54,6 @@ const ( SIPTrunkKey = "sip_trunk" SIPDispatchRuleKey = "sip_dispatch_rule" - SIPParticipantKey = "sip_participant" // RoomParticipantsPrefix is hash of participant_name => ParticipantInfo RoomParticipantsPrefix = "room_participants:" @@ -908,37 +908,3 @@ func (s *RedisStore) ListSIPDispatchRule(ctx context.Context) (infos []*livekit. return infos, err } - -func (s *RedisStore) StoreSIPParticipant(ctx context.Context, info *livekit.SIPParticipantInfo) error { - data, err := proto.Marshal(info) - if err != nil { - return err - } - - return s.rc.HSet(s.ctx, SIPParticipantKey, info.SipParticipantId, data).Err() -} -func (s *RedisStore) LoadSIPParticipant(ctx context.Context, sipParticipantId string) (*livekit.SIPParticipantInfo, error) { - info := &livekit.SIPParticipantInfo{} - if err := s.loadOne(ctx, SIPParticipantKey, sipParticipantId, info, ErrSIPParticipantNotFound); err != nil { - return nil, err - } - - return info, nil -} - -func (s *RedisStore) DeleteSIPParticipant(ctx context.Context, info *livekit.SIPParticipantInfo) error { - return s.rc.HDel(s.ctx, SIPParticipantKey, info.SipParticipantId).Err() -} - -func (s *RedisStore) ListSIPParticipant(ctx context.Context) (infos []*livekit.SIPParticipantInfo, err error) { - err = s.loadMany(ctx, SIPParticipantKey, func() proto.Message { - infos = append(infos, &livekit.SIPParticipantInfo{}) - return infos[len(infos)-1] - }) - - return infos, err -} - -func (s *RedisStore) SendSIPParticipantDTMF(ctx context.Context, info *livekit.SendSIPParticipantDTMFRequest) (*livekit.SIPParticipantDTMFInfo, error) { - return nil, fmt.Errorf("TODO") -} diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 8bba2fdca..a95dbad09 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -16,6 +16,7 @@ package service import ( "context" + "time" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -161,89 +162,43 @@ func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.Crea return nil, ErrSIPNotConnected } - info := &livekit.SIPParticipantInfo{ - SipParticipantId: utils.NewGuid(utils.SIPParticipantPrefix), - SipTrunkId: req.SipTrunkId, - SipCallTo: req.SipCallTo, + AppendLogFields(ctx, "room", req.RoomName, "trunk", req.SipTrunkId, "to", req.SipCallTo) + ireq := &rpc.InternalCreateSIPParticipantRequest{ + CallTo: req.SipCallTo, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, } - - if err := s.store.StoreSIPParticipant(ctx, info); err != nil { - return nil, err - } - s.updateParticipant(ctx, info) - return info, nil -} - -func (s *SIPService) updateParticipant(ctx context.Context, info *livekit.SIPParticipantInfo) { - AppendLogFields(ctx, "participantId", info.SipParticipantId, "room", info.RoomName, "trunk", info.SipTrunkId, "to", info.SipCallTo) - req := &rpc.InternalUpdateSIPParticipantRequest{ - ParticipantId: info.SipParticipantId, - CallTo: info.SipCallTo, - RoomName: info.RoomName, - ParticipantIdentity: info.ParticipantIdentity, - } - if info.SipTrunkId != "" { - trunk, err := s.store.LoadSIPTrunk(ctx, info.SipTrunkId) + if req.SipTrunkId != "" { + trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId) if err != nil { logger.Errorw("cannot get trunk to update sip participant", err) - return + return nil, err } - req.Address = trunk.OutboundAddress - req.Number = trunk.OutboundNumber - req.Username = trunk.OutboundUsername - req.Password = trunk.OutboundPassword + ireq.Address = trunk.OutboundAddress + ireq.Number = trunk.OutboundNumber + ireq.Username = trunk.OutboundUsername + ireq.Password = trunk.OutboundPassword } - if _, err := s.psrpcClient.UpdateSIPParticipant(ctx, "", req); err != nil { + + // CreateSIPParticipant will wait for LiveKit Participant to be created and that can take some time. + // Thus, we must set a higher deadline for it, if it's not set already. + // TODO: support context timeouts in psrpc + timeout := 30 * time.Second + if deadline, ok := ctx.Deadline(); ok { + timeout = time.Until(deadline) + } else { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + resp, err := s.psrpcClient.CreateSIPParticipant(ctx, "", ireq, psrpc.WithRequestTimeout(timeout)) + if err != nil { logger.Errorw("cannot update sip participant", err) - } -} - -func (s *SIPService) ListSIPParticipant(ctx context.Context, req *livekit.ListSIPParticipantRequest) (*livekit.ListSIPParticipantResponse, error) { - if s.store == nil { - return nil, ErrSIPNotConnected - } - - participants, err := s.store.ListSIPParticipant(ctx) - if err != nil { return nil, err } - - return &livekit.ListSIPParticipantResponse{Items: participants}, nil -} - -func (s *SIPService) DeleteSIPParticipant(ctx context.Context, req *livekit.DeleteSIPParticipantRequest) (*livekit.SIPParticipantInfo, error) { - if s.store == nil { - return nil, ErrSIPNotConnected - } - - info, err := s.store.LoadSIPParticipant(ctx, req.SipParticipantId) - if err != nil { - return nil, err - } - - if err = s.store.DeleteSIPParticipant(ctx, info); err != nil { - return nil, err - } - // These indicate that the call should be disconnected - info.SipTrunkId = "" - info.SipCallTo = "" - s.updateParticipant(ctx, info) - return info, nil -} - -func (s *SIPService) SendSIPParticipantDTMF(ctx context.Context, req *livekit.SendSIPParticipantDTMFRequest) (*livekit.SIPParticipantDTMFInfo, error) { - if s.store == nil { - return nil, ErrSIPNotConnected - } - AppendLogFields(ctx, "participantId", req.SipParticipantId) - _, err := s.psrpcClient.SendSIPParticipantDTMF(ctx, &rpc.InternalSendSIPParticipantDTMFRequest{ - ParticipantId: req.SipParticipantId, - Digits: req.Digits, - }) - if err != nil { - logger.Errorw("cannot send dtmf to sip participant", err) - } - return nil, err + return &livekit.SIPParticipantInfo{ + ParticipantId: resp.ParticipantId, + ParticipantIdentity: resp.ParticipantIdentity, + RoomName: req.RoomName, + }, nil } From 79cdc2df2ea5bd6004fbfc84e8be9c7ad6709c34 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 25 Jan 2024 01:24:09 +0530 Subject: [PATCH 087/114] Unify muted and unmuted migration paths. (#2406) * Unify muted and unmuted migration paths. If dynacast had disabled all layers, after a migration, the client did not restart publish (it is akin to muted track). That failed migration because migration state machine waits for unmuted tracks to be published (i. e. server has to receive packets). If a migrating track is in muted state, server does not wait for packets. It synthesises the published event and catches up later when packets actually come in. Just treating all migrations as the erstwhile muted case. Sythesise publish whether track is muted or not. In the unmuted case, packets might arrive soon after whereas in muted case, it will depend on when unmute happens. This is tricky stuff. So, will need good testing. * use muted from track info --- pkg/rtc/mediatrack.go | 3 +- pkg/rtc/participant.go | 97 ++++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 57 deletions(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 3cc45dba6..ba714228d 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -125,6 +125,7 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { func (t *MediaTrack) OnSubscribedMaxQualityChange( f func( trackID livekit.TrackID, + trackInfo *livekit.TrackInfo, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality, ) error, @@ -135,7 +136,7 @@ func (t *MediaTrack) OnSubscribedMaxQualityChange( handler := func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) { if f != nil && !t.IsMuted() { - _ = f(t.ID(), subscribedQualities, maxSubscribedQualities) + _ = f(t.ID(), t.ToProto(), subscribedQualities, maxSubscribedQualities) } for _, q := range maxSubscribedQualities { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index de62da35e..b1c9857c5 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -161,8 +161,8 @@ type ParticipantImpl struct { pendingTracksLock utils.RWMutex pendingTracks map[string]*pendingTrackInfo pendingPublishingTracks map[livekit.TrackID]*pendingTrackInfo - // migrated in muted tracks are not fired need close at participant close - mutedTrackNotFired []*MediaTrack + // migrated in tracks are not fired need close at participant close + pendingMigratedTracks []*MediaTrack // supported codecs enabledPublishCodecs []*livekit.Codec @@ -632,13 +632,15 @@ func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) er } if p.MigrateState() == types.MigrateStateSync { - go p.handleMigrateMutedTrack() + go p.handleMigrateTracks() } return nil } -func (p *ParticipantImpl) handleMigrateMutedTrack() { - // muted track won't send rtp packet, so we add mediatrack manually +func (p *ParticipantImpl) handleMigrateTracks() { + // muted track won't send rtp packet, so it is required to add mediatrack manually. + // But, synthesising track publish for unmuted tracks keeps a consistent path. + // In both csaes (muted and unmuted), when publisher sends media packets, OnTrack would register and go from there. var addedTracks []*MediaTrack p.pendingTracksLock.Lock() for cid, pti := range p.pendingTracks { @@ -650,17 +652,14 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { p.pubLogger.Warnw("too many pending migrated tracks", nil, "trackID", pti.trackInfos[0].Sid, "count", len(pti.trackInfos), "cid", cid) } - ti := pti.trackInfos[0] - if ti.Muted { - mt := p.addMigrateMutedTrack(cid, ti) - if mt != nil { - addedTracks = append(addedTracks, mt) - } else { - p.pubLogger.Warnw("could not find migrated muted track", nil, "cid", cid) - } + mt := p.addMigratedTrack(cid, pti.trackInfos[0]) + if mt != nil { + addedTracks = append(addedTracks, mt) + } else { + p.pubLogger.Warnw("could not find migrated muted track", nil, "cid", cid) } } - p.mutedTrackNotFired = append(p.mutedTrackNotFired, addedTracks...) + p.pendingMigratedTracks = append(p.pendingMigratedTracks, addedTracks...) if len(addedTracks) != 0 { p.dirty.Store(true) @@ -676,12 +675,12 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { }() } -func (p *ParticipantImpl) removeMutedTrackNotFired(mt *MediaTrack) { +func (p *ParticipantImpl) removePendingMigratedTrack(mt *MediaTrack) { p.pendingTracksLock.Lock() - for i, t := range p.mutedTrackNotFired { + for i, t := range p.pendingMigratedTracks { if t == mt { - p.mutedTrackNotFired[i] = p.mutedTrackNotFired[len(p.mutedTrackNotFired)-1] - p.mutedTrackNotFired = p.mutedTrackNotFired[:len(p.mutedTrackNotFired)-1] + p.pendingMigratedTracks[i] = p.pendingMigratedTracks[len(p.pendingMigratedTracks)-1] + p.pendingMigratedTracks = p.pendingMigratedTracks[:len(p.pendingMigratedTracks)-1] break } } @@ -766,11 +765,11 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.pendingTracksLock.Lock() p.pendingTracks = make(map[string]*pendingTrackInfo) - closeMutedTrack := p.mutedTrackNotFired - p.mutedTrackNotFired = p.mutedTrackNotFired[:0] + pendingMigratedTracksToClose := p.pendingMigratedTracks + p.pendingMigratedTracks = p.pendingMigratedTracks[:0] p.pendingTracksLock.Unlock() - for _, t := range closeMutedTrack { + for _, t := range pendingMigratedTracksToClose { t.Close(isExpectedToResume) } @@ -1552,7 +1551,12 @@ func (p *ParticipantImpl) onStreamStateChange(update *streamallocator.StreamStat }) } -func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) error { +func (p *ParticipantImpl) onSubscribedMaxQualityChange( + trackID livekit.TrackID, + trackInfo *livekit.TrackInfo, + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, +) error { if p.params.DisableDynacast { return nil } @@ -1561,6 +1565,17 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, return nil } + // send layer info about max subscription changes to telemetry + for _, maxSubscribedQuality := range maxSubscribedQualities { + p.params.Telemetry.TrackMaxSubscribedVideoQuality( + context.Background(), + p.ID(), + trackInfo, + maxSubscribedQuality.CodecMime, + maxSubscribedQuality.Quality, + ) + } + // normalize the codec name for _, subscribedQuality := range subscribedQualities { subscribedQuality.Codec = strings.ToLower(strings.TrimLeft(subscribedQuality.Codec, "video/")) @@ -1572,36 +1587,6 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, SubscribedCodecs: subscribedQualities, } - // send layer info about max subscription changes to telemetry - track := p.UpTrackManager.GetPublishedTrack(trackID) - var layerInfo map[livekit.VideoQuality]*livekit.VideoLayer - if track != nil { - layers := track.ToProto().Layers - layerInfo = make(map[livekit.VideoQuality]*livekit.VideoLayer, len(layers)) - for _, layer := range layers { - layerInfo[layer.Quality] = layer - } - } - - for _, maxSubscribedQuality := range maxSubscribedQualities { - ti := &livekit.TrackInfo{ - Sid: string(trackID), - Type: livekit.TrackType_VIDEO, - } - if info, ok := layerInfo[maxSubscribedQuality.Quality]; ok { - ti.Width = info.Width - ti.Height = info.Height - } - - p.params.Telemetry.TrackMaxSubscribedVideoQuality( - context.Background(), - p.ID(), - ti, - maxSubscribedQuality.CodecMime, - maxSubscribedQuality.Quality, - ) - } - p.pubLogger.Debugw( "sending max subscribed quality", "trackID", trackID, @@ -1864,7 +1849,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei p.pendingTracksLock.Unlock() if mt.AddReceiver(rtpReceiver, track, p.twcc, mid) { - p.removeMutedTrackNotFired(mt) + p.removePendingMigratedTrack(mt) } if newTrack { @@ -1881,8 +1866,8 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei return mt, newTrack } -func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { - p.pubLogger.Infow("add migrate muted track", "cid", cid, "trackID", ti.Sid, "track", logger.Proto(ti)) +func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { + p.pubLogger.Infow("add migrated track", "cid", cid, "trackID", ti.Sid, "track", logger.Proto(ti)) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { p.pubLogger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid) @@ -1929,7 +1914,7 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo } } mt.SetSimulcast(ti.Simulcast) - mt.SetMuted(true) + mt.SetMuted(ti.Muted) return mt } From 43a40eb52d817b5f6a8e4e2de69cfbf4c49c1211 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 25 Jan 2024 10:27:55 +0530 Subject: [PATCH 088/114] Using minimal TrackInfo when reporing to telemetry. (#2407) Used the full TrackInfo in my previous PR, but telemetry might be relying on top level Width/Height. So, make a pared down TrackInfo to report to telemetry. Also, correct some spelling/comments. --- pkg/rtc/participant.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b1c9857c5..ec1bb5596 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -161,7 +161,7 @@ type ParticipantImpl struct { pendingTracksLock utils.RWMutex pendingTracks map[string]*pendingTrackInfo pendingPublishingTracks map[livekit.TrackID]*pendingTrackInfo - // migrated in tracks are not fired need close at participant close + // migrated in tracks that have not fired need close at participant close pendingMigratedTracks []*MediaTrack // supported codecs @@ -640,7 +640,7 @@ func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) er func (p *ParticipantImpl) handleMigrateTracks() { // muted track won't send rtp packet, so it is required to add mediatrack manually. // But, synthesising track publish for unmuted tracks keeps a consistent path. - // In both csaes (muted and unmuted), when publisher sends media packets, OnTrack would register and go from there. + // In both cases (muted and unmuted), when publisher sends media packets, OnTrack would register and go from there. var addedTracks []*MediaTrack p.pendingTracksLock.Lock() for cid, pti := range p.pendingTracks { @@ -656,7 +656,7 @@ func (p *ParticipantImpl) handleMigrateTracks() { if mt != nil { addedTracks = append(addedTracks, mt) } else { - p.pubLogger.Warnw("could not find migrated muted track", nil, "cid", cid) + p.pubLogger.Warnw("could not find migrated track", nil, "cid", cid) } } p.pendingMigratedTracks = append(p.pendingMigratedTracks, addedTracks...) @@ -1567,10 +1567,21 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange( // send layer info about max subscription changes to telemetry for _, maxSubscribedQuality := range maxSubscribedQualities { + ti := &livekit.TrackInfo{ + Sid: trackInfo.Sid, + Type: trackInfo.Type, + } + for _, layer := range trackInfo.Layers { + if layer.Quality == maxSubscribedQuality.Quality { + ti.Width = layer.Width + ti.Height = layer.Height + break + } + } p.params.Telemetry.TrackMaxSubscribedVideoQuality( context.Background(), p.ID(), - trackInfo, + ti, maxSubscribedQuality.CodecMime, maxSubscribedQuality.Quality, ) From d3da94c45e0c09966babe14c8bee6f5db029ce30 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 25 Jan 2024 22:22:46 +0530 Subject: [PATCH 089/114] Augment LeaveRequest with alternate regions to connect. (#2408) * Augment LeaveRequest with alternate regions to connect. * update protocol and issue resume action on close if expected to resume * use current protocol in tests * address feedback --- go.mod | 4 +- go.sum | 8 +- pkg/rtc/participant.go | 61 +++- pkg/rtc/types/interfaces.go | 19 +- pkg/rtc/types/protocol_version.go | 6 +- .../typesfakes/fake_local_participant.go | 39 +++ pkg/service/roommanager.go | 37 ++- pkg/service/servicefakes/fake_sipstore.go | 312 ------------------ test/client/client.go | 7 +- 9 files changed, 146 insertions(+), 347 deletions(-) diff --git a/go.mod b/go.mod index b13a7d01e..bcec95e46 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.5 + github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -101,7 +101,7 @@ require ( golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/grpc v1.61.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 41f44c6e9..d8ae78ab4 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.5 h1:/I6maM05euoUxrV6je16Qj5yCnCSPZ+nhHzm8akLCVk= -github.com/livekit/protocol v1.9.5/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= +github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557 h1:lz11rtmZouO9C7RcDJpg5tWERUEAMLTYjtHX/mRyras= +github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -421,8 +421,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ec1bb5596..ff6758d80 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -221,6 +221,8 @@ type ParticipantImpl struct { // loggers for publisher and subscriber pubLogger logger.Logger subLogger logger.Logger + + regionSettings *livekit.RegionSettings } func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { @@ -582,8 +584,7 @@ func (p *ParticipantImpl) HandleSignalSourceClose() { p.TransportManager.SetSignalSourceValid(false) if !p.HasConnected() { - reason := types.ParticipantCloseReasonJoinFailed - _ = p.Close(false, reason, false) + _ = p.Close(false, types.ParticipantCloseReasonSignalSourceClose, false) } } @@ -749,12 +750,32 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.clearMigrationTimer() // send leave message - if sendLeave { + var leave *livekit.LeaveRequest + if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { + leave = &livekit.LeaveRequest{ + Reason: reason.ToDisconnectReason(), + } + if isExpectedToResume { + leave.Action = livekit.LeaveRequest_RESUME + } else { + leave.Action = livekit.LeaveRequest_DISCONNECT + } + // although regions are not needed when resuming OR disconnecting, + // send it if available, just in case clients want to fall back. + p.lock.RLock() + if p.regionSettings != nil { + leave.Regions = proto.Clone(p.regionSettings).(*livekit.RegionSettings) + } + p.lock.RUnlock() + } else if sendLeave { + leave = &livekit.LeaveRequest{ + Reason: reason.ToDisconnectReason(), + } + } + if leave != nil { _ = p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Leave{ - Leave: &livekit.LeaveRequest{ - Reason: reason.ToDisconnectReason(), - }, + Leave: leave, }, }) } @@ -2282,12 +2303,26 @@ func (p *ParticipantImpl) GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.R } func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason) { + var leave *livekit.LeaveRequest + if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { + leave = &livekit.LeaveRequest{ + Reason: reason.ToDisconnectReason(), + Action: livekit.LeaveRequest_RECONNECT, + } + p.lock.RLock() + if p.regionSettings != nil { + leave.Regions = proto.Clone(p.regionSettings).(*livekit.RegionSettings) + } + p.lock.RUnlock() + } else { + leave = &livekit.LeaveRequest{ + CanReconnect: true, + Reason: reason.ToDisconnectReason(), + } + } _ = p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Leave{ - Leave: &livekit.LeaveRequest{ - CanReconnect: true, - Reason: reason.ToDisconnectReason(), - }, + Leave: leave, }, }) @@ -2451,3 +2486,9 @@ func (p *ParticipantImpl) setupEnabledCodecs(publishEnabledCodecs []*livekit.Cod } p.enabledSubscribeCodecs = subscribeCodecs } + +func (p *ParticipantImpl) SetRegionSettings(regionSettings *livekit.RegionSettings) { + p.lock.Lock() + p.regionSettings = proto.Clone(regionSettings).(*livekit.RegionSettings) + p.lock.Unlock() +} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 152513efc..466615c8a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -105,6 +105,7 @@ const ( ParticipantCloseReasonSubscriptionError ParticipantCloseReasonDataChannelError ParticipantCloseReasonMigrateCodecMismatch + ParticipantCloseReasonSignalSourceClose ) func (p ParticipantCloseReason) String() string { @@ -157,6 +158,8 @@ func (p ParticipantCloseReason) String() string { return "DATA_CHANNEL_ERROR" case ParticipantCloseReasonMigrateCodecMismatch: return "MIGRATE_CODEC_MISMATCH" + case ParticipantCloseReasonSignalSourceClose: + return "SIGNAL_SOURCE_CLOSE" default: return fmt.Sprintf("%d", int(p)) } @@ -173,22 +176,20 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_JOIN_FAILURE case ParticipantCloseReasonPeerConnectionDisconnected: return livekit.DisconnectReason_STATE_MISMATCH - case ParticipantCloseReasonDuplicateIdentity, ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonStale: + case ParticipantCloseReasonDuplicateIdentity, ParticipantCloseReasonStale: return livekit.DisconnectReason_DUPLICATE_IDENTITY + case ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonSimulateMigration: + return livekit.DisconnectReason_MIGRATION case ParticipantCloseReasonServiceRequestRemoveParticipant: return livekit.DisconnectReason_PARTICIPANT_REMOVED case ParticipantCloseReasonServiceRequestDeleteRoom: return livekit.DisconnectReason_ROOM_DELETED - case ParticipantCloseReasonSimulateMigration: - return livekit.DisconnectReason_DUPLICATE_IDENTITY - case ParticipantCloseReasonSimulateNodeFailure: - return livekit.DisconnectReason_SERVER_SHUTDOWN - case ParticipantCloseReasonSimulateServerLeave: - return livekit.DisconnectReason_SERVER_SHUTDOWN - case ParticipantCloseReasonOvercommitted: + case ParticipantCloseReasonSimulateNodeFailure, ParticipantCloseReasonSimulateServerLeave, ParticipantCloseReasonOvercommitted: return livekit.DisconnectReason_SERVER_SHUTDOWN case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError, ParticipantCloseReasonDataChannelError, ParticipantCloseReasonMigrateCodecMismatch: return livekit.DisconnectReason_STATE_MISMATCH + case ParticipantCloseReasonSignalSourceClose: + return livekit.DisconnectReason_SIGNAL_CLOSE default: // the other types will map to unknown reason return livekit.DisconnectReason_UNKNOWN_REASON @@ -421,6 +422,8 @@ type LocalParticipant interface { GetPacer() pacer.Pacer GetTrafficLoad() *TrafficLoad + + SetRegionSettings(regionSettings *livekit.RegionSettings) } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index ac445e9d8..08075184d 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -16,7 +16,7 @@ package types type ProtocolVersion int -const CurrentProtocol = 12 +const CurrentProtocol = 13 func (v ProtocolVersion) SupportsPackedStreamId() bool { return v > 0 @@ -83,3 +83,7 @@ func (v ProtocolVersion) SupportsConnectionQualityLost() bool { func (v ProtocolVersion) SupportsAsyncRoomID() bool { return v > 11 } + +func (v ProtocolVersion) SupportsRegionsInLeaveRequest() bool { + return v > 12 +} diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 0cd75616a..756099d2c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -753,6 +753,11 @@ type FakeLocalParticipant struct { setPermissionReturnsOnCall map[int]struct { result1 bool } + SetRegionSettingsStub func(*livekit.RegionSettings) + setRegionSettingsMutex sync.RWMutex + setRegionSettingsArgsForCall []struct { + arg1 *livekit.RegionSettings + } SetResponseSinkStub func(routing.MessageSink) setResponseSinkMutex sync.RWMutex setResponseSinkArgsForCall []struct { @@ -4981,6 +4986,38 @@ func (fake *FakeLocalParticipant) SetPermissionReturnsOnCall(i int, result1 bool }{result1} } +func (fake *FakeLocalParticipant) SetRegionSettings(arg1 *livekit.RegionSettings) { + fake.setRegionSettingsMutex.Lock() + fake.setRegionSettingsArgsForCall = append(fake.setRegionSettingsArgsForCall, struct { + arg1 *livekit.RegionSettings + }{arg1}) + stub := fake.SetRegionSettingsStub + fake.recordInvocation("SetRegionSettings", []interface{}{arg1}) + fake.setRegionSettingsMutex.Unlock() + if stub != nil { + fake.SetRegionSettingsStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SetRegionSettingsCallCount() int { + fake.setRegionSettingsMutex.RLock() + defer fake.setRegionSettingsMutex.RUnlock() + return len(fake.setRegionSettingsArgsForCall) +} + +func (fake *FakeLocalParticipant) SetRegionSettingsCalls(stub func(*livekit.RegionSettings)) { + fake.setRegionSettingsMutex.Lock() + defer fake.setRegionSettingsMutex.Unlock() + fake.SetRegionSettingsStub = stub +} + +func (fake *FakeLocalParticipant) SetRegionSettingsArgsForCall(i int) *livekit.RegionSettings { + fake.setRegionSettingsMutex.RLock() + defer fake.setRegionSettingsMutex.RUnlock() + argsForCall := fake.setRegionSettingsArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) SetResponseSink(arg1 routing.MessageSink) { fake.setResponseSinkMutex.Lock() fake.setResponseSinkArgsForCall = append(fake.setResponseSinkArgsForCall, struct { @@ -6343,6 +6380,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.setNameMutex.RUnlock() fake.setPermissionMutex.RLock() defer fake.setPermissionMutex.RUnlock() + fake.setRegionSettingsMutex.RLock() + defer fake.setRegionSettingsMutex.RUnlock() fake.setResponseSinkMutex.RLock() defer fake.setResponseSinkMutex.RUnlock() fake.setSignalSourceValidMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 51ab919a0..abfd29331 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -276,12 +276,23 @@ func (r *RoomManager) StartSession( "participant", pi.Identity, "reason", pi.ReconnectReason, ) + + var leave *livekit.LeaveRequest + pv := types.ProtocolVersion(pi.Client.Protocol) + if pv.SupportsRegionsInLeaveRequest() { + leave = &livekit.LeaveRequest{ + Reason: livekit.DisconnectReason_STATE_MISMATCH, + Action: livekit.LeaveRequest_RECONNECT, + } + } else { + leave = &livekit.LeaveRequest{ + CanReconnect: true, + Reason: livekit.DisconnectReason_STATE_MISMATCH, + } + } _ = responseSink.WriteMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Leave{ - Leave: &livekit.LeaveRequest{ - CanReconnect: true, - Reason: livekit.DisconnectReason_STATE_MISMATCH, - }, + Leave: leave, }, }) return errors.New("could not restart closed participant") @@ -321,12 +332,22 @@ func (r *RoomManager) StartSession( } else if pi.Reconnect { // send leave request if participant is trying to reconnect without keep subscribe state // but missing from the room + var leave *livekit.LeaveRequest + pv := types.ProtocolVersion(pi.Client.Protocol) + if pv.SupportsRegionsInLeaveRequest() { + leave = &livekit.LeaveRequest{ + Reason: livekit.DisconnectReason_STATE_MISMATCH, + Action: livekit.LeaveRequest_RECONNECT, + } + } else { + leave = &livekit.LeaveRequest{ + CanReconnect: true, + Reason: livekit.DisconnectReason_STATE_MISMATCH, + } + } _ = responseSink.WriteMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Leave{ - Leave: &livekit.LeaveRequest{ - CanReconnect: true, - Reason: livekit.DisconnectReason_STATE_MISMATCH, - }, + Leave: leave, }, }) return errors.New("could not restart participant") diff --git a/pkg/service/servicefakes/fake_sipstore.go b/pkg/service/servicefakes/fake_sipstore.go index 069eff951..fe88a5359 100644 --- a/pkg/service/servicefakes/fake_sipstore.go +++ b/pkg/service/servicefakes/fake_sipstore.go @@ -22,18 +22,6 @@ type FakeSIPStore struct { deleteSIPDispatchRuleReturnsOnCall map[int]struct { result1 error } - DeleteSIPParticipantStub func(context.Context, *livekit.SIPParticipantInfo) error - deleteSIPParticipantMutex sync.RWMutex - deleteSIPParticipantArgsForCall []struct { - arg1 context.Context - arg2 *livekit.SIPParticipantInfo - } - deleteSIPParticipantReturns struct { - result1 error - } - deleteSIPParticipantReturnsOnCall map[int]struct { - result1 error - } DeleteSIPTrunkStub func(context.Context, *livekit.SIPTrunkInfo) error deleteSIPTrunkMutex sync.RWMutex deleteSIPTrunkArgsForCall []struct { @@ -59,19 +47,6 @@ type FakeSIPStore struct { result1 []*livekit.SIPDispatchRuleInfo result2 error } - ListSIPParticipantStub func(context.Context) ([]*livekit.SIPParticipantInfo, error) - listSIPParticipantMutex sync.RWMutex - listSIPParticipantArgsForCall []struct { - arg1 context.Context - } - listSIPParticipantReturns struct { - result1 []*livekit.SIPParticipantInfo - result2 error - } - listSIPParticipantReturnsOnCall map[int]struct { - result1 []*livekit.SIPParticipantInfo - result2 error - } ListSIPTrunkStub func(context.Context) ([]*livekit.SIPTrunkInfo, error) listSIPTrunkMutex sync.RWMutex listSIPTrunkArgsForCall []struct { @@ -99,20 +74,6 @@ type FakeSIPStore struct { result1 *livekit.SIPDispatchRuleInfo result2 error } - LoadSIPParticipantStub func(context.Context, string) (*livekit.SIPParticipantInfo, error) - loadSIPParticipantMutex sync.RWMutex - loadSIPParticipantArgsForCall []struct { - arg1 context.Context - arg2 string - } - loadSIPParticipantReturns struct { - result1 *livekit.SIPParticipantInfo - result2 error - } - loadSIPParticipantReturnsOnCall map[int]struct { - result1 *livekit.SIPParticipantInfo - result2 error - } LoadSIPTrunkStub func(context.Context, string) (*livekit.SIPTrunkInfo, error) loadSIPTrunkMutex sync.RWMutex loadSIPTrunkArgsForCall []struct { @@ -139,18 +100,6 @@ type FakeSIPStore struct { storeSIPDispatchRuleReturnsOnCall map[int]struct { result1 error } - StoreSIPParticipantStub func(context.Context, *livekit.SIPParticipantInfo) error - storeSIPParticipantMutex sync.RWMutex - storeSIPParticipantArgsForCall []struct { - arg1 context.Context - arg2 *livekit.SIPParticipantInfo - } - storeSIPParticipantReturns struct { - result1 error - } - storeSIPParticipantReturnsOnCall map[int]struct { - result1 error - } StoreSIPTrunkStub func(context.Context, *livekit.SIPTrunkInfo) error storeSIPTrunkMutex sync.RWMutex storeSIPTrunkArgsForCall []struct { @@ -229,68 +178,6 @@ func (fake *FakeSIPStore) DeleteSIPDispatchRuleReturnsOnCall(i int, result1 erro }{result1} } -func (fake *FakeSIPStore) DeleteSIPParticipant(arg1 context.Context, arg2 *livekit.SIPParticipantInfo) error { - fake.deleteSIPParticipantMutex.Lock() - ret, specificReturn := fake.deleteSIPParticipantReturnsOnCall[len(fake.deleteSIPParticipantArgsForCall)] - fake.deleteSIPParticipantArgsForCall = append(fake.deleteSIPParticipantArgsForCall, struct { - arg1 context.Context - arg2 *livekit.SIPParticipantInfo - }{arg1, arg2}) - stub := fake.DeleteSIPParticipantStub - fakeReturns := fake.deleteSIPParticipantReturns - fake.recordInvocation("DeleteSIPParticipant", []interface{}{arg1, arg2}) - fake.deleteSIPParticipantMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeSIPStore) DeleteSIPParticipantCallCount() int { - fake.deleteSIPParticipantMutex.RLock() - defer fake.deleteSIPParticipantMutex.RUnlock() - return len(fake.deleteSIPParticipantArgsForCall) -} - -func (fake *FakeSIPStore) DeleteSIPParticipantCalls(stub func(context.Context, *livekit.SIPParticipantInfo) error) { - fake.deleteSIPParticipantMutex.Lock() - defer fake.deleteSIPParticipantMutex.Unlock() - fake.DeleteSIPParticipantStub = stub -} - -func (fake *FakeSIPStore) DeleteSIPParticipantArgsForCall(i int) (context.Context, *livekit.SIPParticipantInfo) { - fake.deleteSIPParticipantMutex.RLock() - defer fake.deleteSIPParticipantMutex.RUnlock() - argsForCall := fake.deleteSIPParticipantArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeSIPStore) DeleteSIPParticipantReturns(result1 error) { - fake.deleteSIPParticipantMutex.Lock() - defer fake.deleteSIPParticipantMutex.Unlock() - fake.DeleteSIPParticipantStub = nil - fake.deleteSIPParticipantReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeSIPStore) DeleteSIPParticipantReturnsOnCall(i int, result1 error) { - fake.deleteSIPParticipantMutex.Lock() - defer fake.deleteSIPParticipantMutex.Unlock() - fake.DeleteSIPParticipantStub = nil - if fake.deleteSIPParticipantReturnsOnCall == nil { - fake.deleteSIPParticipantReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.deleteSIPParticipantReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeSIPStore) DeleteSIPTrunk(arg1 context.Context, arg2 *livekit.SIPTrunkInfo) error { fake.deleteSIPTrunkMutex.Lock() ret, specificReturn := fake.deleteSIPTrunkReturnsOnCall[len(fake.deleteSIPTrunkArgsForCall)] @@ -417,70 +304,6 @@ func (fake *FakeSIPStore) ListSIPDispatchRuleReturnsOnCall(i int, result1 []*liv }{result1, result2} } -func (fake *FakeSIPStore) ListSIPParticipant(arg1 context.Context) ([]*livekit.SIPParticipantInfo, error) { - fake.listSIPParticipantMutex.Lock() - ret, specificReturn := fake.listSIPParticipantReturnsOnCall[len(fake.listSIPParticipantArgsForCall)] - fake.listSIPParticipantArgsForCall = append(fake.listSIPParticipantArgsForCall, struct { - arg1 context.Context - }{arg1}) - stub := fake.ListSIPParticipantStub - fakeReturns := fake.listSIPParticipantReturns - fake.recordInvocation("ListSIPParticipant", []interface{}{arg1}) - fake.listSIPParticipantMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeSIPStore) ListSIPParticipantCallCount() int { - fake.listSIPParticipantMutex.RLock() - defer fake.listSIPParticipantMutex.RUnlock() - return len(fake.listSIPParticipantArgsForCall) -} - -func (fake *FakeSIPStore) ListSIPParticipantCalls(stub func(context.Context) ([]*livekit.SIPParticipantInfo, error)) { - fake.listSIPParticipantMutex.Lock() - defer fake.listSIPParticipantMutex.Unlock() - fake.ListSIPParticipantStub = stub -} - -func (fake *FakeSIPStore) ListSIPParticipantArgsForCall(i int) context.Context { - fake.listSIPParticipantMutex.RLock() - defer fake.listSIPParticipantMutex.RUnlock() - argsForCall := fake.listSIPParticipantArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeSIPStore) ListSIPParticipantReturns(result1 []*livekit.SIPParticipantInfo, result2 error) { - fake.listSIPParticipantMutex.Lock() - defer fake.listSIPParticipantMutex.Unlock() - fake.ListSIPParticipantStub = nil - fake.listSIPParticipantReturns = struct { - result1 []*livekit.SIPParticipantInfo - result2 error - }{result1, result2} -} - -func (fake *FakeSIPStore) ListSIPParticipantReturnsOnCall(i int, result1 []*livekit.SIPParticipantInfo, result2 error) { - fake.listSIPParticipantMutex.Lock() - defer fake.listSIPParticipantMutex.Unlock() - fake.ListSIPParticipantStub = nil - if fake.listSIPParticipantReturnsOnCall == nil { - fake.listSIPParticipantReturnsOnCall = make(map[int]struct { - result1 []*livekit.SIPParticipantInfo - result2 error - }) - } - fake.listSIPParticipantReturnsOnCall[i] = struct { - result1 []*livekit.SIPParticipantInfo - result2 error - }{result1, result2} -} - func (fake *FakeSIPStore) ListSIPTrunk(arg1 context.Context) ([]*livekit.SIPTrunkInfo, error) { fake.listSIPTrunkMutex.Lock() ret, specificReturn := fake.listSIPTrunkReturnsOnCall[len(fake.listSIPTrunkArgsForCall)] @@ -610,71 +433,6 @@ func (fake *FakeSIPStore) LoadSIPDispatchRuleReturnsOnCall(i int, result1 *livek }{result1, result2} } -func (fake *FakeSIPStore) LoadSIPParticipant(arg1 context.Context, arg2 string) (*livekit.SIPParticipantInfo, error) { - fake.loadSIPParticipantMutex.Lock() - ret, specificReturn := fake.loadSIPParticipantReturnsOnCall[len(fake.loadSIPParticipantArgsForCall)] - fake.loadSIPParticipantArgsForCall = append(fake.loadSIPParticipantArgsForCall, struct { - arg1 context.Context - arg2 string - }{arg1, arg2}) - stub := fake.LoadSIPParticipantStub - fakeReturns := fake.loadSIPParticipantReturns - fake.recordInvocation("LoadSIPParticipant", []interface{}{arg1, arg2}) - fake.loadSIPParticipantMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeSIPStore) LoadSIPParticipantCallCount() int { - fake.loadSIPParticipantMutex.RLock() - defer fake.loadSIPParticipantMutex.RUnlock() - return len(fake.loadSIPParticipantArgsForCall) -} - -func (fake *FakeSIPStore) LoadSIPParticipantCalls(stub func(context.Context, string) (*livekit.SIPParticipantInfo, error)) { - fake.loadSIPParticipantMutex.Lock() - defer fake.loadSIPParticipantMutex.Unlock() - fake.LoadSIPParticipantStub = stub -} - -func (fake *FakeSIPStore) LoadSIPParticipantArgsForCall(i int) (context.Context, string) { - fake.loadSIPParticipantMutex.RLock() - defer fake.loadSIPParticipantMutex.RUnlock() - argsForCall := fake.loadSIPParticipantArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeSIPStore) LoadSIPParticipantReturns(result1 *livekit.SIPParticipantInfo, result2 error) { - fake.loadSIPParticipantMutex.Lock() - defer fake.loadSIPParticipantMutex.Unlock() - fake.LoadSIPParticipantStub = nil - fake.loadSIPParticipantReturns = struct { - result1 *livekit.SIPParticipantInfo - result2 error - }{result1, result2} -} - -func (fake *FakeSIPStore) LoadSIPParticipantReturnsOnCall(i int, result1 *livekit.SIPParticipantInfo, result2 error) { - fake.loadSIPParticipantMutex.Lock() - defer fake.loadSIPParticipantMutex.Unlock() - fake.LoadSIPParticipantStub = nil - if fake.loadSIPParticipantReturnsOnCall == nil { - fake.loadSIPParticipantReturnsOnCall = make(map[int]struct { - result1 *livekit.SIPParticipantInfo - result2 error - }) - } - fake.loadSIPParticipantReturnsOnCall[i] = struct { - result1 *livekit.SIPParticipantInfo - result2 error - }{result1, result2} -} - func (fake *FakeSIPStore) LoadSIPTrunk(arg1 context.Context, arg2 string) (*livekit.SIPTrunkInfo, error) { fake.loadSIPTrunkMutex.Lock() ret, specificReturn := fake.loadSIPTrunkReturnsOnCall[len(fake.loadSIPTrunkArgsForCall)] @@ -802,68 +560,6 @@ func (fake *FakeSIPStore) StoreSIPDispatchRuleReturnsOnCall(i int, result1 error }{result1} } -func (fake *FakeSIPStore) StoreSIPParticipant(arg1 context.Context, arg2 *livekit.SIPParticipantInfo) error { - fake.storeSIPParticipantMutex.Lock() - ret, specificReturn := fake.storeSIPParticipantReturnsOnCall[len(fake.storeSIPParticipantArgsForCall)] - fake.storeSIPParticipantArgsForCall = append(fake.storeSIPParticipantArgsForCall, struct { - arg1 context.Context - arg2 *livekit.SIPParticipantInfo - }{arg1, arg2}) - stub := fake.StoreSIPParticipantStub - fakeReturns := fake.storeSIPParticipantReturns - fake.recordInvocation("StoreSIPParticipant", []interface{}{arg1, arg2}) - fake.storeSIPParticipantMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeSIPStore) StoreSIPParticipantCallCount() int { - fake.storeSIPParticipantMutex.RLock() - defer fake.storeSIPParticipantMutex.RUnlock() - return len(fake.storeSIPParticipantArgsForCall) -} - -func (fake *FakeSIPStore) StoreSIPParticipantCalls(stub func(context.Context, *livekit.SIPParticipantInfo) error) { - fake.storeSIPParticipantMutex.Lock() - defer fake.storeSIPParticipantMutex.Unlock() - fake.StoreSIPParticipantStub = stub -} - -func (fake *FakeSIPStore) StoreSIPParticipantArgsForCall(i int) (context.Context, *livekit.SIPParticipantInfo) { - fake.storeSIPParticipantMutex.RLock() - defer fake.storeSIPParticipantMutex.RUnlock() - argsForCall := fake.storeSIPParticipantArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeSIPStore) StoreSIPParticipantReturns(result1 error) { - fake.storeSIPParticipantMutex.Lock() - defer fake.storeSIPParticipantMutex.Unlock() - fake.StoreSIPParticipantStub = nil - fake.storeSIPParticipantReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeSIPStore) StoreSIPParticipantReturnsOnCall(i int, result1 error) { - fake.storeSIPParticipantMutex.Lock() - defer fake.storeSIPParticipantMutex.Unlock() - fake.StoreSIPParticipantStub = nil - if fake.storeSIPParticipantReturnsOnCall == nil { - fake.storeSIPParticipantReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.storeSIPParticipantReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeSIPStore) StoreSIPTrunk(arg1 context.Context, arg2 *livekit.SIPTrunkInfo) error { fake.storeSIPTrunkMutex.Lock() ret, specificReturn := fake.storeSIPTrunkReturnsOnCall[len(fake.storeSIPTrunkArgsForCall)] @@ -931,26 +627,18 @@ func (fake *FakeSIPStore) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.deleteSIPDispatchRuleMutex.RLock() defer fake.deleteSIPDispatchRuleMutex.RUnlock() - fake.deleteSIPParticipantMutex.RLock() - defer fake.deleteSIPParticipantMutex.RUnlock() fake.deleteSIPTrunkMutex.RLock() defer fake.deleteSIPTrunkMutex.RUnlock() fake.listSIPDispatchRuleMutex.RLock() defer fake.listSIPDispatchRuleMutex.RUnlock() - fake.listSIPParticipantMutex.RLock() - defer fake.listSIPParticipantMutex.RUnlock() fake.listSIPTrunkMutex.RLock() defer fake.listSIPTrunkMutex.RUnlock() fake.loadSIPDispatchRuleMutex.RLock() defer fake.loadSIPDispatchRuleMutex.RUnlock() - fake.loadSIPParticipantMutex.RLock() - defer fake.loadSIPParticipantMutex.RUnlock() fake.loadSIPTrunkMutex.RLock() defer fake.loadSIPTrunkMutex.RUnlock() fake.storeSIPDispatchRuleMutex.RLock() defer fake.storeSIPDispatchRuleMutex.RUnlock() - fake.storeSIPParticipantMutex.RLock() - defer fake.storeSIPParticipantMutex.RUnlock() fake.storeSIPTrunkMutex.RLock() defer fake.storeSIPTrunkMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/test/client/client.go b/test/client/client.go index 70b135a6d..fbe6954d9 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -114,7 +114,7 @@ type Options struct { } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { - u, err := url.Parse(host + "/rtc?protocol=7") + u, err := url.Parse(host + fmt.Sprintf("/rtc?protocol=%d", types.CurrentProtocol)) if err != nil { return nil, err } @@ -493,7 +493,10 @@ func (c *RTCClient) Stop() { logger.Infow("stopping client", "ID", c.ID()) _ = c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_Leave{ - Leave: &livekit.LeaveRequest{}, + Leave: &livekit.LeaveRequest{ + Reason: livekit.DisconnectReason_CLIENT_INITIATED, + Action: livekit.LeaveRequest_DISCONNECT, + }, }, }) c.publisherFullyEstablished.Store(false) From d5b3bbac61d41829e14184d3b8ea55720185531d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:47:15 -0800 Subject: [PATCH 090/114] Update module github.com/livekit/protocol to v1.9.7 (#2337) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bcec95e46..784eba4eb 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557 + github.com/livekit/protocol v1.9.7 github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index d8ae78ab4..f927c8661 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557 h1:lz11rtmZouO9C7RcDJpg5tWERUEAMLTYjtHX/mRyras= -github.com/livekit/protocol v1.9.6-0.20240125083757-31b03e690557/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= +github.com/livekit/protocol v1.9.7 h1:5pYAMS/rzOStpIfRGnhXETPH/NyoFJtbV7FW4NHxg7o= +github.com/livekit/protocol v1.9.7/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 025eb1164ca3d51ec78a75ed0a5fabc3ac99fb74 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 25 Jan 2024 15:48:12 -0800 Subject: [PATCH 091/114] retry signal stream start (#2410) --- go.mod | 3 ++- go.sum | 6 ++++-- pkg/routing/signal.go | 21 +++++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 784eba4eb..a10a85928 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/livekit/livekit-server go 1.20 require ( + github.com/avast/retry-go/v4 v4.5.1 github.com/bep/debounce v1.2.1 github.com/d5/tengo/v2 v2.16.1 github.com/dustin/go-humanize v1.0.1 @@ -101,7 +102,7 @@ require ( golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/grpc v1.61.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index f927c8661..678a999e7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= +github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -421,8 +423,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index e7181038d..fbad8da97 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "github.com/avast/retry-go/v4" "go.uber.org/atomic" "google.golang.org/protobuf/proto" @@ -99,13 +100,19 @@ func (r *signalClient) StartParticipantSignal( l.Debugw("starting signal connection") - stream, err := r.client.RelaySignal(ctx, nodeID) + var stream psrpc.ClientStream[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse] + err = r.retry(ctx, func() (err error) { + stream, err = r.client.RelaySignal(ctx, nodeID) + return + }) if err != nil { prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } - err = stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) + err = r.retry(ctx, func() error { + return stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) + }) if err != nil { stream.Close(err) prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) @@ -141,6 +148,16 @@ func (r *signalClient) StartParticipantSignal( return connectionID, sink, resChan, nil } +func (r *signalClient) retry(ctx context.Context, fn retry.RetryableFunc) error { + return retry.Do( + fn, + retry.Context(ctx), + retry.Delay(r.config.MinRetryInterval), + retry.MaxDelay(r.config.MaxRetryInterval), + retry.DelayType(retry.BackOffDelay), + ) +} + type signalRequestMessageWriter struct{} func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalRequest { From 0ebb861bdf6ae9d70d74a2dc23c0573bb4c1b44a Mon Sep 17 00:00:00 2001 From: aoife cassidy Date: Fri, 26 Jan 2024 01:49:36 +0200 Subject: [PATCH 092/114] Replace /bin/bash with env call (#2409) Fixes build on FreeBSD, which uses /usr/local/bin/bash, and also is just a more idiomatic way to handle shells. --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 3109ddc38..7511dba3b 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2023 LiveKit, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); From 995fddbaf90b5d8b15f869900547525ff43a48ad Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 26 Jan 2024 09:33:35 +0800 Subject: [PATCH 093/114] Add dynamic playout delay if PlayoutDelay enabled in the room (#2403) * Add dynamic playout delay * type for state --- pkg/sfu/downtrack.go | 40 +++++----- pkg/sfu/playoutdelay.go | 138 +++++++++++++++++++++++++++++++++++ pkg/sfu/playoutdelay_test.go | 61 ++++++++++++++++ 3 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 pkg/sfu/playoutdelay.go create mode 100644 pkg/sfu/playoutdelay_test.go diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index a2185063f..22cc6fadc 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -262,8 +262,7 @@ type DownTrack struct { bytesSent atomic.Uint32 bytesRetransmitted atomic.Uint32 - playoutDelayBytes atomic.Value //bytes of marshalled playout delay - playoudDelayAcked atomic.Bool + playoutDelay *PlayoutDelayController pacer pacer.Pacer @@ -318,6 +317,14 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { }) d.deltaStatsSenderSnapshotId = d.rtpStats.NewSenderSnapshotId() + if delay := params.PlayoutDelayLimit; delay.GetEnabled() { + var err error + d.playoutDelay, err = NewPlayoutDelayController(delay.GetMin(), delay.GetMax(), params.Logger, d.rtpStats) + if err != nil { + return nil, err + } + } + d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), @@ -330,23 +337,6 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { } }) - // set initial playout delay to minimum value - if d.params.PlayoutDelayLimit.GetEnabled() { - maxDelay := uint32(rtpextension.PlayoutDelayDefaultMax) - if d.params.PlayoutDelayLimit.GetMax() > 0 { - maxDelay = d.params.PlayoutDelayLimit.GetMax() - } - delay := rtpextension.PlayoutDelayFromValue( - uint16(d.params.PlayoutDelayLimit.GetMin()), - uint16(maxDelay), - ) - b, err := delay.Marshal() - if err == nil { - d.playoutDelayBytes.Store(b) - } else { - d.params.Logger.Errorw("failed to marshal playout delay", err, "playoutDelay", d.params.PlayoutDelayLimit) - } - } if d.kind == webrtc.RTPCodecTypeVideo { go d.maxLayerNotifierWorker() go d.keyFrameRequester() @@ -716,9 +706,9 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { if tp.ddBytes != nil { extensions = []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}} } - if d.playoutDelayExtID != 0 && !d.playoudDelayAcked.Load() { - if val := d.playoutDelayBytes.Load(); val != nil { - extensions = append(extensions, pacer.ExtensionData{ID: uint8(d.playoutDelayExtID), Payload: val.([]byte)}) + if d.playoutDelayExtID != 0 && d.playoutDelay != nil { + if val := d.playoutDelay.GetDelayExtension(hdr.SequenceNumber); val != nil { + extensions = append(extensions, pacer.ExtensionData{ID: uint8(d.playoutDelayExtID), Payload: val}) } } if d.sequencer != nil { @@ -1534,7 +1524,11 @@ func (d *DownTrack) handleRTCP(bytes []byte) { sal.OnRTCPReceiverReport(d, r) } - d.playoudDelayAcked.Store(true) + if d.playoutDelay != nil { + jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) + d.playoutDelay.OnSeqAcked(uint16(r.LastSequenceNumber)) + d.playoutDelay.SetJitter(uint32(jitterMs)) + } } if len(rr.Reports) > 0 { d.listenerLock.RLock() diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go new file mode 100644 index 000000000..eaef62853 --- /dev/null +++ b/pkg/sfu/playoutdelay.go @@ -0,0 +1,138 @@ +package sfu + +import ( + "sync" + "sync/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + "github.com/livekit/protocol/logger" +) + +type PlayoutDelayState int32 + +const ( + PlayoutDelayStateChanged PlayoutDelayState = iota + PlayoutDelaySending + PlayoutDelayAcked + + jitterLowMultiToDelay = 10 + jitterHighMultiToDelay = 15 + jitterHighThreshold = 15 +) + +func (s PlayoutDelayState) String() string { + switch s { + case PlayoutDelayStateChanged: + return "StateChanged" + case PlayoutDelaySending: + return "Sending" + case PlayoutDelayAcked: + return "Acked" + } + return "Unknown" +} + +type PlayoutDelayController struct { + lock sync.Mutex + state atomic.Int32 + minDelay, maxDelay uint32 + currentDelay uint32 + extBytes atomic.Value //[]byte + sendingAtSeq uint16 + logger logger.Logger + rtpStats *buffer.RTPStatsSender + snapshotID uint32 +} + +func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *buffer.RTPStatsSender) (*PlayoutDelayController, error) { + if maxDelay == 0 || maxDelay > rtpextension.PlayoutDelayDefaultMax { + maxDelay = rtpextension.PlayoutDelayDefaultMax + } + c := &PlayoutDelayController{ + currentDelay: minDelay, + minDelay: minDelay, + maxDelay: maxDelay, + logger: logger, + rtpStats: rtpStats, + snapshotID: rtpStats.NewSenderSnapshotId(), + } + return c, c.createExtData() +} + +func (c *PlayoutDelayController) SetJitter(jitter uint32) { + deltaInfo := c.rtpStats.DeltaInfoSender(c.snapshotID) + var nackPercent uint32 + if deltaInfo != nil && deltaInfo.Packets > 0 { + nackPercent = deltaInfo.Nacks * 100 / deltaInfo.Packets + } + + c.lock.Lock() + multi := jitterLowMultiToDelay + if jitter >= jitterHighThreshold { + multi = jitterHighMultiToDelay + } + targetDelay := jitter * uint32(multi) + if nackPercent > 60 { + targetDelay += (nackPercent - 60) * 2 + } + + // increase delay quickly, decrease slowly to make fps more stable + if targetDelay > c.currentDelay { + targetDelay = (targetDelay-c.currentDelay)*3/4 + c.currentDelay + } else { + targetDelay = c.currentDelay - (c.currentDelay-targetDelay)/5 + } + if targetDelay < c.minDelay { + targetDelay = c.minDelay + } + if targetDelay > c.maxDelay { + targetDelay = c.maxDelay + } + if c.currentDelay == targetDelay { + c.lock.Unlock() + return + } + c.currentDelay = targetDelay + c.lock.Unlock() + c.createExtData() +} + +func (c *PlayoutDelayController) OnSeqAcked(seq uint16) { + c.lock.Lock() + defer c.lock.Unlock() + if PlayoutDelayState(c.state.Load()) == PlayoutDelaySending && (seq-c.sendingAtSeq) < 0x8000 { + c.state.Store(int32(PlayoutDelayAcked)) + } +} + +func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte { + switch PlayoutDelayState(c.state.Load()) { + case PlayoutDelayStateChanged: + c.lock.Lock() + c.state.Store(int32(PlayoutDelaySending)) + c.sendingAtSeq = seq + c.lock.Unlock() + return c.extBytes.Load().([]byte) + case PlayoutDelaySending: + return c.extBytes.Load().([]byte) + case PlayoutDelayAcked: + return nil + } + return nil +} + +func (c *PlayoutDelayController) createExtData() error { + delay := rtpextension.PlayoutDelayFromValue( + uint16(c.currentDelay), + uint16(c.maxDelay), + ) + b, err := delay.Marshal() + if err == nil { + c.extBytes.Store(b) + c.state.Store(int32(PlayoutDelayStateChanged)) + } else { + c.logger.Errorw("failed to marshal playout delay", err, "playoutDelay", delay) + } + return err +} diff --git a/pkg/sfu/playoutdelay_test.go b/pkg/sfu/playoutdelay_test.go new file mode 100644 index 000000000..190891034 --- /dev/null +++ b/pkg/sfu/playoutdelay_test.go @@ -0,0 +1,61 @@ +package sfu + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + "github.com/livekit/protocol/logger" +) + +func TestPlayoutDelay(t *testing.T) { + stats := buffer.NewRTPStatsSender(buffer.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}) + c, err := NewPlayoutDelayController(100, 1000, logger.GetLogger(), stats) + require.NoError(t, err) + + ext := c.GetDelayExtension(100) + playoutDelayEqual(t, ext, 100, 1000) + + ext = c.GetDelayExtension(105) + playoutDelayEqual(t, ext, 100, 1000) + + // seq acked before delay changed + c.OnSeqAcked(65534) + ext = c.GetDelayExtension(105) + playoutDelayEqual(t, ext, 100, 1000) + + c.OnSeqAcked(90) + ext = c.GetDelayExtension(105) + playoutDelayEqual(t, ext, 100, 1000) + + // seq acked, no extension sent for new packet + c.OnSeqAcked(103) + ext = c.GetDelayExtension(106) + require.Nil(t, ext) + + // delay on change(can't go below min), no extension sent + c.SetJitter(0) + ext = c.GetDelayExtension(107) + require.Nil(t, ext) + + // delay changed, generate new extension to send + c.SetJitter(50) + ext = c.GetDelayExtension(108) + var delay rtpextension.PlayOutDelay + require.NoError(t, delay.Unmarshal(ext)) + require.Greater(t, delay.Min, uint16(100)) + + // can't go above max + c.SetJitter(10000) + ext = c.GetDelayExtension(109) + playoutDelayEqual(t, ext, 1000, 1000) +} + +func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) { + var delay rtpextension.PlayOutDelay + require.NoError(t, delay.Unmarshal(data)) + require.Equal(t, min, delay.Min) + require.Equal(t, max, delay.Max) +} From 9b4ba2d41dad6f6359377ceab8ed1082c678d672 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 26 Jan 2024 13:32:54 +0800 Subject: [PATCH 094/114] use default max playout delay as chrome (#2411) --- pkg/sfu/playoutdelay.go | 7 +++++-- pkg/sfu/rtpextension/playoutdelay.go | 11 ++++++----- pkg/sfu/rtpextension/playoutdelay_test.go | 8 ++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index eaef62853..2ac26a85b 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -46,8 +46,11 @@ type PlayoutDelayController struct { } func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *buffer.RTPStatsSender) (*PlayoutDelayController, error) { - if maxDelay == 0 || maxDelay > rtpextension.PlayoutDelayDefaultMax { - maxDelay = rtpextension.PlayoutDelayDefaultMax + if maxDelay == 0 && minDelay > 0 { + maxDelay = rtpextension.MaxPlayoutDelayDefault + } + if maxDelay > rtpextension.PlayoutDelayMaxValue { + maxDelay = rtpextension.PlayoutDelayMaxValue } c := &PlayoutDelayController{ currentDelay: minDelay, diff --git a/pkg/sfu/rtpextension/playoutdelay.go b/pkg/sfu/rtpextension/playoutdelay.go index d42322be7..500bb67ea 100644 --- a/pkg/sfu/rtpextension/playoutdelay.go +++ b/pkg/sfu/rtpextension/playoutdelay.go @@ -7,7 +7,8 @@ import ( const ( PlayoutDelayURI = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay" - PlayoutDelayDefaultMax = 4000 // 4s + MaxPlayoutDelayDefault = 10000 // 10s, equal to chrome's default max playout delay + PlayoutDelayMaxValue = 10 * (1<<12 - 1) // max value for playout delay can be represented playoutDelayExtensionSize = 3 ) @@ -28,11 +29,11 @@ type PlayOutDelay struct { } func PlayoutDelayFromValue(min, max uint16) PlayOutDelay { - if min >= (1<<12)*10 { - min = (1<<12 - 1) * 10 + if min > PlayoutDelayMaxValue { + min = PlayoutDelayMaxValue } - if max >= (1<<12)*10 { - max = (1<<12 - 1) * 10 + if max > PlayoutDelayMaxValue { + max = PlayoutDelayMaxValue } return PlayOutDelay{Min: min, Max: max} } diff --git a/pkg/sfu/rtpextension/playoutdelay_test.go b/pkg/sfu/rtpextension/playoutdelay_test.go index a92f4cae5..7262a6475 100644 --- a/pkg/sfu/rtpextension/playoutdelay_test.go +++ b/pkg/sfu/rtpextension/playoutdelay_test.go @@ -32,4 +32,12 @@ func TestPlayoutDelay(t *testing.T) { require.NoError(t, err) require.Equal(t, uint16((1<<12)-1)*10, p5.Min) require.Equal(t, uint16((1<<12)-1)*10, p5.Max) + + p6 := PlayOutDelay{Min: 100, Max: PlayoutDelayMaxValue} + bytes, err := p6.Marshal() + require.NoError(t, err) + p6Unmarshal := PlayOutDelay{} + err = p6Unmarshal.Unmarshal(bytes) + require.NoError(t, err) + require.Equal(t, p6, p6Unmarshal) } From 9eca035738f37482667f3c2a1269d1310d516d93 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 26 Jan 2024 08:14:49 -0800 Subject: [PATCH 095/114] revert signal retry (#2413) --- pkg/routing/signal.go | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index fbad8da97..e7181038d 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -20,7 +20,6 @@ import ( "sync" "time" - "github.com/avast/retry-go/v4" "go.uber.org/atomic" "google.golang.org/protobuf/proto" @@ -100,19 +99,13 @@ func (r *signalClient) StartParticipantSignal( l.Debugw("starting signal connection") - var stream psrpc.ClientStream[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse] - err = r.retry(ctx, func() (err error) { - stream, err = r.client.RelaySignal(ctx, nodeID) - return - }) + stream, err := r.client.RelaySignal(ctx, nodeID) if err != nil { prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } - err = r.retry(ctx, func() error { - return stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) - }) + err = stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) if err != nil { stream.Close(err) prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) @@ -148,16 +141,6 @@ func (r *signalClient) StartParticipantSignal( return connectionID, sink, resChan, nil } -func (r *signalClient) retry(ctx context.Context, fn retry.RetryableFunc) error { - return retry.Do( - fn, - retry.Context(ctx), - retry.Delay(r.config.MinRetryInterval), - retry.MaxDelay(r.config.MaxRetryInterval), - retry.DelayType(retry.BackOffDelay), - ) -} - type signalRequestMessageWriter struct{} func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalRequest { From 654b05638f0a924a30837009bebb8449cfee1671 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 26 Jan 2024 10:39:08 -0800 Subject: [PATCH 096/114] update psrpc (#2414) --- go.mod | 5 ++--- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index a10a85928..03f33a351 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/livekit/livekit-server go 1.20 require ( - github.com/avast/retry-go/v4 v4.5.1 github.com/bep/debounce v1.2.1 github.com/d5/tengo/v2 v2.16.1 github.com/dustin/go-humanize v1.0.1 @@ -20,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f github.com/livekit/protocol v1.9.7 - github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 + github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -71,7 +70,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.5 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect diff --git a/go.sum b/go.sum index 678a999e7..b42e0b644 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= -github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -110,8 +108,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= +github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -130,8 +128,8 @@ github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrw github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= github.com/livekit/protocol v1.9.7 h1:5pYAMS/rzOStpIfRGnhXETPH/NyoFJtbV7FW4NHxg7o= github.com/livekit/protocol v1.9.7/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= -github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9 h1:kXXV/NLVDHZ+Gn7xrR+UPpdwbH48n7WReBjLHAzqzhY= -github.com/livekit/psrpc v0.5.3-0.20231214055026-06ce27a934c9/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= +github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b h1:+860jUlsyW3xI9sp6IdOM7QceOK2rsbZC2P193UI4Kc= +github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= From c2549081c83221174e6556bffa8c15e7d974ce71 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 26 Jan 2024 14:03:46 -0800 Subject: [PATCH 097/114] Allow creating SRT URL pull ingress (#2416) --- pkg/service/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index a8fd81d6c..2acda0064 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -132,7 +132,7 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string if err != nil { return nil, psrpc.NewError(psrpc.InvalidArgument, err) } - if urlObj.Scheme != "http" && urlObj.Scheme != "https" { + if urlObj.Scheme != "http" && urlObj.Scheme != "https" && urlObj.Scheme != "srt" { return nil, ingress.ErrInvalidIngress(fmt.Sprintf("invalid url scheme %s", urlObj.Scheme)) } // Marshall the URL again for sanitization From b71d373f4a1c296c8db936eb645c29d11e0f390f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 Jan 2024 13:48:30 +0530 Subject: [PATCH 098/114] Use Deque in ops queue. (#2418) * Use Seque in ops queue. Standardizing some uses - Change OpsQueue to use Deque so that it can grow/shrink as necessary and need not worry about channel getting full and dropping events. - Change StreamAllocator and TelemetryService to use OpsQueue so that they also need not worry about channel size and overflows. * Address feedback * delete obvious comment * clean up --- pkg/rtc/dynacastmanager.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 40 +++------- pkg/telemetry/telemetryservice.go | 25 +++---- pkg/utils/opsqueue.go | 87 ++++++++++++---------- 4 files changed, 69 insertions(+), 85 deletions(-) diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 8943646fe..1163cb09a 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -59,7 +59,7 @@ func NewDynacastManager(params DynacastManagerParams) *DynacastManager { maxSubscribedQuality: make(map[string]livekit.VideoQuality), committedMaxSubscribedQuality: make(map[string]livekit.VideoQuality), maxSubscribedQualityDebounce: debounce.New(params.DynacastPauseDelay), - qualityNotifyOpQueue: utils.NewOpsQueue(params.Logger, "quality-notify", 100), + qualityNotifyOpQueue: utils.NewOpsQueue("quality-notify", 0, true), } d.qualityNotifyOpQueue.Start() return d diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 364dd0895..6370692b0 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -31,6 +31,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/utils" ) const ( @@ -165,8 +166,7 @@ type StreamAllocator struct { state streamAllocatorState - eventChMu sync.RWMutex - eventCh chan Event + eventsQueue *utils.OpsQueue isStopped atomic.Bool } @@ -180,7 +180,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { }), rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventCh: make(chan Event, 1000), + eventsQueue: utils.NewOpsQueue("stream-allocator", 64, true), } s.probeController = NewProbeController(ProbeControllerParams{ @@ -197,19 +197,18 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { } func (s *StreamAllocator) Start() { - go s.processEvents() + s.eventsQueue.Start() go s.ping() } func (s *StreamAllocator) Stop() { - s.eventChMu.Lock() if s.isStopped.Swap(true) { - s.eventChMu.Unlock() return } - close(s.eventCh) - s.eventChMu.Unlock() + // wait for eventsQueue to be done + <-s.eventsQueue.Stop() + s.probeController.StopProbe() } func (s *StreamAllocator) OnStreamStateChange(f func(update *StreamStateUpdate) error) { @@ -546,30 +545,9 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) } func (s *StreamAllocator) postEvent(event Event) { - s.eventChMu.RLock() - if s.isStopped.Load() { - s.eventChMu.RUnlock() - return - } - - select { - case s.eventCh <- event: - default: - s.params.Logger.Warnw("stream allocator: event queue full", nil, "event", event.String()) - } - s.eventChMu.RUnlock() -} - -func (s *StreamAllocator) processEvents() { - for event := range s.eventCh { - if s.isStopped.Load() { - break - } - + s.eventsQueue.Enqueue(func() { s.handleEvent(&event) - } - - s.probeController.StopProbe() + }) } func (s *StreamAllocator) ping() { diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index 572bba118..fe5e329c2 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -20,6 +20,7 @@ import ( "time" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/webhook" @@ -82,15 +83,15 @@ type TelemetryService interface { } const ( - workerCleanupWait = 3 * time.Minute - jobQueueBufferSize = 10000 + workerCleanupWait = 3 * time.Minute + jobsQueueMinSize = 2048 ) type telemetryService struct { AnalyticsService - notifier webhook.QueuedNotifier - jobsChan chan func() + notifier webhook.QueuedNotifier + jobsQueue *utils.OpsQueue lock sync.RWMutex workers map[livekit.ParticipantID]*StatsWorker @@ -100,11 +101,12 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer t := &telemetryService{ AnalyticsService: analytics, - notifier: notifier, - jobsChan: make(chan func(), jobQueueBufferSize), - workers: make(map[livekit.ParticipantID]*StatsWorker), + notifier: notifier, + jobsQueue: utils.NewOpsQueue("telemetry", jobsQueueMinSize, true), + workers: make(map[livekit.ParticipantID]*StatsWorker), } + t.jobsQueue.Start() go t.run() return t @@ -132,19 +134,12 @@ func (t *telemetryService) run() { t.FlushStats() case <-cleanupTicker.C: t.cleanupWorkers() - case op := <-t.jobsChan: - op() } } } func (t *telemetryService) enqueue(op func()) { - select { - case t.jobsChan <- op: - // success - default: - logger.Warnw("telemetry queue full", nil) - } + t.jobsQueue.Enqueue(op) } func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) { diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 3e992461c..01f9a12ff 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -15,33 +15,34 @@ package utils import ( + "math/bits" "sync" - "github.com/livekit/protocol/logger" + "github.com/gammazero/deque" + "github.com/livekit/protocol/utils" ) type OpsQueue struct { - logger logger.Logger - name string - size int + name string + flushOnStop bool - lock sync.RWMutex - ops chan func() + lock sync.Mutex + ops deque.Deque[func()] + wake chan struct{} isStarted bool + doneChan chan struct{} isStopped bool } -func NewOpsQueue(logger logger.Logger, name string, size int) *OpsQueue { - return &OpsQueue{ - logger: logger, - name: name, - size: size, - ops: make(chan func(), size), +func NewOpsQueue(name string, minSize uint, flushOnStop bool) *OpsQueue { + oq := &OpsQueue{ + name: name, + flushOnStop: flushOnStop, + wake: make(chan struct{}, 1), + doneChan: make(chan struct{}), } -} - -func (oq *OpsQueue) SetLogger(logger logger.Logger) { - oq.logger = logger + oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(minSize-1)), 16))) + return oq } func (oq *OpsQueue) Start() { @@ -57,42 +58,52 @@ func (oq *OpsQueue) Start() { go oq.process() } -func (oq *OpsQueue) Stop() { +func (oq *OpsQueue) Stop() <-chan struct{} { oq.lock.Lock() if oq.isStopped { oq.lock.Unlock() - return + return oq.doneChan } oq.isStopped = true - close(oq.ops) + close(oq.wake) oq.lock.Unlock() -} - -func (oq *OpsQueue) IsStarted() bool { - oq.lock.RLock() - defer oq.lock.RUnlock() - - return oq.isStarted + return oq.doneChan } func (oq *OpsQueue) Enqueue(op func()) { - oq.lock.RLock() - if oq.isStopped { - oq.lock.RUnlock() - return - } + oq.lock.Lock() + defer oq.lock.Unlock() - select { - case oq.ops <- op: - default: - oq.logger.Errorw("ops queue full", nil, "name", oq.name, "size", oq.size) + oq.ops.PushBack(op) + if oq.ops.Len() == 1 && !oq.isStopped { + select { + case oq.wake <- struct{}{}: + default: + } } - oq.lock.RUnlock() } func (oq *OpsQueue) process() { - for op := range oq.ops { - op() + defer close(oq.doneChan) + + for { + <-oq.wake + for { + oq.lock.Lock() + if oq.isStopped && (!oq.flushOnStop || oq.ops.Len() == 0) { + oq.lock.Unlock() + return + } + + if oq.ops.Len() == 0 { + oq.lock.Unlock() + break + } + op := oq.ops.PopFront() + oq.lock.Unlock() + + op() + } } } From 38352b61259c1704aae5159c0bd78c2c659575c8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 Jan 2024 14:28:29 +0530 Subject: [PATCH 099/114] Change transport queue. (#2419) From a channel to OpsQueue. Have seen extreme cases (with a ton of candidates) overflowing the channel. --- pkg/rtc/transport.go | 42 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index d449cf3d9..953b14518 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -41,6 +41,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/livekit-server/pkg/utils" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -213,8 +214,7 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool - eventChMu sync.RWMutex - eventCh chan event + eventsQueue *utils.OpsQueue // the following should be accessed only in event processing go routine cacheLocalCandidates bool @@ -381,7 +381,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params: params, debouncedNegotiate: debounce.New(negotiationFrequency), negotiationState: NegotiationStateNone, - eventCh: make(chan event, 100), + eventsQueue: utils.NewOpsQueue("transport", 64, false), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, connectionDetails: types.NewICEConnectionDetails(params.Transport, params.Logger), @@ -399,7 +399,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { return nil, err } - go t.processEvents() + t.eventsQueue.Start() return t, nil } @@ -938,14 +938,12 @@ func (t *PCTransport) SendDataPacket(dp *livekit.DataPacket, data []byte) error } func (t *PCTransport) Close() { - t.eventChMu.Lock() if t.isClosed.Swap(true) { - t.eventChMu.Unlock() return } - close(t.eventCh) - t.eventChMu.Unlock() + <-t.eventsQueue.Stop() + t.clearSignalStateCheckTimer() if t.streamAllocator != nil { t.streamAllocator.Stop() @@ -1383,27 +1381,7 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map } func (t *PCTransport) postEvent(event event) { - t.eventChMu.RLock() - if t.isClosed.Load() { - t.eventChMu.RUnlock() - return - } - - select { - case t.eventCh <- event: - default: - t.params.Logger.Warnw("event queue full", nil, "event", event.String()) - } - t.eventChMu.RUnlock() -} - -func (t *PCTransport) processEvents() { - for event := range t.eventCh { - if t.isClosed.Load() { - // just drain the channel without processing events - continue - } - + t.eventsQueue.Enqueue(func() { err := t.handleEvent(&event) if err != nil { if !t.isClosed.Load() { @@ -1412,12 +1390,8 @@ func (t *PCTransport) processEvents() { onNegotiationFailed() } } - break } - } - - t.clearSignalStateCheckTimer() - t.params.Logger.Debugw("leaving events processor") + }) } func (t *PCTransport) handleEvent(e *event) error { From bcf9fe3f0f31a68a76517d63e560a9dbcc174f15 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 Jan 2024 22:10:35 +0530 Subject: [PATCH 100/114] Use a participant worker queue in room. (#2420) * Use a participant worker queue in room. Removes selectively needing to call things in goroutine from participant. Also, a bit of drive-by clean up. * spelling * prevent race * don't need to remove in goroutine as it is already running in the worker * worker will get cleaned up in state change callback * create participant worker only if not created already * ref count participant worker * maintain participant list * clean up oldState --- pkg/rtc/mediatracksubscriptions.go | 2 +- pkg/rtc/participant.go | 120 ++++------ pkg/rtc/room.go | 226 ++++++++++++------ pkg/rtc/room_test.go | 14 +- pkg/rtc/subscriptionmanager.go | 2 +- pkg/rtc/testutils.go | 4 + pkg/rtc/transport.go | 2 - pkg/rtc/transportmanager.go | 6 +- pkg/rtc/types/interfaces.go | 6 +- .../typesfakes/fake_local_participant.go | 124 ++++------ pkg/rtc/types/typesfakes/fake_participant.go | 30 --- pkg/rtc/uptrackmanager.go | 3 - 12 files changed, 265 insertions(+), 274 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 0a619bb99..e074c2577 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -196,7 +196,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * }) downTrack.AddReceiverReportListener(func(dt *sfu.DownTrack, report *rtcp.ReceiverReport) { - sub.OnReceiverReport(dt, report) + sub.HandleReceiverReport(dt, report) }) var transceiver *webrtc.RTPTransceiver diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ff6758d80..0aa98e198 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -191,7 +191,6 @@ type ParticipantImpl struct { lastRTT uint32 lock utils.RWMutex - once sync.Once dirty atomic.Bool version atomic.Uint32 @@ -201,7 +200,7 @@ type ParticipantImpl struct { onTrackPublished func(types.LocalParticipant, types.MediaTrack) onTrackUpdated func(types.LocalParticipant, types.MediaTrack) onTrackUnpublished func(types.LocalParticipant, types.MediaTrack) - onStateChange func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) + onStateChange func(p types.LocalParticipant, state livekit.ParticipantInfo_State) onMigrateStateChange func(p types.LocalParticipant, migrateState types.MigrateState) onParticipantUpdate func(types.LocalParticipant) onDataPacket func(types.LocalParticipant, *livekit.DataPacket) @@ -525,18 +524,36 @@ func (p *ParticipantImpl) OnTrackPublished(callback func(types.LocalParticipant, p.lock.Unlock() } +func (p *ParticipantImpl) getOnTrackPublished() func(types.LocalParticipant, types.MediaTrack) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.onTrackPublished +} + func (p *ParticipantImpl) OnTrackUnpublished(callback func(types.LocalParticipant, types.MediaTrack)) { p.lock.Lock() p.onTrackUnpublished = callback p.lock.Unlock() } -func (p *ParticipantImpl) OnStateChange(callback func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State)) { +func (p *ParticipantImpl) getOnTrackUnpublished() func(types.LocalParticipant, types.MediaTrack) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.onTrackUnpublished +} + +func (p *ParticipantImpl) OnStateChange(callback func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) { p.lock.Lock() p.onStateChange = callback p.lock.Unlock() } +func (p *ParticipantImpl) getOnStateChange() func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.onStateChange +} + func (p *ParticipantImpl) OnMigrateStateChange(callback func(p types.LocalParticipant, state types.MigrateState)) { p.lock.Lock() p.onMigrateStateChange = callback @@ -546,7 +563,6 @@ func (p *ParticipantImpl) OnMigrateStateChange(callback func(p types.LocalPartic func (p *ParticipantImpl) getOnMigrateStateChange() func(p types.LocalParticipant, state types.MigrateState) { p.lock.RLock() defer p.lock.RUnlock() - return p.onMigrateStateChange } @@ -556,6 +572,12 @@ func (p *ParticipantImpl) OnTrackUpdated(callback func(types.LocalParticipant, t p.lock.Unlock() } +func (p *ParticipantImpl) getOnTrackUpdated() func(types.LocalParticipant, types.MediaTrack) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.onTrackUpdated +} + func (p *ParticipantImpl) OnParticipantUpdate(callback func(types.LocalParticipant)) { p.lock.Lock() p.onParticipantUpdate = callback @@ -667,13 +689,9 @@ func (p *ParticipantImpl) handleMigrateTracks() { } p.pendingTracksLock.Unlock() - // launch callbacks in goroutine since they could block. - // callbacks handle webhooks as well as db persistence - go func() { - for _, t := range addedTracks { - p.handleTrackPublished(t) - } - }() + for _, t := range addedTracks { + p.handleTrackPublished(t) + } } func (p *ParticipantImpl) removePendingMigratedTrack(mt *MediaTrack) { @@ -728,12 +746,6 @@ func (p *ParticipantImpl) SetMigrateInfo( p.TransportManager.SetMigrateInfo(previousOffer, previousAnswer, dataChannels) } -func (p *ParticipantImpl) Start() { - p.once.Do(func() { - p.UpTrackManager.Start() - }) -} - func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseReason, isExpectedToResume bool) error { if p.isClosed.Swap(true) { // already closed @@ -916,7 +928,7 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { } if onMigrateStateChange := p.getOnMigrateStateChange(); onMigrateStateChange != nil { - go onMigrateStateChange(p, s) + onMigrateStateChange(p, s) } } @@ -1146,7 +1158,6 @@ func (p *ParticipantImpl) setupTransportManager() error { SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), Config: p.params.Config, ProtocolVersion: p.params.ProtocolVersion, - Telemetry: p.params.Telemetry, CongestionControlConfig: p.params.CongestionControlConfig, EnabledPublishCodecs: p.enabledPublishCodecs, EnabledSubscribeCodecs: p.enabledSubscribeCodecs, @@ -1225,12 +1236,8 @@ func (p *ParticipantImpl) setupUpTrackManager() { }) p.UpTrackManager.OnPublishedTrackUpdated(func(track types.MediaTrack) { - p.lock.RLock() - onTrackUpdated := p.onTrackUpdated - p.lock.RUnlock() - p.dirty.Store(true) - if onTrackUpdated != nil { + if onTrackUpdated := p.getOnTrackUpdated(); onTrackUpdated != nil { onTrackUpdated(p, track) } }) @@ -1261,26 +1268,16 @@ func (p *ParticipantImpl) setupParticipantTrafficLoad() { } func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { - oldState := p.State() - if !(p.state.Swap(state) != state) { + oldState := p.state.Swap(state).(livekit.ParticipantInfo_State) + if oldState == state { return } p.params.Logger.Debugw("updating participant state", "state", state.String()) p.dirty.Store(true) - p.lock.RLock() - onStateChange := p.onStateChange - p.lock.RUnlock() - if onStateChange != nil { - go func() { - defer func() { - if r := Recover(p.GetLogger()); r != nil { - os.Exit(1) - } - }() - onStateChange(p, oldState) - }() + if onStateChange := p.getOnStateChange(); onStateChange != nil { + onStateChange(p, state) } } @@ -1367,10 +1364,7 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w ) if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { - p.lock.RLock() - onTrackUpdated := p.onTrackUpdated - p.lock.RUnlock() - if onTrackUpdated != nil { + if onTrackUpdated := p.getOnTrackUpdated(); onTrackUpdated != nil { onTrackUpdated(p, publishedTrack) } } @@ -1885,14 +1879,12 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } if newTrack { - go func() { - p.pubLogger.Debugw( - "track published", - "trackID", mt.ID(), - "track", logger.Proto(mt.ToProto()), - ) - p.handleTrackPublished(mt) - }() + p.pubLogger.Debugw( + "track published", + "trackID", mt.ID(), + "track", logger.Proto(mt.ToProto()), + ) + p.handleTrackPublished(mt) } return mt, newTrack @@ -2000,15 +1992,6 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv p.supervisor.ClearPublishedTrack(trackID, mt) } - // not logged when closing - p.params.Telemetry.TrackUnpublished( - context.Background(), - p.ID(), - p.Identity(), - mt.ToProto(), - !p.IsClosed(), - ) - // re-use Track sid p.pendingTracksLock.Lock() if pti := p.pendingTracks[signalCid]; pti != nil { @@ -2023,10 +2006,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv if !p.IsClosed() { // unpublished events aren't necessary when participant is closed p.pubLogger.Debugw("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti)) - p.lock.RLock() - onTrackUnpublished := p.onTrackUnpublished - p.lock.RUnlock() - if onTrackUnpublished != nil { + if onTrackUnpublished := p.getOnTrackUnpublished(); onTrackUnpublished != nil { onTrackUnpublished(p, mt) } } @@ -2036,22 +2016,10 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv } func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack) { - p.lock.RLock() - onTrackPublished := p.onTrackPublished - p.lock.RUnlock() - if onTrackPublished != nil { + if onTrackPublished := p.getOnTrackPublished(); onTrackPublished != nil { onTrackPublished(p, track) } - // send webhook after callbacks are complete, persistence and state handling happens - // in `onTrackPublished` cb - p.params.Telemetry.TrackPublished( - context.Background(), - p.ID(), - p.Identity(), - track.ToProto(), - ) - p.pendingTracksLock.Lock() delete(p.pendingPublishingTracks, track.ID()) p.pendingTracksLock.Unlock() diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 1bb1f671c..c6c7609f5 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -73,6 +73,11 @@ type disconnectSignalOnResumeNoMessages struct { closedCount int } +type participantWorker struct { + eventsQueue *sutils.OpsQueue + participants []types.LocalParticipant +} + type Room struct { lock sync.RWMutex @@ -94,6 +99,7 @@ type Room struct { // map of identity -> Participant participants map[livekit.ParticipantIdentity]types.LocalParticipant + participantWorkers map[livekit.ParticipantIdentity]*participantWorker participantOpts map[livekit.ParticipantIdentity]*ParticipantOptions participantRequestSources map[livekit.ParticipantIdentity]routing.MessageSource hasPublished map[livekit.ParticipantIdentity]bool @@ -151,6 +157,7 @@ func NewRoom( trackManager: NewRoomTrackManager(), serverInfo: serverInfo, participants: make(map[livekit.ParticipantIdentity]types.LocalParticipant), + participantWorkers: make(map[livekit.ParticipantIdentity]*participantWorker), participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), hasPublished: make(map[livekit.ParticipantIdentity]bool), @@ -321,7 +328,6 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if r.participants[participant.Identity()] != nil { return ErrAlreadyJoined } - if r.protoRoom.MaxParticipants > 0 && !participant.IsRecorder() { numParticipants := uint32(0) for _, p := range r.participants { @@ -338,84 +344,102 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me r.joinedAt.Store(time.Now().Unix()) } - // it's important to set this before connection, we don't want to miss out on any published tracks - participant.OnTrackPublished(r.onTrackPublished) - participant.OnStateChange(func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) { - if r.onParticipantChanged != nil { - r.onParticipantChanged(participant) - } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + pw := r.addParticipantWorkerLocked(participant) - state := p.State() - if state == livekit.ParticipantInfo_ACTIVE { - // subscribe participant to existing published tracks - r.subscribeToExistingTracks(p) - - // start the workers once connectivity is established - p.Start() - - meta := &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), + participant.OnStateChange(func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { + pw.eventsQueue.Enqueue(func() { + if r.onParticipantChanged != nil { + r.onParticipantChanged(p) } - cds := participant.GetICEConnectionDetails() - for _, cd := range cds { - if cd.Type != types.ICEConnectionTypeUnknown { - meta.ConnectionType = string(cd.Type) - break + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + + if state == livekit.ParticipantInfo_ACTIVE { + // subscribe participant to existing published tracks + r.subscribeToExistingTracks(p) + + meta := &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), } - } - r.telemetry.ParticipantActive(context.Background(), - r.ToProto(), - p.ToProto(), - meta, - false, - ) + cds := p.GetICEConnectionDetails() + for _, cd := range cds { + if cd.Type != types.ICEConnectionTypeUnknown { + meta.ConnectionType = string(cd.Type) + break + } + } + r.telemetry.ParticipantActive(context.Background(), + r.ToProto(), + p.ToProto(), + meta, + false, + ) - p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) - } else if state == livekit.ParticipantInfo_DISCONNECTED { - // remove participant from room - go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) - } + p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) + } else if state == livekit.ParticipantInfo_DISCONNECTED { + // remove participant from room + r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) + } + }) + }) + // it's important to set this before connection, we don't want to miss out on any published tracks + participant.OnTrackPublished(func(p types.LocalParticipant, t types.MediaTrack) { + pw.eventsQueue.Enqueue(func() { + r.onTrackPublished(p, t) + }) + }) + participant.OnTrackUpdated(func(p types.LocalParticipant, t types.MediaTrack) { + pw.eventsQueue.Enqueue(func() { + r.onTrackUpdated(p, t) + }) + }) + participant.OnTrackUnpublished(func(p types.LocalParticipant, t types.MediaTrack) { + pw.eventsQueue.Enqueue(func() { + r.onTrackUnpublished(p, t) + }) + }) + participant.OnParticipantUpdate(func(p types.LocalParticipant) { + pw.eventsQueue.Enqueue(func() { + r.onParticipantUpdate(p) + }) }) - participant.OnTrackUpdated(r.onTrackUpdated) - participant.OnTrackUnpublished(r.onTrackUnpublished) - participant.OnParticipantUpdate(r.onParticipantUpdate) participant.OnDataPacket(r.onDataPacket) participant.OnSubscribeStatusChanged(func(publisherID livekit.ParticipantID, subscribed bool) { - if subscribed { - pub := r.GetParticipantByID(publisherID) - if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { - // when a participant subscribes to another participant, - // send speaker update if the subscribed to participant is active. - level, active := pub.GetAudioLevel() - if active { - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(pub.ID()), - Level: float32(level), - Active: active, - }, - }, false) - } + pw.eventsQueue.Enqueue(func() { + if subscribed { + pub := r.GetParticipantByID(publisherID) + if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { + // when a participant subscribes to another participant, + // send speaker update if the subscribed to participant is active. + level, active := pub.GetAudioLevel() + if active { + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(pub.ID()), + Level: float32(level), + Active: active, + }, + }, false) + } - if cq := pub.GetConnectionQuality(); cq != nil { - update := &livekit.ConnectionQualityUpdate{} - update.Updates = append(update.Updates, cq) - _ = participant.SendConnectionQualityUpdate(update) + if cq := pub.GetConnectionQuality(); cq != nil { + update := &livekit.ConnectionQualityUpdate{} + update.Updates = append(update.Updates, cq) + _ = participant.SendConnectionQualityUpdate(update) + } } + } else { + // no longer subscribed to the publisher, clear speaker status + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(publisherID), + Level: 0, + Active: false, + }, + }, true) } - } else { - // no longer subscribed to the publisher, clear speaker status - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(publisherID), - Level: 0, - Active: false, - }, - }, true) - } - + }) }) + r.Logger.Debugw("new participant joined", "pID", participant.ID(), "participant", participant.Identity(), @@ -558,6 +582,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } delete(r.participants, identity) + r.removeParticipantWorkerLocked(p) delete(r.participantOpts, identity) delete(r.participantRequestSources, identity) delete(r.hasPublished, identity) @@ -784,11 +809,14 @@ func (r *Room) Close() { } close(r.closed) r.lock.Unlock() + r.Logger.Infow("closing room") for _, p := range r.GetParticipants() { _ = p.Close(true, types.ParticipantCloseReasonRoomClose, false) } + r.protoProxy.Stop() + if r.onClose != nil { r.onClose() } @@ -1038,6 +1066,14 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. } }() } + + // send webhook after callbacks are complete, i.e. after persistence and state handling + r.telemetry.TrackPublished( + context.Background(), + participant.ID(), + participant.Identity(), + track.ToProto(), + ) } func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { @@ -1049,6 +1085,14 @@ func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { } func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { + r.telemetry.TrackUnpublished( + context.Background(), + p.ID(), + p.Identity(), + track.ToProto(), + !p.IsClosed(), + ) + r.trackManager.RemoveTrack(track) if !p.IsClosed() { r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) @@ -1452,6 +1496,52 @@ func (r *Room) DebugInfo() map[string]interface{} { return info } +func (r *Room) addParticipantWorkerLocked(p types.LocalParticipant) *participantWorker { + identity := p.Identity() + pw := r.participantWorkers[identity] + if pw != nil { + found := false + for _, participant := range pw.participants { + if p == participant { + found = true + break + } + } + if !found { + pw.participants = append(pw.participants, p) + } + return pw + } + + pw = &participantWorker{ + eventsQueue: sutils.NewOpsQueue(fmt.Sprintf("participant-worker-%s-%s", r.Name(), identity), 0, true), + participants: []types.LocalParticipant{p}, + } + pw.eventsQueue.Start() + r.participantWorkers[identity] = pw + return pw +} + +func (r *Room) removeParticipantWorkerLocked(p types.LocalParticipant) { + identity := p.Identity() + if pw, ok := r.participantWorkers[identity]; ok { + n := len(pw.participants) + for idx, participant := range pw.participants { + if p == participant { + pw.participants[idx] = pw.participants[n-1] + pw.participants = pw.participants[:n-1] + break + } + } + if len(pw.participants) == 0 { + pw.eventsQueue.Stop() + delete(r.participantWorkers, identity) + } + } +} + +// ------------------------------------------------------------ + func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp *livekit.DataPacket, logger logger.Logger) { dest := dp.GetUser().GetDestinationSids() var dpData []byte diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 81dfe90c8..4990b64d4 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -110,8 +110,7 @@ func TestRoomJoin(t *testing.T) { stateChangeCB := p.OnStateChangeArgsForCall(0) require.NotNil(t, stateChangeCB) - p.StateReturns(livekit.ParticipantInfo_ACTIVE) - stateChangeCB(p, livekit.ParticipantInfo_JOINED) + stateChangeCB(p, livekit.ParticipantInfo_ACTIVE) // it should become a subscriber when connectivity changes numTracks := 0 @@ -122,7 +121,7 @@ func TestRoomJoin(t *testing.T) { numTracks += len(op.GetPublishedTracks()) } - require.Equal(t, numTracks, p.SubscribeToTrackCallCount()) + require.Eventually(t, func() bool { return p.SubscribeToTrackCallCount() == numTracks }, 5*time.Second, 10*time.Millisecond) }) t.Run("participant state change is broadcasted to others", func(t *testing.T) { @@ -218,7 +217,7 @@ func TestParticipantUpdate(t *testing.T) { expected += 1 } fp := p.(*typesfakes.FakeLocalParticipant) - require.Equal(t, expected, fp.SendParticipantUpdateCallCount()) + require.Eventually(t, func() bool { return fp.SendParticipantUpdateCallCount() == expected }, 5*time.Second, 10*time.Millisecond) } }) } @@ -424,8 +423,8 @@ func TestNewTrack(t *testing.T) { require.NotNil(t, trackCB) trackCB(pub, track) // only p1 should've been subscribed to + require.Eventually(t, func() bool { return p1.SubscribeToTrackCallCount() == 1 }, 5*time.Second, 10*time.Millisecond) require.Equal(t, 0, p0.SubscribeToTrackCallCount()) - require.Equal(t, 1, p1.SubscribeToTrackCallCount()) }) } @@ -679,10 +678,9 @@ func TestHiddenParticipants(t *testing.T) { stateChangeCB := hidden.OnStateChangeArgsForCall(0) require.NotNil(t, stateChangeCB) - hidden.StateReturns(livekit.ParticipantInfo_ACTIVE) - stateChangeCB(hidden, livekit.ParticipantInfo_JOINED) + stateChangeCB(hidden, livekit.ParticipantInfo_ACTIVE) - require.Equal(t, 2, hidden.SubscribeToTrackCallCount()) + require.Eventually(t, func() bool { return hidden.SubscribeToTrackCallCount() == 2 }, 5*time.Second, 10*time.Millisecond) }) } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 5cbbb216f..c53791aa5 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -573,7 +573,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { m.lock.Unlock() if changedCB != nil && firstSubscribe { - go changedCB(publisherID, true) + changedCB(publisherID, true) } return nil } diff --git a/pkg/rtc/testutils.go b/pkg/rtc/testutils.go index f80230f0c..b63c372a8 100644 --- a/pkg/rtc/testutils.go +++ b/pkg/rtc/testutils.go @@ -83,5 +83,9 @@ func NewMockTrack(kind livekit.TrackType, name string) *typesfakes.FakeMediaTrac t.IDReturns(livekit.TrackID(utils.NewGuid(utils.TrackPrefix))) t.KindReturns(kind) t.NameReturns(name) + t.ToProtoReturns(&livekit.TrackInfo{ + Type: kind, + Name: name, + }) return t } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 953b14518..aa416b50c 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -39,7 +39,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/rtpextension" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" sutils "github.com/livekit/livekit-server/pkg/utils" @@ -238,7 +237,6 @@ type TransportParams struct { Config *WebRTCConfig DirectionConfig DirectionConfig CongestionControlConfig config.CongestionControlConfig - Telemetry telemetry.TelemetryService EnabledCodecs []*livekit.Codec Logger logger.Logger Transport livekit.SignalTarget diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index ec3b10600..7207ace3e 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -33,7 +33,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -54,7 +53,6 @@ type TransportManagerParams struct { SubscriberAsPrimary bool Config *WebRTCConfig ProtocolVersion types.ProtocolVersion - Telemetry telemetry.TelemetryService CongestionControlConfig config.CongestionControlConfig EnabledSubscribeCodecs []*livekit.Codec EnabledPublishCodecs []*livekit.Codec @@ -119,7 +117,6 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro Config: params.Config, DirectionConfig: params.Config.Publisher, CongestionControlConfig: params.CongestionControlConfig, - Telemetry: params.Telemetry, EnabledCodecs: params.EnabledPublishCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER), SimTracks: params.SimTracks, @@ -152,7 +149,6 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro Config: params.Config, DirectionConfig: params.Config.Subscriber, CongestionControlConfig: params.CongestionControlConfig, - Telemetry: params.Telemetry, EnabledCodecs: params.EnabledSubscribeCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER), ClientInfo: params.ClientInfo, @@ -722,7 +718,7 @@ func (t *TransportManager) ProcessPendingPublisherDataChannels() { } } -func (t *TransportManager) OnReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) { +func (t *TransportManager) HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) { t.mediaLossProxy.HandleMaxLossFeedback(dt, report) } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 466615c8a..4cbcb117a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -273,7 +273,6 @@ type Participant interface { IsRecorder() bool IsAgent() bool - Start() Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) @@ -379,7 +378,7 @@ type LocalParticipant interface { IssueFullReconnect(reason ParticipantCloseReason) // callbacks - OnStateChange(func(p LocalParticipant, oldState livekit.ParticipantInfo_State)) + OnStateChange(func(p LocalParticipant, state livekit.ParticipantInfo_State)) OnMigrateStateChange(func(p LocalParticipant, migrateState MigrateState)) // OnTrackPublished - remote added a track OnTrackPublished(func(LocalParticipant, MediaTrack)) @@ -393,9 +392,10 @@ type LocalParticipant interface { OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) OnClose(callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) - OnReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) OnTrafficLoad(callback func(trafficLoad *TrafficLoad)) + HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) + // session migration MaybeStartMigration(force bool, onStart func()) bool SetMigrateState(s MigrateState) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 756099d2c..32c1c2869 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -355,6 +355,12 @@ type FakeLocalParticipant struct { handleOfferArgsForCall []struct { arg1 webrtc.SessionDescription } + HandleReceiverReportStub func(*sfu.DownTrack, *rtcp.ReceiverReport) + handleReceiverReportMutex sync.RWMutex + handleReceiverReportArgsForCall []struct { + arg1 *sfu.DownTrack + arg2 *rtcp.ReceiverReport + } HandleReconnectAndSendResponseStub func(livekit.ReconnectReason, *livekit.ReconnectResponse) error handleReconnectAndSendResponseMutex sync.RWMutex handleReconnectAndSendResponseArgsForCall []struct { @@ -571,16 +577,10 @@ type FakeLocalParticipant struct { onParticipantUpdateArgsForCall []struct { arg1 func(types.LocalParticipant) } - OnReceiverReportStub func(*sfu.DownTrack, *rtcp.ReceiverReport) - onReceiverReportMutex sync.RWMutex - onReceiverReportArgsForCall []struct { - arg1 *sfu.DownTrack - arg2 *rtcp.ReceiverReport - } - OnStateChangeStub func(func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State)) + OnStateChangeStub func(func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) onStateChangeMutex sync.RWMutex onStateChangeArgsForCall []struct { - arg1 func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) + arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State) } OnSubscribeStatusChangedStub func(func(publisherID livekit.ParticipantID, subscribed bool)) onSubscribeStatusChangedMutex sync.RWMutex @@ -791,10 +791,6 @@ type FakeLocalParticipant struct { setTrackMutedReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } - StartStub func() - startMutex sync.RWMutex - startArgsForCall []struct { - } StateStub func() livekit.ParticipantInfo_State stateMutex sync.RWMutex stateArgsForCall []struct { @@ -2740,6 +2736,39 @@ func (fake *FakeLocalParticipant) HandleOfferArgsForCall(i int) webrtc.SessionDe return argsForCall.arg1 } +func (fake *FakeLocalParticipant) HandleReceiverReport(arg1 *sfu.DownTrack, arg2 *rtcp.ReceiverReport) { + fake.handleReceiverReportMutex.Lock() + fake.handleReceiverReportArgsForCall = append(fake.handleReceiverReportArgsForCall, struct { + arg1 *sfu.DownTrack + arg2 *rtcp.ReceiverReport + }{arg1, arg2}) + stub := fake.HandleReceiverReportStub + fake.recordInvocation("HandleReceiverReport", []interface{}{arg1, arg2}) + fake.handleReceiverReportMutex.Unlock() + if stub != nil { + fake.HandleReceiverReportStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipant) HandleReceiverReportCallCount() int { + fake.handleReceiverReportMutex.RLock() + defer fake.handleReceiverReportMutex.RUnlock() + return len(fake.handleReceiverReportArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleReceiverReportCalls(stub func(*sfu.DownTrack, *rtcp.ReceiverReport)) { + fake.handleReceiverReportMutex.Lock() + defer fake.handleReceiverReportMutex.Unlock() + fake.HandleReceiverReportStub = stub +} + +func (fake *FakeLocalParticipant) HandleReceiverReportArgsForCall(i int) (*sfu.DownTrack, *rtcp.ReceiverReport) { + fake.handleReceiverReportMutex.RLock() + defer fake.handleReceiverReportMutex.RUnlock() + argsForCall := fake.handleReceiverReportArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeLocalParticipant) HandleReconnectAndSendResponse(arg1 livekit.ReconnectReason, arg2 *livekit.ReconnectResponse) error { fake.handleReconnectAndSendResponseMutex.Lock() ret, specificReturn := fake.handleReconnectAndSendResponseReturnsOnCall[len(fake.handleReconnectAndSendResponseArgsForCall)] @@ -3935,43 +3964,10 @@ func (fake *FakeLocalParticipant) OnParticipantUpdateArgsForCall(i int) func(typ return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnReceiverReport(arg1 *sfu.DownTrack, arg2 *rtcp.ReceiverReport) { - fake.onReceiverReportMutex.Lock() - fake.onReceiverReportArgsForCall = append(fake.onReceiverReportArgsForCall, struct { - arg1 *sfu.DownTrack - arg2 *rtcp.ReceiverReport - }{arg1, arg2}) - stub := fake.OnReceiverReportStub - fake.recordInvocation("OnReceiverReport", []interface{}{arg1, arg2}) - fake.onReceiverReportMutex.Unlock() - if stub != nil { - fake.OnReceiverReportStub(arg1, arg2) - } -} - -func (fake *FakeLocalParticipant) OnReceiverReportCallCount() int { - fake.onReceiverReportMutex.RLock() - defer fake.onReceiverReportMutex.RUnlock() - return len(fake.onReceiverReportArgsForCall) -} - -func (fake *FakeLocalParticipant) OnReceiverReportCalls(stub func(*sfu.DownTrack, *rtcp.ReceiverReport)) { - fake.onReceiverReportMutex.Lock() - defer fake.onReceiverReportMutex.Unlock() - fake.OnReceiverReportStub = stub -} - -func (fake *FakeLocalParticipant) OnReceiverReportArgsForCall(i int) (*sfu.DownTrack, *rtcp.ReceiverReport) { - fake.onReceiverReportMutex.RLock() - defer fake.onReceiverReportMutex.RUnlock() - argsForCall := fake.onReceiverReportArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeLocalParticipant) OnStateChange(arg1 func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State)) { +func (fake *FakeLocalParticipant) OnStateChange(arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) { fake.onStateChangeMutex.Lock() fake.onStateChangeArgsForCall = append(fake.onStateChangeArgsForCall, struct { - arg1 func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) + arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State) }{arg1}) stub := fake.OnStateChangeStub fake.recordInvocation("OnStateChange", []interface{}{arg1}) @@ -3987,13 +3983,13 @@ func (fake *FakeLocalParticipant) OnStateChangeCallCount() int { return len(fake.onStateChangeArgsForCall) } -func (fake *FakeLocalParticipant) OnStateChangeCalls(stub func(func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State))) { +func (fake *FakeLocalParticipant) OnStateChangeCalls(stub func(func(p types.LocalParticipant, state livekit.ParticipantInfo_State))) { fake.onStateChangeMutex.Lock() defer fake.onStateChangeMutex.Unlock() fake.OnStateChangeStub = stub } -func (fake *FakeLocalParticipant) OnStateChangeArgsForCall(i int) func(p types.LocalParticipant, oldState livekit.ParticipantInfo_State) { +func (fake *FakeLocalParticipant) OnStateChangeArgsForCall(i int) func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { fake.onStateChangeMutex.RLock() defer fake.onStateChangeMutex.RUnlock() argsForCall := fake.onStateChangeArgsForCall[i] @@ -5209,30 +5205,6 @@ func (fake *FakeLocalParticipant) SetTrackMutedReturnsOnCall(i int, result1 *liv }{result1} } -func (fake *FakeLocalParticipant) Start() { - fake.startMutex.Lock() - fake.startArgsForCall = append(fake.startArgsForCall, struct { - }{}) - stub := fake.StartStub - fake.recordInvocation("Start", []interface{}{}) - fake.startMutex.Unlock() - if stub != nil { - fake.StartStub() - } -} - -func (fake *FakeLocalParticipant) StartCallCount() int { - fake.startMutex.RLock() - defer fake.startMutex.RUnlock() - return len(fake.startArgsForCall) -} - -func (fake *FakeLocalParticipant) StartCalls(stub func()) { - fake.startMutex.Lock() - defer fake.startMutex.Unlock() - fake.StartStub = stub -} - func (fake *FakeLocalParticipant) State() livekit.ParticipantInfo_State { fake.stateMutex.Lock() ret, specificReturn := fake.stateReturnsOnCall[len(fake.stateArgsForCall)] @@ -6282,6 +6254,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.handleAnswerMutex.RUnlock() fake.handleOfferMutex.RLock() defer fake.handleOfferMutex.RUnlock() + fake.handleReceiverReportMutex.RLock() + defer fake.handleReceiverReportMutex.RUnlock() fake.handleReconnectAndSendResponseMutex.RLock() defer fake.handleReconnectAndSendResponseMutex.RUnlock() fake.handleSignalSourceCloseMutex.RLock() @@ -6334,8 +6308,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.onMigrateStateChangeMutex.RUnlock() fake.onParticipantUpdateMutex.RLock() defer fake.onParticipantUpdateMutex.RUnlock() - fake.onReceiverReportMutex.RLock() - defer fake.onReceiverReportMutex.RUnlock() fake.onStateChangeMutex.RLock() defer fake.onStateChangeMutex.RUnlock() fake.onSubscribeStatusChangedMutex.RLock() @@ -6392,8 +6364,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.setSubscriberChannelCapacityMutex.RUnlock() fake.setTrackMutedMutex.RLock() defer fake.setTrackMutedMutex.RUnlock() - fake.startMutex.RLock() - defer fake.startMutex.RUnlock() fake.stateMutex.RLock() defer fake.stateMutex.RUnlock() fake.subscribeToTrackMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 3f46bdbba..0c5c3929a 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -165,10 +165,6 @@ type FakeParticipant struct { setNameArgsForCall []struct { arg1 string } - StartStub func() - startMutex sync.RWMutex - startArgsForCall []struct { - } StateStub func() livekit.ParticipantInfo_State stateMutex sync.RWMutex stateArgsForCall []struct { @@ -1047,30 +1043,6 @@ func (fake *FakeParticipant) SetNameArgsForCall(i int) string { return argsForCall.arg1 } -func (fake *FakeParticipant) Start() { - fake.startMutex.Lock() - fake.startArgsForCall = append(fake.startArgsForCall, struct { - }{}) - stub := fake.StartStub - fake.recordInvocation("Start", []interface{}{}) - fake.startMutex.Unlock() - if stub != nil { - fake.StartStub() - } -} - -func (fake *FakeParticipant) StartCallCount() int { - fake.startMutex.RLock() - defer fake.startMutex.RUnlock() - return len(fake.startArgsForCall) -} - -func (fake *FakeParticipant) StartCalls(stub func()) { - fake.startMutex.Lock() - defer fake.startMutex.Unlock() - fake.StartStub = stub -} - func (fake *FakeParticipant) State() livekit.ParticipantInfo_State { fake.stateMutex.Lock() ret, specificReturn := fake.stateReturnsOnCall[len(fake.stateArgsForCall)] @@ -1393,8 +1365,6 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.setMetadataMutex.RUnlock() fake.setNameMutex.RLock() defer fake.setNameMutex.RUnlock() - fake.startMutex.RLock() - defer fake.startMutex.RUnlock() fake.stateMutex.RLock() defer fake.stateMutex.RUnlock() fake.subscriptionPermissionMutex.RLock() diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index ce5177661..ed01e1f79 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -62,9 +62,6 @@ func NewUpTrackManager(params UpTrackManagerParams) *UpTrackManager { } } -func (u *UpTrackManager) Start() { -} - func (u *UpTrackManager) Close(willBeResumed bool) { u.lock.Lock() u.closed = true From ea2fa30cf86ca9a3765c81ec144db2a84fae96fa Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 Jan 2024 23:12:33 +0530 Subject: [PATCH 101/114] Plug worker leaks (#2422) Thank you @paulwe --- pkg/rtc/room.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index c6c7609f5..745ae9f4f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -574,6 +574,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek r.lock.Unlock() return } + r.removeParticipantWorkerLocked(p) if pID != "" && p.ID() != pID { // participant session has been replaced @@ -582,7 +583,6 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } delete(r.participants, identity) - r.removeParticipantWorkerLocked(p) delete(r.participantOpts, identity) delete(r.participantRequestSources, identity) delete(r.hasPublished, identity) @@ -1529,6 +1529,7 @@ func (r *Room) removeParticipantWorkerLocked(p types.LocalParticipant) { for idx, participant := range pw.participants { if p == participant { pw.participants[idx] = pw.participants[n-1] + pw.participants[n-1] = nil pw.participants = pw.participants[:n-1] break } From 5f3bd7cf59f747d7c040d86bfeaa1153c6e4edad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:28:46 -0800 Subject: [PATCH 102/114] Update actions/upload-artifact action to v4 (#2317) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/buildtest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 7fb5b2842..0b906e6f0 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -71,7 +71,7 @@ jobs: # Upload the original go test log as an artifact for later review. - name: Upload test log - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: test-log From 134b6f05b418a4698887a60e2acf8ce1fe15f96a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:29:07 -0800 Subject: [PATCH 103/114] Update module github.com/pion/dtls/v2 to v2.2.9 (#2355) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 03f33a351..10b54ea8e 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pion/dtls/v2 v2.2.8 + github.com/pion/dtls/v2 v2.2.9 github.com/pion/ice/v2 v2.3.12 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.13 diff --git a/go.sum b/go.sum index b42e0b644..fb47d5edd 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= -github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/dtls/v2 v2.2.9 h1:K+D/aVf9/REahQvqk6G5JavdrD8W1PWDKC11UlwN7ts= +github.com/pion/dtls/v2 v2.2.9/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= github.com/pion/ice/v2 v2.3.12 h1:NWKW2b3+oSZS3klbQMIEWQ0i52Kuo0KBg505a5kQv4s= github.com/pion/ice/v2 v2.3.12/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= From efbc985c82c4c8eec74200a7a1c5b36a57d5053c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 10:57:41 +0530 Subject: [PATCH 104/114] Cache data synchronously for processing in worker. (#2424) It is possible that state of underlying object has changed between event posting and event processing. So, cache data synchronously and use it during event processing. This is still not perfect as things like `hidden` and `IsClosed` is accessed in worker. Ideally, it can be a snapshot of current state of all required values that can be posted to the worker and the worker just operates with data. --- pkg/rtc/room.go | 84 +++++++++++++++++++++++--------------- pkg/rtc/room_test.go | 2 +- pkg/service/roommanager.go | 8 ++-- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 745ae9f4f..d95044ed9 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -118,7 +118,7 @@ type Room struct { trailer []byte - onParticipantChanged func(p types.LocalParticipant) + onParticipantChanged func(p types.LocalParticipant, pi *livekit.ParticipantInfo) onRoomUpdated func() onClose func() @@ -347,11 +347,13 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me pw := r.addParticipantWorkerLocked(participant) participant.OnStateChange(func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { + ri := r.ToProto() + pi := p.ToProto() pw.eventsQueue.Enqueue(func() { if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + r.onParticipantChanged(p, pi) } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) if state == livekit.ParticipantInfo_ACTIVE { // subscribe participant to existing published tracks @@ -368,8 +370,8 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me } } r.telemetry.ParticipantActive(context.Background(), - r.ToProto(), - p.ToProto(), + ri, + pi, meta, false, ) @@ -383,23 +385,29 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me }) // it's important to set this before connection, we don't want to miss out on any published tracks participant.OnTrackPublished(func(p types.LocalParticipant, t types.MediaTrack) { + pi := p.ToProto() + ti := t.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackPublished(p, t) + r.onTrackPublished(p, pi, t, ti) }) }) participant.OnTrackUpdated(func(p types.LocalParticipant, t types.MediaTrack) { + pi := p.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackUpdated(p, t) + r.onTrackUpdated(p, pi, t) }) }) participant.OnTrackUnpublished(func(p types.LocalParticipant, t types.MediaTrack) { + pi := p.ToProto() + ti := t.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackUnpublished(p, t) + r.onTrackUnpublished(p, pi, t, ti) }) }) participant.OnParticipantUpdate(func(p types.LocalParticipant) { + pi := p.ToProto() pw.eventsQueue.Enqueue(func() { - r.onParticipantUpdate(p) + r.onParticipantUpdate(p, pi) }) }) participant.OnDataPacket(r.onDataPacket) @@ -460,7 +468,7 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me r.participantRequestSources[participant.Identity()] = requestSource if r.onParticipantChanged != nil { - r.onParticipantChanged(participant) + r.onParticipantChanged(participant, participant.ToProto()) } time.AfterFunc(time.Minute, func() { @@ -637,10 +645,11 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek r.leftAt.Store(time.Now().Unix()) if sendUpdates { + pi := p.ToProto() if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + r.onParticipantChanged(p, pi) } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) } } @@ -826,7 +835,7 @@ func (r *Room) OnClose(f func()) { r.onClose = f } -func (r *Room) OnParticipantChanged(f func(participant types.LocalParticipant)) { +func (r *Room) OnParticipantChanged(f func(p types.LocalParticipant, pi *livekit.ParticipantInfo)) { r.onParticipantChanged = f } @@ -988,9 +997,14 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS } // a ParticipantImpl in the room added a new track, subscribe other participants to it -func (r *Room) onTrackPublished(participant types.LocalParticipant, track types.MediaTrack) { +func (r *Room) onTrackPublished( + participant types.LocalParticipant, + pi *livekit.ParticipantInfo, + track types.MediaTrack, + ti *livekit.TrackInfo, +) { // publish participant update, since track state is changed - r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(participant, pi, broadcastOptions{skipSource: true}) r.lock.RLock() // subscribe all existing participants to this MediaTrack @@ -1019,7 +1033,7 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. r.lock.RUnlock() if onParticipantChanged != nil { - onParticipantChanged(participant) + onParticipantChanged(participant, pi) } r.trackManager.AddTrack(track, participant.Identity(), participant.ID()) @@ -1072,42 +1086,51 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. context.Background(), participant.ID(), participant.Identity(), - track.ToProto(), + ti, ) } -func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { +func (r *Room) onTrackUpdated( + p types.LocalParticipant, + pi *livekit.ParticipantInfo, + _ types.MediaTrack, +) { // send track updates to everyone, especially if track was updated by admin - r.broadcastParticipantState(p, broadcastOptions{}) + r.broadcastParticipantState(p, pi, broadcastOptions{}) if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + r.onParticipantChanged(p, pi) } } -func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { +func (r *Room) onTrackUnpublished( + p types.LocalParticipant, + pi *livekit.ParticipantInfo, + track types.MediaTrack, + ti *livekit.TrackInfo, +) { r.telemetry.TrackUnpublished( context.Background(), p.ID(), p.Identity(), - track.ToProto(), + ti, !p.IsClosed(), ) r.trackManager.RemoveTrack(track) if !p.IsClosed() { - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) } if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + r.onParticipantChanged(p, pi) } } -func (r *Room) onParticipantUpdate(p types.LocalParticipant) { +func (r *Room) onParticipantUpdate(p types.LocalParticipant, pi *livekit.ParticipantInfo) { r.protoProxy.MarkDirty(false) // immediately notify when permissions or metadata changed - r.broadcastParticipantState(p, broadcastOptions{immediate: true}) + r.broadcastParticipantState(p, pi, broadcastOptions{immediate: true}) if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + r.onParticipantChanged(p, pi) } } @@ -1142,16 +1165,13 @@ func (r *Room) subscribeToExistingTracks(p types.LocalParticipant) { } // broadcast an update about participant p -func (r *Room) broadcastParticipantState(p types.LocalParticipant, opts broadcastOptions) { - pi := p.ToProto() - +func (r *Room) broadcastParticipantState(p types.LocalParticipant, pi *livekit.ParticipantInfo, opts broadcastOptions) { if p.Hidden() { if !opts.skipSource { // send update only to hidden participant err := p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi}) if err != nil { - r.Logger.Errorw("could not send update to participant", err, - "participant", p.Identity(), "pID", p.ID()) + p.GetLogger().Errorw("could not send update to participant", err) } } return diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 4990b64d4..ec564c4af 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -127,7 +127,7 @@ func TestRoomJoin(t *testing.T) { t.Run("participant state change is broadcasted to others", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants}) var changedParticipant types.Participant - rm.OnParticipantChanged(func(participant types.LocalParticipant) { + rm.OnParticipantChanged(func(participant types.LocalParticipant, _pi *livekit.ParticipantInfo) { changedParticipant = participant }) participants := rm.GetParticipants() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index abfd29331..1c010acbb 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -577,10 +577,10 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room } }) - newRoom.OnParticipantChanged(func(p types.LocalParticipant) { - if !p.IsDisconnected() { - if err := r.roomStore.StoreParticipant(ctx, roomName, p.ToProto()); err != nil { - newRoom.Logger.Errorw("could not handle participant change", err) + newRoom.OnParticipantChanged(func(p types.LocalParticipant, pi *livekit.ParticipantInfo) { + if pi.State != livekit.ParticipantInfo_DISCONNECTED { + if err := r.roomStore.StoreParticipant(ctx, roomName, pi); err != nil { + p.GetLogger().Errorw("could not handle participant change", err) } } }) From 0be241eed8836f14e8a53e40ef246ec2f9f95609 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 28 Jan 2024 21:35:25 -0800 Subject: [PATCH 105/114] refactor transport callbacks as interface (#2423) * refactor transport callbacks as interface * test --- pkg/rtc/participant.go | 104 ++- pkg/rtc/transport.go | 301 ++------- pkg/rtc/transport/handler.go | 55 ++ pkg/rtc/transport/negotiationstate.go | 26 + .../transport/transportfakes/fake_handler.go | 593 ++++++++++++++++++ pkg/rtc/transport_test.go | 124 ++-- pkg/rtc/transportmanager.go | 120 +--- test/client/client.go | 23 +- 8 files changed, 928 insertions(+), 418 deletions(-) create mode 100644 pkg/rtc/transport/handler.go create mode 100644 pkg/rtc/transport/negotiationstate.go create mode 100644 pkg/rtc/transport/transportfakes/fake_handler.go diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 0aa98e198..df60f272a 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -36,6 +36,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc/supervisor" + "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -1149,13 +1150,91 @@ func (p *ParticipantImpl) UpdateMediaRTT(rtt uint32) { } } +type AnyTransportHandler struct { + transport.UnimplementedHandler + p *ParticipantImpl +} + +func (h AnyTransportHandler) OnFailed(isShortLived bool) { + h.p.onAnyTransportFailed() +} + +func (h AnyTransportHandler) OnNegotiationFailed() { + h.p.onAnyTransportNegotiationFailed() +} + +func (h AnyTransportHandler) OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { + return h.p.onICECandidate(c, target) +} + +type PublisherTransportHandler struct { + AnyTransportHandler +} + +func (h PublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription) error { + return h.p.onPublisherAnswer(sd) +} + +func (h PublisherTransportHandler) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + h.p.onMediaTrack(track, rtpReceiver) +} + +func (h PublisherTransportHandler) OnInitialConnected() { + h.p.onPublisherInitialConnected() +} + +func (h PublisherTransportHandler) OnDataPacket(kind livekit.DataPacket_Kind, data []byte) { + h.p.onDataMessage(kind, data) +} + +type SubscriberTransportHandler struct { + AnyTransportHandler +} + +func (h SubscriberTransportHandler) OnOffer(sd webrtc.SessionDescription) error { + return h.p.onSubscriberOffer(sd) +} + +func (h SubscriberTransportHandler) OnStreamStateChange(update *streamallocator.StreamStateUpdate) error { + return h.p.onStreamStateChange(update) +} + +func (h SubscriberTransportHandler) OnInitialConnected() { + h.p.onSubscriberInitialConnected() +} + +type PrimaryTransportHandler struct { + transport.Handler + p *ParticipantImpl +} + +func (h PrimaryTransportHandler) OnInitialConnected() { + h.Handler.OnInitialConnected() + h.p.onPrimaryTransportInitialConnected() +} + +func (h PrimaryTransportHandler) OnFullyEstablished() { + h.p.onPrimaryTransportFullyEstablished() +} + func (p *ParticipantImpl) setupTransportManager() error { + ath := AnyTransportHandler{p: p} + var pth transport.Handler = PublisherTransportHandler{ath} + var sth transport.Handler = SubscriberTransportHandler{ath} + + subscriberAsPrimary := p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe() + if subscriberAsPrimary { + sth = PrimaryTransportHandler{sth, p} + } else { + pth = PrimaryTransportHandler{pth, p} + } + params := TransportManagerParams{ Identity: p.params.Identity, SID: p.params.SID, // primary connection does not change, canSubscribe can change if permission was updated // after the participant has joined - SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), + SubscriberAsPrimary: subscriberAsPrimary, Config: p.params.Config, ProtocolVersion: p.params.ProtocolVersion, CongestionControlConfig: p.params.CongestionControlConfig, @@ -1171,6 +1250,8 @@ func (p *ParticipantImpl) setupTransportManager() error { AllowPlayoutDelay: p.params.PlayoutDelay.GetEnabled(), DataChannelMaxBufferedAmount: p.params.DataChannelMaxBufferedAmount, Logger: p.params.Logger.WithComponent(sutils.ComponentTransport), + PublisherHandler: pth, + SubscriberHandler: sth, } if p.params.SyncStreams && p.params.PlayoutDelay.GetEnabled() && p.params.ClientInfo.isFirefox() { // we will disable playout delay for Firefox if the user is expecting @@ -1202,27 +1283,6 @@ func (p *ParticipantImpl) setupTransportManager() error { } }) - tm.OnPublisherICECandidate(func(c *webrtc.ICECandidate) error { - return p.onICECandidate(c, livekit.SignalTarget_PUBLISHER) - }) - tm.OnPublisherAnswer(p.onPublisherAnswer) - tm.OnPublisherTrack(p.onMediaTrack) - tm.OnPublisherInitialConnected(p.onPublisherInitialConnected) - - tm.OnSubscriberOffer(p.onSubscriberOffer) - tm.OnSubscriberICECandidate(func(c *webrtc.ICECandidate) error { - return p.onICECandidate(c, livekit.SignalTarget_SUBSCRIBER) - }) - tm.OnSubscriberInitialConnected(p.onSubscriberInitialConnected) - tm.OnSubscriberStreamStateChange(p.onStreamStateChange) - - tm.OnPrimaryTransportInitialConnected(p.onPrimaryTransportInitialConnected) - tm.OnPrimaryTransportFullyEstablished(p.onPrimaryTransportFullyEstablished) - tm.OnAnyTransportFailed(p.onAnyTransportFailed) - tm.OnAnyTransportNegotiationFailed(p.onAnyTransportNegotiationFailed) - - tm.OnDataMessage(p.onDataMessage) - tm.SetSubscriberAllowPause(p.params.SubscriberAllowPause) p.TransportManager = tm return nil diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index aa416b50c..c0e804d65 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -35,12 +35,12 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/rtpextension" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" - "github.com/livekit/livekit-server/pkg/utils" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -77,9 +77,6 @@ var ( ErrIceRestartOnClosedPeerConnection = errors.New("ICE restart on closed peer connection") ErrNoTransceiver = errors.New("no transceiver") ErrNoSender = errors.New("no sender") - ErrNoICECandidateHandler = errors.New("no ICE candidate handler") - ErrNoOfferHandler = errors.New("no offer handler") - ErrNoAnswerHandler = errors.New("no answer handler") ErrMidNotFound = errors.New("mid not found") ) @@ -128,31 +125,6 @@ func (e event) String() string { // ------------------------------------------------------- -type NegotiationState int - -const ( - NegotiationStateNone NegotiationState = iota - // waiting for remote description - NegotiationStateRemote - // need to Negotiate again - NegotiationStateRetry -) - -func (n NegotiationState) String() string { - switch n { - case NegotiationStateNone: - return "NONE" - case NegotiationStateRemote: - return "WAITING_FOR_REMOTE" - case NegotiationStateRetry: - return "RETRY" - default: - return fmt.Sprintf("%d", int(n)) - } -} - -// ------------------------------------------------------- - type SimulcastTrackInfo struct { Mid string Rid string @@ -175,7 +147,6 @@ type PCTransport struct { reliableDCOpened bool lossyDC *webrtc.DataChannel lossyDCOpened bool - onDataPacket func(kind livekit.DataPacket_Kind, data []byte) iceStartedAt time.Time iceConnectedAt time.Time @@ -186,18 +157,10 @@ type PCTransport struct { resetShortConnOnICERestart atomic.Bool signalingRTT atomic.Uint32 // milliseconds - onFullyEstablished func() - debouncedNegotiate func(func()) debouncePending bool - onICECandidate func(c *webrtc.ICECandidate) error - onOffer func(offer webrtc.SessionDescription) error - onAnswer func(answer webrtc.SessionDescription) error - onInitialConnected func() - onFailed func(isShortLived bool) - onNegotiationStateChanged func(state NegotiationState) - onNegotiationFailed func() + onNegotiationStateChanged func(state transport.NegotiationState) // stream allocator for subscriber PC streamAllocator *streamallocator.StreamAllocator @@ -213,7 +176,7 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool - eventsQueue *utils.OpsQueue + eventsQueue *sutils.OpsQueue // the following should be accessed only in event processing go routine cacheLocalCandidates bool @@ -221,7 +184,7 @@ type PCTransport struct { pendingRemoteCandidates []*webrtc.ICECandidateInit restartAfterGathering bool restartAtNextOffer bool - negotiationState NegotiationState + negotiationState transport.NegotiationState negotiateCounter atomic.Int32 signalStateCheckTimer *time.Timer currentOfferIceCredential string // ice user:pwd, for publish side ice restart checking @@ -231,6 +194,7 @@ type PCTransport struct { } type TransportParams struct { + Handler transport.Handler ParticipantID livekit.ParticipantID ParticipantIdentity livekit.ParticipantIdentity ProtocolVersion types.ProtocolVersion @@ -378,8 +342,8 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { t := &PCTransport{ params: params, debouncedNegotiate: debounce.New(negotiationFrequency), - negotiationState: NegotiationStateNone, - eventsQueue: utils.NewOpsQueue("transport", 64, false), + negotiationState: transport.NegotiationStateNone, + eventsQueue: sutils.NewOpsQueue("transport", 64, false), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, connectionDetails: types.NewICEConnectionDetails(params.Transport, params.Logger), @@ -389,6 +353,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { Config: params.CongestionControlConfig, Logger: params.Logger.WithComponent(sutils.ComponentCongestionControl), }) + t.streamAllocator.OnStreamStateChange(params.Handler.OnStreamStateChange) t.streamAllocator.Start() t.pacer = pacer.NewPassThrough(params.Logger) } @@ -419,6 +384,7 @@ func (t *PCTransport) createPeerConnection() error { t.pc.OnConnectionStateChange(t.onPeerConnectionStateChange) t.pc.OnDataChannel(t.onDataChannel) + t.pc.OnTrack(t.params.Handler.OnTrack) t.me = me @@ -608,9 +574,7 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) { t.params.Logger.Infow("force short ICE connection") } - if onFailed := t.getOnFailed(); onFailed != nil { - onFailed(isShort) - } + t.params.Handler.OnFailed(isShort) } func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState) { @@ -644,9 +608,7 @@ func (t *PCTransport) onPeerConnectionStateChange(state webrtc.PeerConnectionSta t.clearConnTimer() isInitialConnection := t.setConnectedAt(time.Now()) if isInitialConnection { - if onInitialConnected := t.getOnInitialConnected(); onInitialConnected != nil { - onInitialConnected() - } + t.params.Handler.OnInitialConnected() t.maybeNotifyFullyEstablished() } @@ -666,9 +628,7 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { t.reliableDCOpened = true t.lock.Unlock() dc.OnMessage(func(msg webrtc.DataChannelMessage) { - if onDataPacket := t.getOnDataPacket(); onDataPacket != nil { - onDataPacket(livekit.DataPacket_RELIABLE, msg.Data) - } + t.params.Handler.OnDataPacket(livekit.DataPacket_RELIABLE, msg.Data) }) t.maybeNotifyFullyEstablished() @@ -678,9 +638,7 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { t.lossyDCOpened = true t.lock.Unlock() dc.OnMessage(func(msg webrtc.DataChannelMessage) { - if onDataPacket := t.getOnDataPacket(); onDataPacket != nil { - onDataPacket(livekit.DataPacket_LOSSY, msg.Data) - } + t.params.Handler.OnDataPacket(livekit.DataPacket_LOSSY, msg.Data) }) t.maybeNotifyFullyEstablished() @@ -691,9 +649,7 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { func (t *PCTransport) maybeNotifyFullyEstablished() { if t.isFullyEstablished() { - if onFullyEstablished := t.getOnFullyEstablished(); onFullyEstablished != nil { - onFullyEstablished() - } + t.params.Handler.OnFullyEstablished() } } @@ -975,134 +931,19 @@ func (t *PCTransport) HandleRemoteDescription(sd webrtc.SessionDescription) { }) } -func (t *PCTransport) OnICECandidate(f func(c *webrtc.ICECandidate) error) { - t.lock.Lock() - t.onICECandidate = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnICECandidate() func(c *webrtc.ICECandidate) error { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onICECandidate -} - -func (t *PCTransport) OnInitialConnected(f func()) { - t.lock.Lock() - t.onInitialConnected = f - t.lock.Unlock() - - if f != nil { - if t.pc.ConnectionState() == webrtc.PeerConnectionStateConnected { - go f() - } - } -} - -func (t *PCTransport) getOnInitialConnected() func() { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onInitialConnected -} - -func (t *PCTransport) OnFullyEstablished(f func()) { - t.lock.Lock() - t.onFullyEstablished = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnFullyEstablished() func() { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onFullyEstablished -} - -func (t *PCTransport) OnFailed(f func(isShortLived bool)) { - t.lock.Lock() - t.onFailed = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnFailed() func(isShortLived bool) { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onFailed -} - -func (t *PCTransport) OnTrack(f func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver)) { - t.pc.OnTrack(f) -} - -func (t *PCTransport) OnDataPacket(f func(kind livekit.DataPacket_Kind, data []byte)) { - t.lock.Lock() - t.onDataPacket = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnDataPacket() func(kind livekit.DataPacket_Kind, data []byte) { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onDataPacket -} - -// OnOffer is called when the PeerConnection starts negotiation and prepares an offer -func (t *PCTransport) OnOffer(f func(sd webrtc.SessionDescription) error) { - t.lock.Lock() - t.onOffer = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnOffer() func(sd webrtc.SessionDescription) error { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onOffer -} - -func (t *PCTransport) OnAnswer(f func(sd webrtc.SessionDescription) error) { - t.lock.Lock() - t.onAnswer = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnAnswer() func(sd webrtc.SessionDescription) error { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onAnswer -} - -func (t *PCTransport) OnNegotiationStateChanged(f func(state NegotiationState)) { +func (t *PCTransport) OnNegotiationStateChanged(f func(state transport.NegotiationState)) { t.lock.Lock() t.onNegotiationStateChanged = f t.lock.Unlock() } -func (t *PCTransport) getOnNegotiationStateChanged() func(state NegotiationState) { +func (t *PCTransport) getOnNegotiationStateChanged() func(state transport.NegotiationState) { t.lock.RLock() defer t.lock.RUnlock() return t.onNegotiationStateChanged } -func (t *PCTransport) OnNegotiationFailed(f func()) { - t.lock.Lock() - t.onNegotiationFailed = f - t.lock.Unlock() -} - -func (t *PCTransport) getOnNegotiationFailed() func() { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.onNegotiationFailed -} - func (t *PCTransport) Negotiate(force bool) { if t.isClosed.Load() { return @@ -1153,14 +994,6 @@ func (t *PCTransport) ResetShortConnOnICERestart() { t.resetShortConnOnICERestart.Store(true) } -func (t *PCTransport) OnStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { - if t.streamAllocator == nil { - return - } - - t.streamAllocator.OnStreamStateChange(f) -} - func (t *PCTransport) AddTrackToStreamAllocator(subTrack types.SubscribedTrack) { if t.streamAllocator == nil { return @@ -1322,9 +1155,7 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) { // when there is no previous answer, cannot migrate, force a full reconnect if answer == nil { - if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { - onNegotiationFailed() - } + t.params.Handler.OnNegotiationFailed() return } @@ -1335,9 +1166,7 @@ func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) { t.params.Logger.Errorw("initPCWithPreviousAnswer failed", err) t.lock.Unlock() - if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { - onNegotiationFailed() - } + t.params.Handler.OnNegotiationFailed() return } else if offer != nil { // in migration case, can't reuse transceiver before negotiated except track subscribed at previous node @@ -1384,9 +1213,7 @@ func (t *PCTransport) postEvent(event event) { if err != nil { if !t.isClosed.Load() { t.params.Logger.Errorw("error handling event", err, "event", event.String()) - if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { - onNegotiationFailed() - } + t.params.Handler.OnNegotiationFailed() } } }) @@ -1455,17 +1282,12 @@ func (t *PCTransport) localDescriptionSent() error { cachedLocalCandidates := t.cachedLocalCandidates t.cachedLocalCandidates = nil - if onICECandidate := t.getOnICECandidate(); onICECandidate != nil { - for _, c := range cachedLocalCandidates { - if err := onICECandidate(c); err != nil { - return err - } + for _, c := range cachedLocalCandidates { + if err := t.params.Handler.OnICECandidate(c, t.params.Transport); err != nil { + return err } - - return nil } - - return ErrNoICECandidateHandler + return nil } func (t *PCTransport) clearLocalDescriptionSent() { @@ -1498,11 +1320,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { return nil } - if onICECandidate := t.getOnICECandidate(); onICECandidate != nil { - return onICECandidate(c) - } - - return ErrNoICECandidateHandler + return t.params.Handler.OnICECandidate(c, t.params.Transport) } func (t *PCTransport) handleRemoteICECandidate(e *event) error { @@ -1531,7 +1349,7 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { return nil } -func (t *PCTransport) setNegotiationState(state NegotiationState) { +func (t *PCTransport) setNegotiationState(state transport.NegotiationState) { t.negotiationState = state if onNegotiationStateChanged := t.getOnNegotiationStateChanged(); onNegotiationStateChanged != nil { onNegotiationStateChanged(t.negotiationState) @@ -1592,7 +1410,7 @@ func (t *PCTransport) setupSignalStateCheckTimer() { t.signalStateCheckTimer = time.AfterFunc(negotiationFailedTimeout, func() { t.clearSignalStateCheckTimer() - failed := t.negotiationState != NegotiationStateNone + failed := t.negotiationState != transport.NegotiationStateNone if t.negotiateCounter.Load() == negotiateVersion && failed && t.pc.ConnectionState() == webrtc.PeerConnectionStateConnected { t.params.Logger.Infow( @@ -1602,9 +1420,7 @@ func (t *PCTransport) setupSignalStateCheckTimer() { "remoteCurrent", t.pc.CurrentRemoteDescription(), "remotePending", t.pc.PendingRemoteDescription(), ) - if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { - onNegotiationFailed() - } + t.params.Handler.OnNegotiationFailed() } }) } @@ -1616,11 +1432,11 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { } // when there's an ongoing negotiation, let it finish and not disrupt its state - if t.negotiationState == NegotiationStateRemote { + if t.negotiationState == transport.NegotiationStateRemote { t.params.Logger.Debugw("skipping negotiation, trying again later") - t.setNegotiationState(NegotiationStateRetry) + t.setNegotiationState(transport.NegotiationStateRetry) return nil - } else if t.negotiationState == NegotiationStateRetry { + } else if t.negotiationState == transport.NegotiationStateRetry { // already set to retry, we can safely skip this attempt return nil } @@ -1690,21 +1506,17 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { } // indicate waiting for remote - t.setNegotiationState(NegotiationStateRemote) + t.setNegotiationState(transport.NegotiationStateRemote) t.setupSignalStateCheckTimer() - if onOffer := t.getOnOffer(); onOffer != nil { - if err := onOffer(offer); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) - return errors.Wrap(err, "could not send offer") - } - - prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) - return t.localDescriptionSent() + if err := t.params.Handler.OnOffer(offer); err != nil { + prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) + return errors.Wrap(err, "could not send offer") } - return ErrNoOfferHandler + prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) + return t.localDescriptionSent() } func (t *PCTransport) handleSendOffer(_ *event) error { @@ -1811,17 +1623,13 @@ func (t *PCTransport) createAndSendAnswer() error { t.params.Logger.Debugw("local answer (filtered)", "sdp", answer.SDP) } - if onAnswer := t.getOnAnswer(); onAnswer != nil { - if err := onAnswer(answer); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("answer", "error", "write_message").Add(1) - return errors.Wrap(err, "could not send answer") - } - - prometheus.ServiceOperationCounter.WithLabelValues("answer", "success", "").Add(1) - return t.localDescriptionSent() + if err := t.params.Handler.OnAnswer(answer); err != nil { + prometheus.ServiceOperationCounter.WithLabelValues("answer", "error", "write_message").Add(1) + return errors.Wrap(err, "could not send answer") } - return ErrNoAnswerHandler + prometheus.ServiceOperationCounter.WithLabelValues("answer", "success", "").Add(1) + return t.localDescriptionSent() } func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) error { @@ -1868,14 +1676,14 @@ func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) } } - if t.negotiationState == NegotiationStateRetry { - t.setNegotiationState(NegotiationStateNone) + if t.negotiationState == transport.NegotiationStateRetry { + t.setNegotiationState(transport.NegotiationStateNone) t.params.Logger.Debugw("re-negotiate after receiving answer") return t.createAndSendOffer(nil) } - t.setNegotiationState(NegotiationStateNone) + t.setNegotiationState(transport.NegotiationStateNone) return nil } @@ -1896,7 +1704,7 @@ func (t *PCTransport) doICERestart() error { t.resetShortConn() } - if t.negotiationState == NegotiationStateNone { + if t.negotiationState == transport.NegotiationStateNone { return t.createAndSendOffer(&webrtc.OfferOptions{ICERestart: true}) } @@ -1910,18 +1718,15 @@ func (t *PCTransport) doICERestart() error { return ErrIceRestartWithoutLocalSDP } else { t.params.Logger.Infow("deferring ice restart to next offer") - t.setNegotiationState(NegotiationStateRetry) + t.setNegotiationState(transport.NegotiationStateRetry) t.restartAtNextOffer = true - if onOffer := t.getOnOffer(); onOffer != nil { - err := onOffer(*offer) - if err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) - } else { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) - } - return err + err := t.params.Handler.OnOffer(*offer) + if err != nil { + prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) + } else { + prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) } - return ErrNoOfferHandler + return err } } else { // recover by re-applying the last answer @@ -1930,7 +1735,7 @@ func (t *PCTransport) doICERestart() error { prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "remote_description").Add(1) return errors.Wrap(err, "set remote description failed") } else { - t.setNegotiationState(NegotiationStateNone) + t.setNegotiationState(transport.NegotiationStateNone) return t.createAndSendOffer(&webrtc.OfferOptions{ICERestart: true}) } } diff --git a/pkg/rtc/transport/handler.go b/pkg/rtc/transport/handler.go new file mode 100644 index 000000000..eab3de349 --- /dev/null +++ b/pkg/rtc/transport/handler.go @@ -0,0 +1,55 @@ +package transport + +import ( + "errors" + + "github.com/pion/webrtc/v3" + + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" + "github.com/livekit/protocol/livekit" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +var ( + ErrNoICECandidateHandler = errors.New("no ICE candidate handler") + ErrNoOfferHandler = errors.New("no offer handler") + ErrNoAnswerHandler = errors.New("no answer handler") +) + +//counterfeiter:generate . Handler +type Handler interface { + OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error + OnInitialConnected() + OnFullyEstablished() + OnFailed(isShortLived bool) + OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) + OnDataPacket(kind livekit.DataPacket_Kind, data []byte) + OnOffer(sd webrtc.SessionDescription) error + OnAnswer(sd webrtc.SessionDescription) error + OnNegotiationStateChanged(state NegotiationState) + OnNegotiationFailed() + OnStreamStateChange(update *streamallocator.StreamStateUpdate) error +} + +type UnimplementedHandler struct{} + +func (h UnimplementedHandler) OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { + return ErrNoICECandidateHandler +} +func (h UnimplementedHandler) OnInitialConnected() {} +func (h UnimplementedHandler) OnFullyEstablished() {} +func (h UnimplementedHandler) OnFailed(isShortLived bool) {} +func (h UnimplementedHandler) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) {} +func (h UnimplementedHandler) OnDataPacket(kind livekit.DataPacket_Kind, data []byte) {} +func (h UnimplementedHandler) OnOffer(sd webrtc.SessionDescription) error { + return ErrNoOfferHandler +} +func (h UnimplementedHandler) OnAnswer(sd webrtc.SessionDescription) error { + return ErrNoAnswerHandler +} +func (h UnimplementedHandler) OnNegotiationStateChanged(state NegotiationState) {} +func (h UnimplementedHandler) OnNegotiationFailed() {} +func (h UnimplementedHandler) OnStreamStateChange(update *streamallocator.StreamStateUpdate) error { + return nil +} diff --git a/pkg/rtc/transport/negotiationstate.go b/pkg/rtc/transport/negotiationstate.go new file mode 100644 index 000000000..8f074f83f --- /dev/null +++ b/pkg/rtc/transport/negotiationstate.go @@ -0,0 +1,26 @@ +package transport + +import "fmt" + +type NegotiationState int + +const ( + NegotiationStateNone NegotiationState = iota + // waiting for remote description + NegotiationStateRemote + // need to Negotiate again + NegotiationStateRetry +) + +func (n NegotiationState) String() string { + switch n { + case NegotiationStateNone: + return "NONE" + case NegotiationStateRemote: + return "WAITING_FOR_REMOTE" + case NegotiationStateRetry: + return "RETRY" + default: + return fmt.Sprintf("%d", int(n)) + } +} diff --git a/pkg/rtc/transport/transportfakes/fake_handler.go b/pkg/rtc/transport/transportfakes/fake_handler.go new file mode 100644 index 000000000..dc7ad3c3e --- /dev/null +++ b/pkg/rtc/transport/transportfakes/fake_handler.go @@ -0,0 +1,593 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package transportfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/transport" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" + "github.com/livekit/protocol/livekit" + webrtc "github.com/pion/webrtc/v3" +) + +type FakeHandler struct { + OnAnswerStub func(webrtc.SessionDescription) error + onAnswerMutex sync.RWMutex + onAnswerArgsForCall []struct { + arg1 webrtc.SessionDescription + } + onAnswerReturns struct { + result1 error + } + onAnswerReturnsOnCall map[int]struct { + result1 error + } + OnDataPacketStub func(livekit.DataPacket_Kind, []byte) + onDataPacketMutex sync.RWMutex + onDataPacketArgsForCall []struct { + arg1 livekit.DataPacket_Kind + arg2 []byte + } + OnFailedStub func(bool) + onFailedMutex sync.RWMutex + onFailedArgsForCall []struct { + arg1 bool + } + OnFullyEstablishedStub func() + onFullyEstablishedMutex sync.RWMutex + onFullyEstablishedArgsForCall []struct { + } + OnICECandidateStub func(*webrtc.ICECandidate, livekit.SignalTarget) error + onICECandidateMutex sync.RWMutex + onICECandidateArgsForCall []struct { + arg1 *webrtc.ICECandidate + arg2 livekit.SignalTarget + } + onICECandidateReturns struct { + result1 error + } + onICECandidateReturnsOnCall map[int]struct { + result1 error + } + OnInitialConnectedStub func() + onInitialConnectedMutex sync.RWMutex + onInitialConnectedArgsForCall []struct { + } + OnNegotiationFailedStub func() + onNegotiationFailedMutex sync.RWMutex + onNegotiationFailedArgsForCall []struct { + } + OnNegotiationStateChangedStub func(transport.NegotiationState) + onNegotiationStateChangedMutex sync.RWMutex + onNegotiationStateChangedArgsForCall []struct { + arg1 transport.NegotiationState + } + OnOfferStub func(webrtc.SessionDescription) error + onOfferMutex sync.RWMutex + onOfferArgsForCall []struct { + arg1 webrtc.SessionDescription + } + onOfferReturns struct { + result1 error + } + onOfferReturnsOnCall map[int]struct { + result1 error + } + OnStreamStateChangeStub func(*streamallocator.StreamStateUpdate) error + onStreamStateChangeMutex sync.RWMutex + onStreamStateChangeArgsForCall []struct { + arg1 *streamallocator.StreamStateUpdate + } + onStreamStateChangeReturns struct { + result1 error + } + onStreamStateChangeReturnsOnCall map[int]struct { + result1 error + } + OnTrackStub func(*webrtc.TrackRemote, *webrtc.RTPReceiver) + onTrackMutex sync.RWMutex + onTrackArgsForCall []struct { + arg1 *webrtc.TrackRemote + arg2 *webrtc.RTPReceiver + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeHandler) OnAnswer(arg1 webrtc.SessionDescription) error { + fake.onAnswerMutex.Lock() + ret, specificReturn := fake.onAnswerReturnsOnCall[len(fake.onAnswerArgsForCall)] + fake.onAnswerArgsForCall = append(fake.onAnswerArgsForCall, struct { + arg1 webrtc.SessionDescription + }{arg1}) + stub := fake.OnAnswerStub + fakeReturns := fake.onAnswerReturns + fake.recordInvocation("OnAnswer", []interface{}{arg1}) + fake.onAnswerMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeHandler) OnAnswerCallCount() int { + fake.onAnswerMutex.RLock() + defer fake.onAnswerMutex.RUnlock() + return len(fake.onAnswerArgsForCall) +} + +func (fake *FakeHandler) OnAnswerCalls(stub func(webrtc.SessionDescription) error) { + fake.onAnswerMutex.Lock() + defer fake.onAnswerMutex.Unlock() + fake.OnAnswerStub = stub +} + +func (fake *FakeHandler) OnAnswerArgsForCall(i int) webrtc.SessionDescription { + fake.onAnswerMutex.RLock() + defer fake.onAnswerMutex.RUnlock() + argsForCall := fake.onAnswerArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeHandler) OnAnswerReturns(result1 error) { + fake.onAnswerMutex.Lock() + defer fake.onAnswerMutex.Unlock() + fake.OnAnswerStub = nil + fake.onAnswerReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnAnswerReturnsOnCall(i int, result1 error) { + fake.onAnswerMutex.Lock() + defer fake.onAnswerMutex.Unlock() + fake.OnAnswerStub = nil + if fake.onAnswerReturnsOnCall == nil { + fake.onAnswerReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onAnswerReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnDataPacket(arg1 livekit.DataPacket_Kind, arg2 []byte) { + var arg2Copy []byte + if arg2 != nil { + arg2Copy = make([]byte, len(arg2)) + copy(arg2Copy, arg2) + } + fake.onDataPacketMutex.Lock() + fake.onDataPacketArgsForCall = append(fake.onDataPacketArgsForCall, struct { + arg1 livekit.DataPacket_Kind + arg2 []byte + }{arg1, arg2Copy}) + stub := fake.OnDataPacketStub + fake.recordInvocation("OnDataPacket", []interface{}{arg1, arg2Copy}) + fake.onDataPacketMutex.Unlock() + if stub != nil { + fake.OnDataPacketStub(arg1, arg2) + } +} + +func (fake *FakeHandler) OnDataPacketCallCount() int { + fake.onDataPacketMutex.RLock() + defer fake.onDataPacketMutex.RUnlock() + return len(fake.onDataPacketArgsForCall) +} + +func (fake *FakeHandler) OnDataPacketCalls(stub func(livekit.DataPacket_Kind, []byte)) { + fake.onDataPacketMutex.Lock() + defer fake.onDataPacketMutex.Unlock() + fake.OnDataPacketStub = stub +} + +func (fake *FakeHandler) OnDataPacketArgsForCall(i int) (livekit.DataPacket_Kind, []byte) { + fake.onDataPacketMutex.RLock() + defer fake.onDataPacketMutex.RUnlock() + argsForCall := fake.onDataPacketArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeHandler) OnFailed(arg1 bool) { + fake.onFailedMutex.Lock() + fake.onFailedArgsForCall = append(fake.onFailedArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.OnFailedStub + fake.recordInvocation("OnFailed", []interface{}{arg1}) + fake.onFailedMutex.Unlock() + if stub != nil { + fake.OnFailedStub(arg1) + } +} + +func (fake *FakeHandler) OnFailedCallCount() int { + fake.onFailedMutex.RLock() + defer fake.onFailedMutex.RUnlock() + return len(fake.onFailedArgsForCall) +} + +func (fake *FakeHandler) OnFailedCalls(stub func(bool)) { + fake.onFailedMutex.Lock() + defer fake.onFailedMutex.Unlock() + fake.OnFailedStub = stub +} + +func (fake *FakeHandler) OnFailedArgsForCall(i int) bool { + fake.onFailedMutex.RLock() + defer fake.onFailedMutex.RUnlock() + argsForCall := fake.onFailedArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeHandler) OnFullyEstablished() { + fake.onFullyEstablishedMutex.Lock() + fake.onFullyEstablishedArgsForCall = append(fake.onFullyEstablishedArgsForCall, struct { + }{}) + stub := fake.OnFullyEstablishedStub + fake.recordInvocation("OnFullyEstablished", []interface{}{}) + fake.onFullyEstablishedMutex.Unlock() + if stub != nil { + fake.OnFullyEstablishedStub() + } +} + +func (fake *FakeHandler) OnFullyEstablishedCallCount() int { + fake.onFullyEstablishedMutex.RLock() + defer fake.onFullyEstablishedMutex.RUnlock() + return len(fake.onFullyEstablishedArgsForCall) +} + +func (fake *FakeHandler) OnFullyEstablishedCalls(stub func()) { + fake.onFullyEstablishedMutex.Lock() + defer fake.onFullyEstablishedMutex.Unlock() + fake.OnFullyEstablishedStub = stub +} + +func (fake *FakeHandler) OnICECandidate(arg1 *webrtc.ICECandidate, arg2 livekit.SignalTarget) error { + fake.onICECandidateMutex.Lock() + ret, specificReturn := fake.onICECandidateReturnsOnCall[len(fake.onICECandidateArgsForCall)] + fake.onICECandidateArgsForCall = append(fake.onICECandidateArgsForCall, struct { + arg1 *webrtc.ICECandidate + arg2 livekit.SignalTarget + }{arg1, arg2}) + stub := fake.OnICECandidateStub + fakeReturns := fake.onICECandidateReturns + fake.recordInvocation("OnICECandidate", []interface{}{arg1, arg2}) + fake.onICECandidateMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeHandler) OnICECandidateCallCount() int { + fake.onICECandidateMutex.RLock() + defer fake.onICECandidateMutex.RUnlock() + return len(fake.onICECandidateArgsForCall) +} + +func (fake *FakeHandler) OnICECandidateCalls(stub func(*webrtc.ICECandidate, livekit.SignalTarget) error) { + fake.onICECandidateMutex.Lock() + defer fake.onICECandidateMutex.Unlock() + fake.OnICECandidateStub = stub +} + +func (fake *FakeHandler) OnICECandidateArgsForCall(i int) (*webrtc.ICECandidate, livekit.SignalTarget) { + fake.onICECandidateMutex.RLock() + defer fake.onICECandidateMutex.RUnlock() + argsForCall := fake.onICECandidateArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeHandler) OnICECandidateReturns(result1 error) { + fake.onICECandidateMutex.Lock() + defer fake.onICECandidateMutex.Unlock() + fake.OnICECandidateStub = nil + fake.onICECandidateReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnICECandidateReturnsOnCall(i int, result1 error) { + fake.onICECandidateMutex.Lock() + defer fake.onICECandidateMutex.Unlock() + fake.OnICECandidateStub = nil + if fake.onICECandidateReturnsOnCall == nil { + fake.onICECandidateReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onICECandidateReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnInitialConnected() { + fake.onInitialConnectedMutex.Lock() + fake.onInitialConnectedArgsForCall = append(fake.onInitialConnectedArgsForCall, struct { + }{}) + stub := fake.OnInitialConnectedStub + fake.recordInvocation("OnInitialConnected", []interface{}{}) + fake.onInitialConnectedMutex.Unlock() + if stub != nil { + fake.OnInitialConnectedStub() + } +} + +func (fake *FakeHandler) OnInitialConnectedCallCount() int { + fake.onInitialConnectedMutex.RLock() + defer fake.onInitialConnectedMutex.RUnlock() + return len(fake.onInitialConnectedArgsForCall) +} + +func (fake *FakeHandler) OnInitialConnectedCalls(stub func()) { + fake.onInitialConnectedMutex.Lock() + defer fake.onInitialConnectedMutex.Unlock() + fake.OnInitialConnectedStub = stub +} + +func (fake *FakeHandler) OnNegotiationFailed() { + fake.onNegotiationFailedMutex.Lock() + fake.onNegotiationFailedArgsForCall = append(fake.onNegotiationFailedArgsForCall, struct { + }{}) + stub := fake.OnNegotiationFailedStub + fake.recordInvocation("OnNegotiationFailed", []interface{}{}) + fake.onNegotiationFailedMutex.Unlock() + if stub != nil { + fake.OnNegotiationFailedStub() + } +} + +func (fake *FakeHandler) OnNegotiationFailedCallCount() int { + fake.onNegotiationFailedMutex.RLock() + defer fake.onNegotiationFailedMutex.RUnlock() + return len(fake.onNegotiationFailedArgsForCall) +} + +func (fake *FakeHandler) OnNegotiationFailedCalls(stub func()) { + fake.onNegotiationFailedMutex.Lock() + defer fake.onNegotiationFailedMutex.Unlock() + fake.OnNegotiationFailedStub = stub +} + +func (fake *FakeHandler) OnNegotiationStateChanged(arg1 transport.NegotiationState) { + fake.onNegotiationStateChangedMutex.Lock() + fake.onNegotiationStateChangedArgsForCall = append(fake.onNegotiationStateChangedArgsForCall, struct { + arg1 transport.NegotiationState + }{arg1}) + stub := fake.OnNegotiationStateChangedStub + fake.recordInvocation("OnNegotiationStateChanged", []interface{}{arg1}) + fake.onNegotiationStateChangedMutex.Unlock() + if stub != nil { + fake.OnNegotiationStateChangedStub(arg1) + } +} + +func (fake *FakeHandler) OnNegotiationStateChangedCallCount() int { + fake.onNegotiationStateChangedMutex.RLock() + defer fake.onNegotiationStateChangedMutex.RUnlock() + return len(fake.onNegotiationStateChangedArgsForCall) +} + +func (fake *FakeHandler) OnNegotiationStateChangedCalls(stub func(transport.NegotiationState)) { + fake.onNegotiationStateChangedMutex.Lock() + defer fake.onNegotiationStateChangedMutex.Unlock() + fake.OnNegotiationStateChangedStub = stub +} + +func (fake *FakeHandler) OnNegotiationStateChangedArgsForCall(i int) transport.NegotiationState { + fake.onNegotiationStateChangedMutex.RLock() + defer fake.onNegotiationStateChangedMutex.RUnlock() + argsForCall := fake.onNegotiationStateChangedArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeHandler) OnOffer(arg1 webrtc.SessionDescription) error { + fake.onOfferMutex.Lock() + ret, specificReturn := fake.onOfferReturnsOnCall[len(fake.onOfferArgsForCall)] + fake.onOfferArgsForCall = append(fake.onOfferArgsForCall, struct { + arg1 webrtc.SessionDescription + }{arg1}) + stub := fake.OnOfferStub + fakeReturns := fake.onOfferReturns + fake.recordInvocation("OnOffer", []interface{}{arg1}) + fake.onOfferMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeHandler) OnOfferCallCount() int { + fake.onOfferMutex.RLock() + defer fake.onOfferMutex.RUnlock() + return len(fake.onOfferArgsForCall) +} + +func (fake *FakeHandler) OnOfferCalls(stub func(webrtc.SessionDescription) error) { + fake.onOfferMutex.Lock() + defer fake.onOfferMutex.Unlock() + fake.OnOfferStub = stub +} + +func (fake *FakeHandler) OnOfferArgsForCall(i int) webrtc.SessionDescription { + fake.onOfferMutex.RLock() + defer fake.onOfferMutex.RUnlock() + argsForCall := fake.onOfferArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeHandler) OnOfferReturns(result1 error) { + fake.onOfferMutex.Lock() + defer fake.onOfferMutex.Unlock() + fake.OnOfferStub = nil + fake.onOfferReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnOfferReturnsOnCall(i int, result1 error) { + fake.onOfferMutex.Lock() + defer fake.onOfferMutex.Unlock() + fake.OnOfferStub = nil + if fake.onOfferReturnsOnCall == nil { + fake.onOfferReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onOfferReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnStreamStateChange(arg1 *streamallocator.StreamStateUpdate) error { + fake.onStreamStateChangeMutex.Lock() + ret, specificReturn := fake.onStreamStateChangeReturnsOnCall[len(fake.onStreamStateChangeArgsForCall)] + fake.onStreamStateChangeArgsForCall = append(fake.onStreamStateChangeArgsForCall, struct { + arg1 *streamallocator.StreamStateUpdate + }{arg1}) + stub := fake.OnStreamStateChangeStub + fakeReturns := fake.onStreamStateChangeReturns + fake.recordInvocation("OnStreamStateChange", []interface{}{arg1}) + fake.onStreamStateChangeMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeHandler) OnStreamStateChangeCallCount() int { + fake.onStreamStateChangeMutex.RLock() + defer fake.onStreamStateChangeMutex.RUnlock() + return len(fake.onStreamStateChangeArgsForCall) +} + +func (fake *FakeHandler) OnStreamStateChangeCalls(stub func(*streamallocator.StreamStateUpdate) error) { + fake.onStreamStateChangeMutex.Lock() + defer fake.onStreamStateChangeMutex.Unlock() + fake.OnStreamStateChangeStub = stub +} + +func (fake *FakeHandler) OnStreamStateChangeArgsForCall(i int) *streamallocator.StreamStateUpdate { + fake.onStreamStateChangeMutex.RLock() + defer fake.onStreamStateChangeMutex.RUnlock() + argsForCall := fake.onStreamStateChangeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeHandler) OnStreamStateChangeReturns(result1 error) { + fake.onStreamStateChangeMutex.Lock() + defer fake.onStreamStateChangeMutex.Unlock() + fake.OnStreamStateChangeStub = nil + fake.onStreamStateChangeReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnStreamStateChangeReturnsOnCall(i int, result1 error) { + fake.onStreamStateChangeMutex.Lock() + defer fake.onStreamStateChangeMutex.Unlock() + fake.OnStreamStateChangeStub = nil + if fake.onStreamStateChangeReturnsOnCall == nil { + fake.onStreamStateChangeReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onStreamStateChangeReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnTrack(arg1 *webrtc.TrackRemote, arg2 *webrtc.RTPReceiver) { + fake.onTrackMutex.Lock() + fake.onTrackArgsForCall = append(fake.onTrackArgsForCall, struct { + arg1 *webrtc.TrackRemote + arg2 *webrtc.RTPReceiver + }{arg1, arg2}) + stub := fake.OnTrackStub + fake.recordInvocation("OnTrack", []interface{}{arg1, arg2}) + fake.onTrackMutex.Unlock() + if stub != nil { + fake.OnTrackStub(arg1, arg2) + } +} + +func (fake *FakeHandler) OnTrackCallCount() int { + fake.onTrackMutex.RLock() + defer fake.onTrackMutex.RUnlock() + return len(fake.onTrackArgsForCall) +} + +func (fake *FakeHandler) OnTrackCalls(stub func(*webrtc.TrackRemote, *webrtc.RTPReceiver)) { + fake.onTrackMutex.Lock() + defer fake.onTrackMutex.Unlock() + fake.OnTrackStub = stub +} + +func (fake *FakeHandler) OnTrackArgsForCall(i int) (*webrtc.TrackRemote, *webrtc.RTPReceiver) { + fake.onTrackMutex.RLock() + defer fake.onTrackMutex.RUnlock() + argsForCall := fake.onTrackArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeHandler) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.onAnswerMutex.RLock() + defer fake.onAnswerMutex.RUnlock() + fake.onDataPacketMutex.RLock() + defer fake.onDataPacketMutex.RUnlock() + fake.onFailedMutex.RLock() + defer fake.onFailedMutex.RUnlock() + fake.onFullyEstablishedMutex.RLock() + defer fake.onFullyEstablishedMutex.RUnlock() + fake.onICECandidateMutex.RLock() + defer fake.onICECandidateMutex.RUnlock() + fake.onInitialConnectedMutex.RLock() + defer fake.onInitialConnectedMutex.RUnlock() + fake.onNegotiationFailedMutex.RLock() + defer fake.onNegotiationFailedMutex.RUnlock() + fake.onNegotiationStateChangedMutex.RLock() + defer fake.onNegotiationStateChangedMutex.RUnlock() + fake.onOfferMutex.RLock() + defer fake.onOfferMutex.RUnlock() + fake.onStreamStateChangeMutex.RLock() + defer fake.onStreamStateChangeMutex.RUnlock() + fake.onTrackMutex.RLock() + defer fake.onTrackMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeHandler) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ transport.Handler = new(FakeHandler) diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index e66670ea5..7bd1067b3 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" + "github.com/livekit/livekit-server/pkg/rtc/transport" + "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" "github.com/livekit/livekit-server/pkg/testutils" "github.com/livekit/protocol/livekit" ) @@ -37,33 +39,39 @@ func TestMissingAnswerDuringICERestart(t *testing.T) { Config: &WebRTCConfig{}, IsOfferer: true, } - transportA, err := NewPCTransport(params) + + paramsA := params + handlerA := &transportfakes.FakeHandler{} + paramsA.Handler = handlerA + transportA, err := NewPCTransport(paramsA) require.NoError(t, err) _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) paramsB := params + handlerB := &transportfakes.FakeHandler{} + paramsB.Handler = handlerB paramsB.IsOfferer = false transportB, err := NewPCTransport(paramsB) require.NoError(t, err) // exchange ICE - handleICEExchange(t, transportA, transportB) + handleICEExchange(t, transportA, transportB, handlerA, handlerB) - connectTransports(t, transportA, transportB, false, 1, 1) + connectTransports(t, transportA, transportB, handlerA, handlerB, false, 1, 1) require.Equal(t, webrtc.ICEConnectionStateConnected, transportA.pc.ICEConnectionState()) require.Equal(t, webrtc.ICEConnectionStateConnected, transportB.pc.ICEConnectionState()) var negotiationState atomic.Value - transportA.OnNegotiationStateChanged(func(state NegotiationState) { + transportA.OnNegotiationStateChanged(func(state transport.NegotiationState) { negotiationState.Store(state) }) // offer again, but missed var offerReceived atomic.Bool - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { require.Equal(t, webrtc.SignalingStateHaveLocalOffer, transportA.pc.SignalingState()) - require.Equal(t, NegotiationStateRemote, negotiationState.Load().(NegotiationState)) + require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState)) offerReceived.Store(true) return nil }) @@ -72,7 +80,7 @@ func TestMissingAnswerDuringICERestart(t *testing.T) { return offerReceived.Load() }, 10*time.Second, time.Millisecond*10, "transportA offer not received") - connectTransports(t, transportA, transportB, true, 1, 1) + connectTransports(t, transportA, transportB, handlerA, handlerB, true, 1, 1) require.Equal(t, webrtc.ICEConnectionStateConnected, transportA.pc.ICEConnectionState()) require.Equal(t, webrtc.ICEConnectionStateConnected, transportB.pc.ICEConnectionState()) @@ -87,70 +95,76 @@ func TestNegotiationTiming(t *testing.T) { Config: &WebRTCConfig{}, IsOfferer: true, } - transportA, err := NewPCTransport(params) + + paramsA := params + handlerA := &transportfakes.FakeHandler{} + paramsA.Handler = handlerA + transportA, err := NewPCTransport(paramsA) require.NoError(t, err) _, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil) require.NoError(t, err) paramsB := params + handlerB := &transportfakes.FakeHandler{} + paramsB.Handler = handlerB paramsB.IsOfferer = false - transportB, err := NewPCTransport(params) + transportB, err := NewPCTransport(paramsB) require.NoError(t, err) require.False(t, transportA.IsEstablished()) require.False(t, transportB.IsEstablished()) - handleICEExchange(t, transportA, transportB) + handleICEExchange(t, transportA, transportB, handlerA, handlerB) offer := atomic.Value{} - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { offer.Store(&sd) return nil }) var negotiationState atomic.Value - transportA.OnNegotiationStateChanged(func(state NegotiationState) { + transportA.OnNegotiationStateChanged(func(state transport.NegotiationState) { negotiationState.Store(state) }) // initial offer transportA.Negotiate(true) require.Eventually(t, func() bool { - state, ok := negotiationState.Load().(NegotiationState) + state, ok := negotiationState.Load().(transport.NegotiationState) if !ok { return false } - return state == NegotiationStateRemote + return state == transport.NegotiationStateRemote }, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRemote") // second try, should've flipped transport status to retry transportA.Negotiate(true) require.Eventually(t, func() bool { - state, ok := negotiationState.Load().(NegotiationState) + state, ok := negotiationState.Load().(transport.NegotiationState) if !ok { return false } - return state == NegotiationStateRetry + return state == transport.NegotiationStateRetry }, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRetry") // third try, should've stayed at retry transportA.Negotiate(true) time.Sleep(100 * time.Millisecond) // some time to process the negotiate event require.Eventually(t, func() bool { - state, ok := negotiationState.Load().(NegotiationState) + state, ok := negotiationState.Load().(transport.NegotiationState) if !ok { return false } - return state == NegotiationStateRetry + return state == transport.NegotiationStateRetry }, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRetry") time.Sleep(5 * time.Millisecond) actualOffer, ok := offer.Load().(*webrtc.SessionDescription) require.True(t, ok) - transportB.OnAnswer(func(answer webrtc.SessionDescription) error { + handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error { transportA.HandleRemoteDescription(answer) return nil }) @@ -164,7 +178,7 @@ func TestNegotiationTiming(t *testing.T) { }, 10*time.Second, time.Millisecond*10, "transportB is not established") // it should still be negotiating again - require.Equal(t, NegotiationStateRemote, negotiationState.Load().(NegotiationState)) + require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState)) offer2, ok := offer.Load().(*webrtc.SessionDescription) require.True(t, ok) require.False(t, offer2 == actualOffer) @@ -180,22 +194,28 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { Config: &WebRTCConfig{}, IsOfferer: true, } - transportA, err := NewPCTransport(params) + + paramsA := params + handlerA := &transportfakes.FakeHandler{} + paramsA.Handler = handlerA + transportA, err := NewPCTransport(paramsA) require.NoError(t, err) _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) paramsB := params + handlerB := &transportfakes.FakeHandler{} + paramsB.Handler = handlerB paramsB.IsOfferer = false transportB, err := NewPCTransport(paramsB) require.NoError(t, err) // exchange ICE - handleICEExchange(t, transportA, transportB) + handleICEExchange(t, transportA, transportB, handlerA, handlerB) // first offer missed var firstOfferReceived atomic.Bool - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { firstOfferReceived.Store(true) return nil }) @@ -207,13 +227,13 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { // set offer/answer with restart ICE, will negotiate twice, // first one is recover from missed offer // second one is restartICE - transportB.OnAnswer(func(answer webrtc.SessionDescription) error { + handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error { transportA.HandleRemoteDescription(answer) return nil }) var offerCount atomic.Int32 - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { offerCount.Inc() // the second offer is a ice restart offer, so we wait transportB complete the ice gathering @@ -248,22 +268,28 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { Config: &WebRTCConfig{}, IsOfferer: true, } - transportA, err := NewPCTransport(params) + + paramsA := params + handlerA := &transportfakes.FakeHandler{} + paramsA.Handler = handlerA + transportA, err := NewPCTransport(paramsA) require.NoError(t, err) _, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil) require.NoError(t, err) paramsB := params + handlerB := &transportfakes.FakeHandler{} + paramsB.Handler = handlerB paramsB.IsOfferer = false transportB, err := NewPCTransport(paramsB) require.NoError(t, err) // exchange ICE - handleICEExchange(t, transportA, transportB) + handleICEExchange(t, transportA, transportB, handlerA, handlerB) // first answer missed var firstAnswerReceived atomic.Bool - transportB.OnAnswer(func(sd webrtc.SessionDescription) error { + handlerB.OnAnswerCalls(func(sd webrtc.SessionDescription) error { if firstAnswerReceived.Load() { transportA.HandleRemoteDescription(sd) } else { @@ -272,7 +298,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { } return nil }) - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { transportB.HandleRemoteDescription(sd) return nil }) @@ -286,7 +312,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { // first one is recover from missed offer // second one is restartICE var offerCount atomic.Int32 - transportA.OnOffer(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { offerCount.Inc() // the second offer is a ice restart offer, so we wait transportB complete the ice gathering @@ -321,26 +347,32 @@ func TestNegotiationFailed(t *testing.T) { Config: &WebRTCConfig{}, IsOfferer: true, } - transportA, err := NewPCTransport(params) + + paramsA := params + handlerA := &transportfakes.FakeHandler{} + paramsA.Handler = handlerA + transportA, err := NewPCTransport(paramsA) require.NoError(t, err) _, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil) require.NoError(t, err) paramsB := params + handlerB := &transportfakes.FakeHandler{} + paramsB.Handler = handlerB paramsB.IsOfferer = false transportB, err := NewPCTransport(paramsB) require.NoError(t, err) // exchange ICE - handleICEExchange(t, transportA, transportB) + handleICEExchange(t, transportA, transportB, handlerA, handlerB) // wait for transport to be connected before maiming the signalling channel - connectTransports(t, transportA, transportB, false, 1, 1) + connectTransports(t, transportA, transportB, handlerA, handlerB, false, 1, 1) // reset OnOffer to force a negotiation failure - transportA.OnOffer(func(sd webrtc.SessionDescription) error { return nil }) + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { return nil }) var failed atomic.Int32 - transportA.OnNegotiationFailed(func() { + handlerA.OnNegotiationFailedCalls(func() { failed.Inc() }) transportA.Negotiate(true) @@ -361,6 +393,7 @@ func TestFilteringCandidates(t *testing.T) { {Mime: webrtc.MimeTypeVP8}, {Mime: webrtc.MimeTypeH264}, }, + Handler: &transportfakes.FakeHandler{}, } transport, err := NewPCTransport(params) require.NoError(t, err) @@ -471,8 +504,8 @@ func TestFilteringCandidates(t *testing.T) { transport.Close() } -func handleICEExchange(t *testing.T, a, b *PCTransport) { - a.OnICECandidate(func(candidate *webrtc.ICECandidate) error { +func handleICEExchange(t *testing.T, a, b *PCTransport, ah, bh *transportfakes.FakeHandler) { + ah.OnICECandidateCalls(func(candidate *webrtc.ICECandidate, target livekit.SignalTarget) error { if candidate == nil { return nil } @@ -480,7 +513,7 @@ func handleICEExchange(t *testing.T, a, b *PCTransport) { b.AddICECandidate(candidate.ToJSON()) return nil }) - b.OnICECandidate(func(candidate *webrtc.ICECandidate) error { + bh.OnICECandidateCalls(func(candidate *webrtc.ICECandidate, target livekit.SignalTarget) error { if candidate == nil { return nil } @@ -490,16 +523,16 @@ func handleICEExchange(t *testing.T, a, b *PCTransport) { }) } -func connectTransports(t *testing.T, offerer, answerer *PCTransport, isICERestart bool, expectedOfferCount int32, expectedAnswerCount int32) { +func connectTransports(t *testing.T, offerer, answerer *PCTransport, offererHandler, answererHandler *transportfakes.FakeHandler, isICERestart bool, expectedOfferCount int32, expectedAnswerCount int32) { var offerCount atomic.Int32 var answerCount atomic.Int32 - answerer.OnAnswer(func(answer webrtc.SessionDescription) error { + answererHandler.OnAnswerCalls(func(answer webrtc.SessionDescription) error { answerCount.Inc() offerer.HandleRemoteDescription(answer) return nil }) - offerer.OnOffer(func(offer webrtc.SessionDescription) error { + offererHandler.OnOfferCalls(func(offer webrtc.SessionDescription) error { offerCount.Inc() answerer.HandleRemoteDescription(offer) return nil @@ -527,11 +560,11 @@ func connectTransports(t *testing.T, offerer, answerer *PCTransport, isICERestar return answerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected }, 10*time.Second, time.Millisecond*10, "answerer did not become connected") - transportsConnected := untilTransportsConnected(offerer, answerer) + transportsConnected := untilTransportsConnected(offererHandler, answererHandler) transportsConnected.Wait() } -func untilTransportsConnected(transports ...*PCTransport) *sync.WaitGroup { +func untilTransportsConnected(transports ...*transportfakes.FakeHandler) *sync.WaitGroup { var triggered sync.WaitGroup triggered.Add(len(transports)) @@ -545,7 +578,10 @@ func untilTransportsConnected(transports ...*PCTransport) *sync.WaitGroup { } } - t.OnInitialConnected(hdlr) + if t.OnInitialConnectedCallCount() != 0 { + hdlr() + } + t.OnInitialConnectedCalls(hdlr) } return &triggered } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 7207ace3e..401fea8e0 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -29,10 +29,10 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -47,6 +47,25 @@ const ( udpLossUnstableCountThreshold = 20 ) +type TransportManagerTransportHandler struct { + transport.Handler + t *TransportManager +} + +func (h TransportManagerTransportHandler) OnFailed(isShortLived bool) { + h.t.handleConnectionFailed(isShortLived) + h.Handler.OnFailed(isShortLived) +} + +type TransportManagerPublisherTransportHandler struct { + TransportManagerTransportHandler +} + +func (h TransportManagerPublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription) error { + h.t.lastPublisherAnswer.Store(sd) + return h.Handler.OnAnswer(sd) +} + type TransportManagerParams struct { Identity livekit.ParticipantIdentity SID livekit.ParticipantID @@ -66,6 +85,8 @@ type TransportManagerParams struct { AllowPlayoutDelay bool DataChannelMaxBufferedAmount uint64 Logger logger.Logger + PublisherHandler transport.Handler + SubscriberHandler transport.Handler } type TransportManager struct { @@ -91,11 +112,6 @@ type TransportManager struct { udpLossUnstableCount uint32 signalingRTT, udpRTT uint32 - onPublisherInitialConnected func() - onSubscriberInitialConnected func() - onPrimaryTransportInitialConnected func() - onAnyTransportFailed func() - onICEConfigChanged func(iceConfig *livekit.ICEConfig) } @@ -122,25 +138,12 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro SimTracks: params.SimTracks, ClientInfo: params.ClientInfo, Transport: livekit.SignalTarget_PUBLISHER, + Handler: TransportManagerPublisherTransportHandler{TransportManagerTransportHandler{params.PublisherHandler, t}}, }) if err != nil { return nil, err } t.publisher = publisher - t.publisher.OnInitialConnected(func() { - if t.onPublisherInitialConnected != nil { - t.onPublisherInitialConnected() - } - if !t.params.SubscriberAsPrimary && t.onPrimaryTransportInitialConnected != nil { - t.onPrimaryTransportInitialConnected() - } - }) - t.publisher.OnFailed(func(isShortLived bool) { - t.handleConnectionFailed(isShortLived) - if t.onAnyTransportFailed != nil { - t.onAnyTransportFailed() - } - }) subscriber, err := NewPCTransport(TransportParams{ ParticipantID: params.SID, @@ -157,25 +160,12 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro AllowPlayoutDelay: params.AllowPlayoutDelay, DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount, Transport: livekit.SignalTarget_SUBSCRIBER, + Handler: TransportManagerTransportHandler{params.SubscriberHandler, t}, }) if err != nil { return nil, err } t.subscriber = subscriber - t.subscriber.OnInitialConnected(func() { - if t.onSubscriberInitialConnected != nil { - t.onSubscriberInitialConnected() - } - if t.params.SubscriberAsPrimary && t.onPrimaryTransportInitialConnected != nil { - t.onPrimaryTransportInitialConnected() - } - }) - t.subscriber.OnFailed(func(isShortLived bool) { - t.handleConnectionFailed(isShortLived) - if t.onAnyTransportFailed != nil { - t.onAnyTransportFailed() - } - }) if !t.params.Migration { if err := t.createDataChannelsForSubscriber(nil); err != nil { return nil, err @@ -195,25 +185,6 @@ func (t *TransportManager) SubscriberClose() { t.subscriber.Close() } -func (t *TransportManager) OnPublisherICECandidate(f func(c *webrtc.ICECandidate) error) { - t.publisher.OnICECandidate(f) -} - -func (t *TransportManager) OnPublisherAnswer(f func(answer webrtc.SessionDescription) error) { - t.publisher.OnAnswer(func(sd webrtc.SessionDescription) error { - t.lastPublisherAnswer.Store(sd) - return f(sd) - }) -} - -func (t *TransportManager) OnPublisherInitialConnected(f func()) { - t.onPublisherInitialConnected = f -} - -func (t *TransportManager) OnPublisherTrack(f func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver)) { - t.publisher.OnTrack(f) -} - func (t *TransportManager) HasPublisherEverConnected() bool { return t.publisher.HasEverConnected() } @@ -234,22 +205,6 @@ func (t *TransportManager) WritePublisherRTCP(pkts []rtcp.Packet) error { return t.publisher.WriteRTCP(pkts) } -func (t *TransportManager) OnSubscriberICECandidate(f func(c *webrtc.ICECandidate) error) { - t.subscriber.OnICECandidate(f) -} - -func (t *TransportManager) OnSubscriberOffer(f func(offer webrtc.SessionDescription) error) { - t.subscriber.OnOffer(f) -} - -func (t *TransportManager) OnSubscriberInitialConnected(f func()) { - t.onSubscriberInitialConnected = f -} - -func (t *TransportManager) OnSubscriberStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { - t.subscriber.OnStreamStateChange(f) -} - func (t *TransportManager) HasSubscriberEverConnected() bool { return t.subscriber.HasEverConnected() } @@ -274,23 +229,6 @@ func (t *TransportManager) GetSubscriberPacer() pacer.Pacer { return t.subscriber.GetPacer() } -func (t *TransportManager) OnPrimaryTransportInitialConnected(f func()) { - t.onPrimaryTransportInitialConnected = f -} - -func (t *TransportManager) OnPrimaryTransportFullyEstablished(f func()) { - t.getTransport(true).OnFullyEstablished(f) -} - -func (t *TransportManager) OnAnyTransportFailed(f func()) { - t.onAnyTransportFailed = f -} - -func (t *TransportManager) OnAnyTransportNegotiationFailed(f func()) { - t.publisher.OnNegotiationFailed(f) - t.subscriber.OnNegotiationFailed(f) -} - func (t *TransportManager) AddSubscribedTrack(subTrack types.SubscribedTrack) { t.subscriber.AddTrackToStreamAllocator(subTrack) } @@ -299,11 +237,6 @@ func (t *TransportManager) RemoveSubscribedTrack(subTrack types.SubscribedTrack) t.subscriber.RemoveTrackFromStreamAllocator(subTrack) } -func (t *TransportManager) OnDataMessage(f func(kind livekit.DataPacket_Kind, data []byte)) { - // upstream data always comes in via publisher peer connection irrespective of which is primary - t.publisher.OnDataPacket(f) -} - func (t *TransportManager) SendDataPacket(dp *livekit.DataPacket, data []byte) error { // downstream data is sent via primary peer connection return t.getTransport(true).SendDataPacket(dp, data) @@ -736,10 +669,7 @@ func (t *TransportManager) onMediaLossUpdate(loss uint8) { t.lock.Unlock() t.params.Logger.Infow("udp connection unstable, switch to tcp", "signalingRTT", t.signalingRTT) - t.handleConnectionFailed(true) - if t.onAnyTransportFailed != nil { - t.onAnyTransportFailed() - } + t.params.SubscriberHandler.OnFailed(true) return } } diff --git a/test/client/client.go b/test/client/client.go index fbe6954d9..e0be22e47 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -39,6 +39,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" "github.com/livekit/livekit-server/pkg/rtc/types" ) @@ -199,33 +200,37 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { // i. e. the publisher transport on client side has SUBSCRIBER signal target (i. e. publisher is offerer). // Same applies for subscriber transport also // + publisherHandler := &transportfakes.FakeHandler{} c.publisher, err = rtc.NewPCTransport(rtc.TransportParams{ Config: &conf, DirectionConfig: conf.Subscriber, EnabledCodecs: codecs, IsOfferer: true, IsSendSide: true, + Handler: publisherHandler, }) if err != nil { return nil, err } + subscriberHandler := &transportfakes.FakeHandler{} c.subscriber, err = rtc.NewPCTransport(rtc.TransportParams{ Config: &conf, DirectionConfig: conf.Publisher, EnabledCodecs: codecs, + Handler: subscriberHandler, }) if err != nil { return nil, err } - c.publisher.OnICECandidate(func(ic *webrtc.ICECandidate) error { + publisherHandler.OnICECandidateCalls(func(ic *webrtc.ICECandidate, t livekit.SignalTarget) error { if ic == nil { return nil } return c.SendIceCandidate(ic, livekit.SignalTarget_PUBLISHER) }) - c.publisher.OnOffer(c.onOffer) - c.publisher.OnFullyEstablished(func() { + publisherHandler.OnOfferCalls(c.onOffer) + publisherHandler.OnFullyEstablishedCalls(func() { logger.Debugw("publisher fully established", "participant", c.localParticipant.Identity, "pID", c.localParticipant.Sid) c.publisherFullyEstablished.Store(true) }) @@ -245,17 +250,17 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { return nil, err } - c.subscriber.OnICECandidate(func(ic *webrtc.ICECandidate) error { + subscriberHandler.OnICECandidateCalls(func(ic *webrtc.ICECandidate, t livekit.SignalTarget) error { if ic == nil { return nil } return c.SendIceCandidate(ic, livekit.SignalTarget_SUBSCRIBER) }) - c.subscriber.OnTrack(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + subscriberHandler.OnTrackCalls(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { go c.processTrack(track) }) - c.subscriber.OnDataPacket(c.handleDataMessage) - c.subscriber.OnInitialConnected(func() { + subscriberHandler.OnDataPacketCalls(c.handleDataMessage) + subscriberHandler.OnInitialConnectedCalls(func() { logger.Debugw("subscriber initial connected", "participant", c.localParticipant.Identity) c.lock.Lock() @@ -272,11 +277,11 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { go c.OnConnected() } }) - c.subscriber.OnFullyEstablished(func() { + subscriberHandler.OnFullyEstablishedCalls(func() { logger.Debugw("subscriber fully established", "participant", c.localParticipant.Identity, "pID", c.localParticipant.Sid) c.subscriberFullyEstablished.Store(true) }) - c.subscriber.OnAnswer(func(answer webrtc.SessionDescription) error { + subscriberHandler.OnAnswerCalls(func(answer webrtc.SessionDescription) error { // send remote an answer logger.Infow("sending subscriber answer", "participant", c.localParticipant.Identity, From 846121e781e043b5955b8d49be5035f4c8457083 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 12:36:27 +0530 Subject: [PATCH 106/114] Revert "Cache data synchronously for processing in worker." (#2425) --- pkg/rtc/room.go | 84 +++++++++++++++----------------------- pkg/rtc/room_test.go | 2 +- pkg/service/roommanager.go | 8 ++-- 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index d95044ed9..745ae9f4f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -118,7 +118,7 @@ type Room struct { trailer []byte - onParticipantChanged func(p types.LocalParticipant, pi *livekit.ParticipantInfo) + onParticipantChanged func(p types.LocalParticipant) onRoomUpdated func() onClose func() @@ -347,13 +347,11 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me pw := r.addParticipantWorkerLocked(participant) participant.OnStateChange(func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { - ri := r.ToProto() - pi := p.ToProto() pw.eventsQueue.Enqueue(func() { if r.onParticipantChanged != nil { - r.onParticipantChanged(p, pi) + r.onParticipantChanged(p) } - r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) if state == livekit.ParticipantInfo_ACTIVE { // subscribe participant to existing published tracks @@ -370,8 +368,8 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me } } r.telemetry.ParticipantActive(context.Background(), - ri, - pi, + r.ToProto(), + p.ToProto(), meta, false, ) @@ -385,29 +383,23 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me }) // it's important to set this before connection, we don't want to miss out on any published tracks participant.OnTrackPublished(func(p types.LocalParticipant, t types.MediaTrack) { - pi := p.ToProto() - ti := t.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackPublished(p, pi, t, ti) + r.onTrackPublished(p, t) }) }) participant.OnTrackUpdated(func(p types.LocalParticipant, t types.MediaTrack) { - pi := p.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackUpdated(p, pi, t) + r.onTrackUpdated(p, t) }) }) participant.OnTrackUnpublished(func(p types.LocalParticipant, t types.MediaTrack) { - pi := p.ToProto() - ti := t.ToProto() pw.eventsQueue.Enqueue(func() { - r.onTrackUnpublished(p, pi, t, ti) + r.onTrackUnpublished(p, t) }) }) participant.OnParticipantUpdate(func(p types.LocalParticipant) { - pi := p.ToProto() pw.eventsQueue.Enqueue(func() { - r.onParticipantUpdate(p, pi) + r.onParticipantUpdate(p) }) }) participant.OnDataPacket(r.onDataPacket) @@ -468,7 +460,7 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me r.participantRequestSources[participant.Identity()] = requestSource if r.onParticipantChanged != nil { - r.onParticipantChanged(participant, participant.ToProto()) + r.onParticipantChanged(participant) } time.AfterFunc(time.Minute, func() { @@ -645,11 +637,10 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek r.leftAt.Store(time.Now().Unix()) if sendUpdates { - pi := p.ToProto() if r.onParticipantChanged != nil { - r.onParticipantChanged(p, pi) + r.onParticipantChanged(p) } - r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) } } @@ -835,7 +826,7 @@ func (r *Room) OnClose(f func()) { r.onClose = f } -func (r *Room) OnParticipantChanged(f func(p types.LocalParticipant, pi *livekit.ParticipantInfo)) { +func (r *Room) OnParticipantChanged(f func(participant types.LocalParticipant)) { r.onParticipantChanged = f } @@ -997,14 +988,9 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS } // a ParticipantImpl in the room added a new track, subscribe other participants to it -func (r *Room) onTrackPublished( - participant types.LocalParticipant, - pi *livekit.ParticipantInfo, - track types.MediaTrack, - ti *livekit.TrackInfo, -) { +func (r *Room) onTrackPublished(participant types.LocalParticipant, track types.MediaTrack) { // publish participant update, since track state is changed - r.broadcastParticipantState(participant, pi, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) r.lock.RLock() // subscribe all existing participants to this MediaTrack @@ -1033,7 +1019,7 @@ func (r *Room) onTrackPublished( r.lock.RUnlock() if onParticipantChanged != nil { - onParticipantChanged(participant, pi) + onParticipantChanged(participant) } r.trackManager.AddTrack(track, participant.Identity(), participant.ID()) @@ -1086,51 +1072,42 @@ func (r *Room) onTrackPublished( context.Background(), participant.ID(), participant.Identity(), - ti, + track.ToProto(), ) } -func (r *Room) onTrackUpdated( - p types.LocalParticipant, - pi *livekit.ParticipantInfo, - _ types.MediaTrack, -) { +func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { // send track updates to everyone, especially if track was updated by admin - r.broadcastParticipantState(p, pi, broadcastOptions{}) + r.broadcastParticipantState(p, broadcastOptions{}) if r.onParticipantChanged != nil { - r.onParticipantChanged(p, pi) + r.onParticipantChanged(p) } } -func (r *Room) onTrackUnpublished( - p types.LocalParticipant, - pi *livekit.ParticipantInfo, - track types.MediaTrack, - ti *livekit.TrackInfo, -) { +func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { r.telemetry.TrackUnpublished( context.Background(), p.ID(), p.Identity(), - ti, + track.ToProto(), !p.IsClosed(), ) r.trackManager.RemoveTrack(track) if !p.IsClosed() { - r.broadcastParticipantState(p, pi, broadcastOptions{skipSource: true}) + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) } if r.onParticipantChanged != nil { - r.onParticipantChanged(p, pi) + r.onParticipantChanged(p) } } -func (r *Room) onParticipantUpdate(p types.LocalParticipant, pi *livekit.ParticipantInfo) { +func (r *Room) onParticipantUpdate(p types.LocalParticipant) { r.protoProxy.MarkDirty(false) // immediately notify when permissions or metadata changed - r.broadcastParticipantState(p, pi, broadcastOptions{immediate: true}) + r.broadcastParticipantState(p, broadcastOptions{immediate: true}) if r.onParticipantChanged != nil { - r.onParticipantChanged(p, pi) + r.onParticipantChanged(p) } } @@ -1165,13 +1142,16 @@ func (r *Room) subscribeToExistingTracks(p types.LocalParticipant) { } // broadcast an update about participant p -func (r *Room) broadcastParticipantState(p types.LocalParticipant, pi *livekit.ParticipantInfo, opts broadcastOptions) { +func (r *Room) broadcastParticipantState(p types.LocalParticipant, opts broadcastOptions) { + pi := p.ToProto() + if p.Hidden() { if !opts.skipSource { // send update only to hidden participant err := p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi}) if err != nil { - p.GetLogger().Errorw("could not send update to participant", err) + r.Logger.Errorw("could not send update to participant", err, + "participant", p.Identity(), "pID", p.ID()) } } return diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index ec564c4af..4990b64d4 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -127,7 +127,7 @@ func TestRoomJoin(t *testing.T) { t.Run("participant state change is broadcasted to others", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants}) var changedParticipant types.Participant - rm.OnParticipantChanged(func(participant types.LocalParticipant, _pi *livekit.ParticipantInfo) { + rm.OnParticipantChanged(func(participant types.LocalParticipant) { changedParticipant = participant }) participants := rm.GetParticipants() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 1c010acbb..abfd29331 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -577,10 +577,10 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room } }) - newRoom.OnParticipantChanged(func(p types.LocalParticipant, pi *livekit.ParticipantInfo) { - if pi.State != livekit.ParticipantInfo_DISCONNECTED { - if err := r.roomStore.StoreParticipant(ctx, roomName, pi); err != nil { - p.GetLogger().Errorw("could not handle participant change", err) + newRoom.OnParticipantChanged(func(p types.LocalParticipant) { + if !p.IsDisconnected() { + if err := r.roomStore.StoreParticipant(ctx, roomName, p.ToProto()); err != nil { + newRoom.Logger.Errorw("could not handle participant change", err) } } }) From ad072f083681c7aa626a22f3cb0ca37446253d81 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 12:40:55 +0530 Subject: [PATCH 107/114] Revert "Plug worker leaks" (#2427) --- pkg/rtc/room.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 745ae9f4f..c6c7609f5 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -574,7 +574,6 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek r.lock.Unlock() return } - r.removeParticipantWorkerLocked(p) if pID != "" && p.ID() != pID { // participant session has been replaced @@ -583,6 +582,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } delete(r.participants, identity) + r.removeParticipantWorkerLocked(p) delete(r.participantOpts, identity) delete(r.participantRequestSources, identity) delete(r.hasPublished, identity) @@ -1529,7 +1529,6 @@ func (r *Room) removeParticipantWorkerLocked(p types.LocalParticipant) { for idx, participant := range pw.participants { if p == participant { pw.participants[idx] = pw.participants[n-1] - pw.participants[n-1] = nil pw.participants = pw.participants[:n-1] break } From 2a3de843513413afd3e1945960be7bc38344c285 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 13:03:32 +0530 Subject: [PATCH 108/114] Reverting participant worker. (#2428) * Reverting participant worker. Reverts https://github.com/livekit/livekit/pull/2420 partially. This did not revert clean. So, reverting manually. Also, keeping the drive-by clean up bits. * fix test --- pkg/rtc/participant.go | 46 ++++++--- pkg/rtc/room.go | 214 ++++++++++++----------------------------- pkg/rtc/room_test.go | 6 +- 3 files changed, 100 insertions(+), 166 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index df60f272a..ee4f20da8 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -690,9 +690,13 @@ func (p *ParticipantImpl) handleMigrateTracks() { } p.pendingTracksLock.Unlock() - for _, t := range addedTracks { - p.handleTrackPublished(t) - } + // launch callbacks in goroutine since they could block. + // callbacks handle webhooks as well as db persistence + go func() { + for _, t := range addedTracks { + p.handleTrackPublished(t) + } + }() } func (p *ParticipantImpl) removePendingMigratedTrack(mt *MediaTrack) { @@ -929,7 +933,7 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { } if onMigrateStateChange := p.getOnMigrateStateChange(); onMigrateStateChange != nil { - onMigrateStateChange(p, s) + go onMigrateStateChange(p, s) } } @@ -1337,7 +1341,7 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { p.dirty.Store(true) if onStateChange := p.getOnStateChange(); onStateChange != nil { - onStateChange(p, state) + go onStateChange(p, state) } } @@ -1939,12 +1943,14 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } if newTrack { - p.pubLogger.Debugw( - "track published", - "trackID", mt.ID(), - "track", logger.Proto(mt.ToProto()), - ) - p.handleTrackPublished(mt) + go func() { + p.pubLogger.Debugw( + "track published", + "trackID", mt.ID(), + "track", logger.Proto(mt.ToProto()), + ) + p.handleTrackPublished(mt) + }() } return mt, newTrack @@ -2052,6 +2058,15 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv p.supervisor.ClearPublishedTrack(trackID, mt) } + // not logged when closing + p.params.Telemetry.TrackUnpublished( + context.Background(), + p.ID(), + p.Identity(), + mt.ToProto(), + !p.IsClosed(), + ) + // re-use Track sid p.pendingTracksLock.Lock() if pti := p.pendingTracks[signalCid]; pti != nil { @@ -2080,6 +2095,15 @@ func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack) { onTrackPublished(p, track) } + // send webhook after callbacks are complete, persistence and state handling happens + // in `onTrackPublished` cb + p.params.Telemetry.TrackPublished( + context.Background(), + p.ID(), + p.Identity(), + track.ToProto(), + ) + p.pendingTracksLock.Lock() delete(p.pendingPublishingTracks, track.ID()) p.pendingTracksLock.Unlock() diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index c6c7609f5..1daf41cb6 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -73,11 +73,6 @@ type disconnectSignalOnResumeNoMessages struct { closedCount int } -type participantWorker struct { - eventsQueue *sutils.OpsQueue - participants []types.LocalParticipant -} - type Room struct { lock sync.RWMutex @@ -99,7 +94,6 @@ type Room struct { // map of identity -> Participant participants map[livekit.ParticipantIdentity]types.LocalParticipant - participantWorkers map[livekit.ParticipantIdentity]*participantWorker participantOpts map[livekit.ParticipantIdentity]*ParticipantOptions participantRequestSources map[livekit.ParticipantIdentity]routing.MessageSource hasPublished map[livekit.ParticipantIdentity]bool @@ -157,7 +151,6 @@ func NewRoom( trackManager: NewRoomTrackManager(), serverInfo: serverInfo, participants: make(map[livekit.ParticipantIdentity]types.LocalParticipant), - participantWorkers: make(map[livekit.ParticipantIdentity]*participantWorker), participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), hasPublished: make(map[livekit.ParticipantIdentity]bool), @@ -344,100 +337,78 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me r.joinedAt.Store(time.Now().Unix()) } - pw := r.addParticipantWorkerLocked(participant) - participant.OnStateChange(func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { - pw.eventsQueue.Enqueue(func() { - if r.onParticipantChanged != nil { - r.onParticipantChanged(p) + if r.onParticipantChanged != nil { + r.onParticipantChanged(p) + } + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + + if state == livekit.ParticipantInfo_ACTIVE { + // subscribe participant to existing published tracks + r.subscribeToExistingTracks(p) + + meta := &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) - - if state == livekit.ParticipantInfo_ACTIVE { - // subscribe participant to existing published tracks - r.subscribeToExistingTracks(p) - - meta := &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), + cds := p.GetICEConnectionDetails() + for _, cd := range cds { + if cd.Type != types.ICEConnectionTypeUnknown { + meta.ConnectionType = string(cd.Type) + break } - cds := p.GetICEConnectionDetails() - for _, cd := range cds { - if cd.Type != types.ICEConnectionTypeUnknown { - meta.ConnectionType = string(cd.Type) - break - } - } - r.telemetry.ParticipantActive(context.Background(), - r.ToProto(), - p.ToProto(), - meta, - false, - ) - - p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) - } else if state == livekit.ParticipantInfo_DISCONNECTED { - // remove participant from room - r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) } - }) + r.telemetry.ParticipantActive(context.Background(), + r.ToProto(), + p.ToProto(), + meta, + false, + ) + + p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) + } else if state == livekit.ParticipantInfo_DISCONNECTED { + // remove participant from room + go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) + } }) // it's important to set this before connection, we don't want to miss out on any published tracks - participant.OnTrackPublished(func(p types.LocalParticipant, t types.MediaTrack) { - pw.eventsQueue.Enqueue(func() { - r.onTrackPublished(p, t) - }) - }) - participant.OnTrackUpdated(func(p types.LocalParticipant, t types.MediaTrack) { - pw.eventsQueue.Enqueue(func() { - r.onTrackUpdated(p, t) - }) - }) - participant.OnTrackUnpublished(func(p types.LocalParticipant, t types.MediaTrack) { - pw.eventsQueue.Enqueue(func() { - r.onTrackUnpublished(p, t) - }) - }) - participant.OnParticipantUpdate(func(p types.LocalParticipant) { - pw.eventsQueue.Enqueue(func() { - r.onParticipantUpdate(p) - }) - }) + participant.OnTrackPublished(r.onTrackPublished) + participant.OnTrackUpdated(r.onTrackUpdated) + participant.OnTrackUnpublished(r.onTrackUnpublished) + participant.OnParticipantUpdate(r.onParticipantUpdate) participant.OnDataPacket(r.onDataPacket) participant.OnSubscribeStatusChanged(func(publisherID livekit.ParticipantID, subscribed bool) { - pw.eventsQueue.Enqueue(func() { - if subscribed { - pub := r.GetParticipantByID(publisherID) - if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { - // when a participant subscribes to another participant, - // send speaker update if the subscribed to participant is active. - level, active := pub.GetAudioLevel() - if active { - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(pub.ID()), - Level: float32(level), - Active: active, - }, - }, false) - } - - if cq := pub.GetConnectionQuality(); cq != nil { - update := &livekit.ConnectionQualityUpdate{} - update.Updates = append(update.Updates, cq) - _ = participant.SendConnectionQualityUpdate(update) - } + if subscribed { + pub := r.GetParticipantByID(publisherID) + if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { + // when a participant subscribes to another participant, + // send speaker update if the subscribed to participant is active. + level, active := pub.GetAudioLevel() + if active { + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(pub.ID()), + Level: float32(level), + Active: active, + }, + }, false) + } + + if cq := pub.GetConnectionQuality(); cq != nil { + update := &livekit.ConnectionQualityUpdate{} + update.Updates = append(update.Updates, cq) + _ = participant.SendConnectionQualityUpdate(update) } - } else { - // no longer subscribed to the publisher, clear speaker status - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(publisherID), - Level: 0, - Active: false, - }, - }, true) } - }) + } else { + // no longer subscribed to the publisher, clear speaker status + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(publisherID), + Level: 0, + Active: false, + }, + }, true) + } }) r.Logger.Debugw("new participant joined", @@ -582,7 +553,6 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } delete(r.participants, identity) - r.removeParticipantWorkerLocked(p) delete(r.participantOpts, identity) delete(r.participantRequestSources, identity) delete(r.hasPublished, identity) @@ -1066,14 +1036,6 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. } }() } - - // send webhook after callbacks are complete, i.e. after persistence and state handling - r.telemetry.TrackPublished( - context.Background(), - participant.ID(), - participant.Identity(), - track.ToProto(), - ) } func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { @@ -1085,14 +1047,6 @@ func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { } func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { - r.telemetry.TrackUnpublished( - context.Background(), - p.ID(), - p.Identity(), - track.ToProto(), - !p.IsClosed(), - ) - r.trackManager.RemoveTrack(track) if !p.IsClosed() { r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) @@ -1496,50 +1450,6 @@ func (r *Room) DebugInfo() map[string]interface{} { return info } -func (r *Room) addParticipantWorkerLocked(p types.LocalParticipant) *participantWorker { - identity := p.Identity() - pw := r.participantWorkers[identity] - if pw != nil { - found := false - for _, participant := range pw.participants { - if p == participant { - found = true - break - } - } - if !found { - pw.participants = append(pw.participants, p) - } - return pw - } - - pw = &participantWorker{ - eventsQueue: sutils.NewOpsQueue(fmt.Sprintf("participant-worker-%s-%s", r.Name(), identity), 0, true), - participants: []types.LocalParticipant{p}, - } - pw.eventsQueue.Start() - r.participantWorkers[identity] = pw - return pw -} - -func (r *Room) removeParticipantWorkerLocked(p types.LocalParticipant) { - identity := p.Identity() - if pw, ok := r.participantWorkers[identity]; ok { - n := len(pw.participants) - for idx, participant := range pw.participants { - if p == participant { - pw.participants[idx] = pw.participants[n-1] - pw.participants = pw.participants[:n-1] - break - } - } - if len(pw.participants) == 0 { - pw.eventsQueue.Stop() - delete(r.participantWorkers, identity) - } - } -} - // ------------------------------------------------------------ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp *livekit.DataPacket, logger logger.Logger) { diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 4990b64d4..4bf6ff6b6 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -121,7 +121,7 @@ func TestRoomJoin(t *testing.T) { numTracks += len(op.GetPublishedTracks()) } - require.Eventually(t, func() bool { return p.SubscribeToTrackCallCount() == numTracks }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, numTracks, p.SubscribeToTrackCallCount()) }) t.Run("participant state change is broadcasted to others", func(t *testing.T) { @@ -217,7 +217,7 @@ func TestParticipantUpdate(t *testing.T) { expected += 1 } fp := p.(*typesfakes.FakeLocalParticipant) - require.Eventually(t, func() bool { return fp.SendParticipantUpdateCallCount() == expected }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, expected, fp.SendParticipantUpdateCallCount()) } }) } @@ -423,8 +423,8 @@ func TestNewTrack(t *testing.T) { require.NotNil(t, trackCB) trackCB(pub, track) // only p1 should've been subscribed to - require.Eventually(t, func() bool { return p1.SubscribeToTrackCallCount() == 1 }, 5*time.Second, 10*time.Millisecond) require.Equal(t, 0, p0.SubscribeToTrackCallCount()) + require.Equal(t, 1, p1.SubscribeToTrackCallCount()) }) } From d53f167b316e4a2b0c5ab00dbbd322d0c8a93870 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 13:04:18 +0530 Subject: [PATCH 109/114] LeaveRequest changes. (#2426) Reworking this a bit 1. Send leave whenever the signal channel is closed to induce a resume. 2. Use a getter to get regions rather than setting. --- pkg/rtc/participant.go | 65 ++----------------- pkg/rtc/participant_signal.go | 31 +++++++++ pkg/rtc/types/interfaces.go | 4 +- .../typesfakes/fake_local_participant.go | 39 ----------- 4 files changed, 38 insertions(+), 101 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ee4f20da8..65f558326 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -120,6 +120,7 @@ type ParticipantParams struct { AllowUDPUnstableFallback bool TURNSEnabled bool GetParticipantInfo func(pID livekit.ParticipantID) *livekit.ParticipantInfo + GetRegionSettings func(ip string) *livekit.RegionSettings DisableSupervisor bool ReconnectOnPublicationError bool ReconnectOnSubscriptionError bool @@ -221,8 +222,6 @@ type ParticipantImpl struct { // loggers for publisher and subscriber pubLogger logger.Logger subLogger logger.Logger - - regionSettings *livekit.RegionSettings } func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { @@ -766,35 +765,8 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.clearDisconnectTimer() p.clearMigrationTimer() - // send leave message - var leave *livekit.LeaveRequest - if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { - leave = &livekit.LeaveRequest{ - Reason: reason.ToDisconnectReason(), - } - if isExpectedToResume { - leave.Action = livekit.LeaveRequest_RESUME - } else { - leave.Action = livekit.LeaveRequest_DISCONNECT - } - // although regions are not needed when resuming OR disconnecting, - // send it if available, just in case clients want to fall back. - p.lock.RLock() - if p.regionSettings != nil { - leave.Regions = proto.Clone(p.regionSettings).(*livekit.RegionSettings) - } - p.lock.RUnlock() - } else if sendLeave { - leave = &livekit.LeaveRequest{ - Reason: reason.ToDisconnectReason(), - } - } - if leave != nil { - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Leave{ - Leave: leave, - }, - }) + if sendLeave { + p.sendLeaveRequest(reason, isExpectedToResume, false) } if p.supervisor != nil { @@ -870,6 +842,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { onStart() } + p.sendLeaveRequest(types.ParticipantCloseReasonMigrationRequested, true, false) p.CloseSignalConnection(types.SignallingCloseReasonMigration) // @@ -1533,6 +1506,7 @@ func (p *ParticipantImpl) setupDisconnectTimer() { func (p *ParticipantImpl) onAnyTransportFailed() { // clients support resuming of connections when websocket becomes disconnected + p.sendLeaveRequest(types.ParticipantCloseReasonPeerConnectionDisconnected, true, false) p.CloseSignalConnection(types.SignallingCloseReasonTransportFailure) // detect when participant has actually left. @@ -2355,28 +2329,7 @@ func (p *ParticipantImpl) GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.R } func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason) { - var leave *livekit.LeaveRequest - if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { - leave = &livekit.LeaveRequest{ - Reason: reason.ToDisconnectReason(), - Action: livekit.LeaveRequest_RECONNECT, - } - p.lock.RLock() - if p.regionSettings != nil { - leave.Regions = proto.Clone(p.regionSettings).(*livekit.RegionSettings) - } - p.lock.RUnlock() - } else { - leave = &livekit.LeaveRequest{ - CanReconnect: true, - Reason: reason.ToDisconnectReason(), - } - } - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Leave{ - Leave: leave, - }, - }) + p.sendLeaveRequest(reason, false, true) scr := types.SignallingCloseReasonUnknown switch reason { @@ -2538,9 +2491,3 @@ func (p *ParticipantImpl) setupEnabledCodecs(publishEnabledCodecs []*livekit.Cod } p.enabledSubscribeCodecs = subscribeCodecs } - -func (p *ParticipantImpl) SetRegionSettings(regionSettings *livekit.RegionSettings) { - p.lock.Lock() - p.regionSettings = proto.Clone(regionSettings).(*livekit.RegionSettings) - p.lock.Unlock() -} diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 0a1bb31f1..04529f59b 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -311,3 +311,34 @@ func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReas p.SetResponseSink(nil) } } + +func (p *ParticipantImpl) sendLeaveRequest(reason types.ParticipantCloseReason, isExpectedToResume bool, isExpectedToReconnect bool) { + var leave *livekit.LeaveRequest + if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { + leave = &livekit.LeaveRequest{ + Reason: reason.ToDisconnectReason(), + } + switch { + case isExpectedToResume: + leave.Action = livekit.LeaveRequest_RESUME + case isExpectedToReconnect: + leave.Action = livekit.LeaveRequest_RECONNECT + default: + leave.Action = livekit.LeaveRequest_DISCONNECT + } + if leave.Action != livekit.LeaveRequest_DISCONNECT && p.params.GetRegionSettings != nil { + // sending region settings even for RESUME just in case client wants to a full reconnect despite server saying RESUME + leave.Regions = p.params.GetRegionSettings(p.params.ClientInfo.Address) + } + } else { + leave = &livekit.LeaveRequest{ + CanReconnect: isExpectedToReconnect, + Reason: reason.ToDisconnectReason(), + } + } + _ = p.writeMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: leave, + }, + }) +} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 4cbcb117a..af18021d0 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -178,7 +178,7 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_STATE_MISMATCH case ParticipantCloseReasonDuplicateIdentity, ParticipantCloseReasonStale: return livekit.DisconnectReason_DUPLICATE_IDENTITY - case ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonSimulateMigration: + case ParticipantCloseReasonMigrationRequested, ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonSimulateMigration: return livekit.DisconnectReason_MIGRATION case ParticipantCloseReasonServiceRequestRemoveParticipant: return livekit.DisconnectReason_PARTICIPANT_REMOVED @@ -422,8 +422,6 @@ type LocalParticipant interface { GetPacer() pacer.Pacer GetTrafficLoad() *TrafficLoad - - SetRegionSettings(regionSettings *livekit.RegionSettings) } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 32c1c2869..f101e4398 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -753,11 +753,6 @@ type FakeLocalParticipant struct { setPermissionReturnsOnCall map[int]struct { result1 bool } - SetRegionSettingsStub func(*livekit.RegionSettings) - setRegionSettingsMutex sync.RWMutex - setRegionSettingsArgsForCall []struct { - arg1 *livekit.RegionSettings - } SetResponseSinkStub func(routing.MessageSink) setResponseSinkMutex sync.RWMutex setResponseSinkArgsForCall []struct { @@ -4982,38 +4977,6 @@ func (fake *FakeLocalParticipant) SetPermissionReturnsOnCall(i int, result1 bool }{result1} } -func (fake *FakeLocalParticipant) SetRegionSettings(arg1 *livekit.RegionSettings) { - fake.setRegionSettingsMutex.Lock() - fake.setRegionSettingsArgsForCall = append(fake.setRegionSettingsArgsForCall, struct { - arg1 *livekit.RegionSettings - }{arg1}) - stub := fake.SetRegionSettingsStub - fake.recordInvocation("SetRegionSettings", []interface{}{arg1}) - fake.setRegionSettingsMutex.Unlock() - if stub != nil { - fake.SetRegionSettingsStub(arg1) - } -} - -func (fake *FakeLocalParticipant) SetRegionSettingsCallCount() int { - fake.setRegionSettingsMutex.RLock() - defer fake.setRegionSettingsMutex.RUnlock() - return len(fake.setRegionSettingsArgsForCall) -} - -func (fake *FakeLocalParticipant) SetRegionSettingsCalls(stub func(*livekit.RegionSettings)) { - fake.setRegionSettingsMutex.Lock() - defer fake.setRegionSettingsMutex.Unlock() - fake.SetRegionSettingsStub = stub -} - -func (fake *FakeLocalParticipant) SetRegionSettingsArgsForCall(i int) *livekit.RegionSettings { - fake.setRegionSettingsMutex.RLock() - defer fake.setRegionSettingsMutex.RUnlock() - argsForCall := fake.setRegionSettingsArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalParticipant) SetResponseSink(arg1 routing.MessageSink) { fake.setResponseSinkMutex.Lock() fake.setResponseSinkArgsForCall = append(fake.setResponseSinkArgsForCall, struct { @@ -6352,8 +6315,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.setNameMutex.RUnlock() fake.setPermissionMutex.RLock() defer fake.setPermissionMutex.RUnlock() - fake.setRegionSettingsMutex.RLock() - defer fake.setRegionSettingsMutex.RUnlock() fake.setResponseSinkMutex.RLock() defer fake.setResponseSinkMutex.RUnlock() fake.setSignalSourceValidMutex.RLock() From a68500d4a1b470693788424ce157d5e1bdc55825 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 Jan 2024 14:49:53 +0530 Subject: [PATCH 110/114] Selective send of LeaveRequest. (#2429) Cannot send old style leave request during migration and other scenarios when client is expected to resume. The old style can only do a full reconnect or disconnect. If `CanReconnect: false` which will be the case for resume, client will disconnect. Add a parameter to selectively send leave request to older clients. --- pkg/rtc/participant.go | 8 ++++---- pkg/rtc/participant_signal.go | 29 ++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 65f558326..52da57e30 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -766,7 +766,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.clearMigrationTimer() if sendLeave { - p.sendLeaveRequest(reason, isExpectedToResume, false) + p.sendLeaveRequest(reason, isExpectedToResume, false, false) } if p.supervisor != nil { @@ -842,7 +842,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { onStart() } - p.sendLeaveRequest(types.ParticipantCloseReasonMigrationRequested, true, false) + p.sendLeaveRequest(types.ParticipantCloseReasonMigrationRequested, true, false, true) p.CloseSignalConnection(types.SignallingCloseReasonMigration) // @@ -1506,7 +1506,7 @@ func (p *ParticipantImpl) setupDisconnectTimer() { func (p *ParticipantImpl) onAnyTransportFailed() { // clients support resuming of connections when websocket becomes disconnected - p.sendLeaveRequest(types.ParticipantCloseReasonPeerConnectionDisconnected, true, false) + p.sendLeaveRequest(types.ParticipantCloseReasonPeerConnectionDisconnected, true, false, true) p.CloseSignalConnection(types.SignallingCloseReasonTransportFailure) // detect when participant has actually left. @@ -2329,7 +2329,7 @@ func (p *ParticipantImpl) GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.R } func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason) { - p.sendLeaveRequest(reason, false, true) + p.sendLeaveRequest(reason, false, true, false) scr := types.SignallingCloseReasonUnknown switch reason { diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 04529f59b..63e944e41 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -312,7 +312,12 @@ func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReas } } -func (p *ParticipantImpl) sendLeaveRequest(reason types.ParticipantCloseReason, isExpectedToResume bool, isExpectedToReconnect bool) { +func (p *ParticipantImpl) sendLeaveRequest( + reason types.ParticipantCloseReason, + isExpectedToResume bool, + isExpectedToReconnect bool, + sendOnlyIfSupportingLeaveRequestWithAction bool, +) error { var leave *livekit.LeaveRequest if p.ProtocolVersion().SupportsRegionsInLeaveRequest() { leave = &livekit.LeaveRequest{ @@ -331,14 +336,20 @@ func (p *ParticipantImpl) sendLeaveRequest(reason types.ParticipantCloseReason, leave.Regions = p.params.GetRegionSettings(p.params.ClientInfo.Address) } } else { - leave = &livekit.LeaveRequest{ - CanReconnect: isExpectedToReconnect, - Reason: reason.ToDisconnectReason(), + if !sendOnlyIfSupportingLeaveRequestWithAction { + leave = &livekit.LeaveRequest{ + CanReconnect: isExpectedToReconnect, + Reason: reason.ToDisconnectReason(), + } } } - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Leave{ - Leave: leave, - }, - }) + if leave != nil { + return p.writeMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: leave, + }, + }) + } + + return nil } From f960a4f9fb02ca16a55b736dfb368b99f6ad0864 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 29 Jan 2024 16:57:50 -0800 Subject: [PATCH 111/114] update egress client (#2431) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 10b54ea8e..1d6f8666e 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f - github.com/livekit/protocol v1.9.7 - github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b + github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e + github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 diff --git a/go.sum b/go.sum index fb47d5edd..231b870b9 100644 --- a/go.sum +++ b/go.sum @@ -126,10 +126,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f h1:XHrwGwLNGQB3ZqolH1YdMH/22hgXKr4vm+2M7JKMMGg= github.com/livekit/mediatransportutil v0.0.0-20231213075826-cccbf2b93d3f/go.mod h1:GBzn9xL+mivI1pW+tyExcKgbc0VOc29I9yJsNcAVaAc= -github.com/livekit/protocol v1.9.7 h1:5pYAMS/rzOStpIfRGnhXETPH/NyoFJtbV7FW4NHxg7o= -github.com/livekit/protocol v1.9.7/go.mod h1:daddOPw85C9nq6f9w1uiuc1i/He6X2gArlFcKUPELI4= -github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b h1:+860jUlsyW3xI9sp6IdOM7QceOK2rsbZC2P193UI4Kc= -github.com/livekit/psrpc v0.5.3-0.20240126182121-829a885bf21b/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= +github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e h1:aQGmivssjKlvca8Mmmm4XzHQrFV+vwrGIhAxQE8tnZk= +github.com/livekit/protocol v1.9.8-0.20240130003842-54f76fc6865e/go.mod h1:lSJlMeTJfQBEv8/D2p3zdCo+i+jTmTtn24ysL4ePK28= +github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289 h1:oTgNH7v9TXsBgoltKk5mnWjv4qqcPF2iV+WtEVQ6ROM= +github.com/livekit/psrpc v0.5.3-0.20240129223932-473b29cda289/go.mod h1:cQjxg1oCxYHhxxv6KJH1gSvdtCHQoRZCHgPdm5N8v2g= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= From c8b7d486b9ca3435f99823234c46e4fb36a635e1 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 31 Jan 2024 11:36:50 +0530 Subject: [PATCH 112/114] Do not synthesise DISCONNECT on session change. (#2412) * Do not synthesise DISCONNECT on session change. v12 clients can handle session change based on identity. * change for testf * Squelch participant update if close reason is DUPLICATE_IDENTITY. * fix test * comment * Clean up participant close reason a bit * fix test * test --- pkg/rtc/participant.go | 16 +++- pkg/rtc/room.go | 88 ++++++++++++------ pkg/rtc/room_test.go | 93 ++++++++++--------- pkg/rtc/types/interfaces.go | 17 ++-- pkg/rtc/types/protocol_version.go | 4 + .../typesfakes/fake_local_participant.go | 65 +++++++++++++ pkg/rtc/types/typesfakes/fake_participant.go | 65 +++++++++++++ pkg/service/roommanager.go | 10 +- test/webhook_test.go | 3 +- 9 files changed, 268 insertions(+), 93 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 52da57e30..8c476d5e7 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -140,9 +140,13 @@ type ParticipantImpl struct { params ParticipantParams isClosed atomic.Bool - state atomic.Value // livekit.ParticipantInfo_State - resSinkMu sync.Mutex - resSink routing.MessageSink + closeReason atomic.Value // types.ParticipantCloseReason + + state atomic.Value // livekit.ParticipantInfo_State + + resSinkMu sync.Mutex + resSink routing.MessageSink + grants *auth.ClaimGrants hidden atomic.Bool isPublisher atomic.Bool @@ -253,6 +257,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { if !params.DisableSupervisor { p.supervisor = supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}) } + p.closeReason.Store(types.ParticipantCloseReasonNone) p.version.Store(params.InitialVersion) p.timedVersion.Update(params.VersionGenerator.New()) p.migrateState.Store(types.MigrateStateInit) @@ -762,6 +767,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea "reason", reason.String(), "isExpectedToResume", isExpectedToResume, ) + p.closeReason.Store(reason) p.clearDisconnectTimer() p.clearMigrationTimer() @@ -812,6 +818,10 @@ func (p *ParticipantImpl) IsClosed() bool { return p.isClosed.Load() } +func (p *ParticipantImpl) CloseReason() types.ParticipantCloseReason { + return p.closeReason.Load().(types.ParticipantCloseReason) +} + // Negotiate subscriber SDP with client, if force is true, will cancel pending // negotiate task and negotiate immediately func (p *ParticipantImpl) Negotiate(force bool) { diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 1daf41cb6..4e848a7c5 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -68,6 +68,12 @@ type broadcastOptions struct { immediate bool } +type participantUpdate struct { + pi *livekit.ParticipantInfo + isSynthesizedDisconnect bool + closeReason types.ParticipantCloseReason +} + type disconnectSignalOnResumeNoMessages struct { expiry time.Time closedCount int @@ -100,7 +106,7 @@ type Room struct { bufferFactory *buffer.FactoryOfBufferFactory // batch update participant info for non-publishers - batchedUpdates map[livekit.ParticipantIdentity]*livekit.ParticipantInfo + batchedUpdates map[livekit.ParticipantIdentity]*participantUpdate batchedUpdatesMu sync.Mutex // time the first participant joined the room @@ -155,7 +161,7 @@ func NewRoom( participantRequestSources: make(map[livekit.ParticipantIdentity]routing.MessageSource), hasPublished: make(map[livekit.ParticipantIdentity]bool), bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), - batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), + batchedUpdates: make(map[livekit.ParticipantIdentity]*participantUpdate), closed: make(chan struct{}), trailer: []byte(utils.RandomSecret()), disconnectSignalOnResumeParticipants: make(map[livekit.ParticipantIdentity]time.Time), @@ -367,7 +373,8 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me p.GetLogger().Infow("participant active", connectionDetailsFields(cds)...) } else if state == livekit.ParticipantInfo_DISCONNECTED { // remove participant from room - go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) + // participant should already be closed and have a close reason, so NONE is fine here + go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonNone) } }) // it's important to set this before connection, we don't want to miss out on any published tracks @@ -764,11 +771,11 @@ func (r *Room) CloseIfEmpty() { r.lock.Unlock() if elapsed >= int64(timeout) { - r.Close() + r.Close(types.ParticipantCloseReasonNone) } } -func (r *Room) Close() { +func (r *Room) Close(reason types.ParticipantCloseReason) { r.lock.Lock() select { case <-r.closed: @@ -782,7 +789,7 @@ func (r *Room) Close() { r.Logger.Infow("closing room") for _, p := range r.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomClose, false) + _ = p.Close(true, reason, false) } r.protoProxy.Stop() @@ -1104,27 +1111,49 @@ func (r *Room) broadcastParticipantState(p types.LocalParticipant, opts broadcas // send update only to hidden participant err := p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi}) if err != nil { - r.Logger.Errorw("could not send update to participant", err, - "participant", p.Identity(), "pID", p.ID()) + p.GetLogger().Errorw("could not send update to participant", err) } } return } - updates := r.pushAndDequeueUpdates(pi, opts.immediate) + updates := r.pushAndDequeueUpdates(pi, p.CloseReason(), opts.immediate) r.sendParticipantUpdates(updates) } -func (r *Room) sendParticipantUpdates(updates []*livekit.ParticipantInfo) { +func (r *Room) sendParticipantUpdates(updates []*participantUpdate) { if len(updates) == 0 { return } + // For filtered updates, skip + // 1. synthesized DISCONNECT - this happens on SID change + // 2. close reasons of DUPLICATE_IDENTITY/STALE - A newer session for that identity exists. + // + // Filtered updates are used with clients that can handle identity based reconnect and hence those + // conditions can be skipped. + var filteredUpdates []*livekit.ParticipantInfo + for _, update := range updates { + if update.isSynthesizedDisconnect || IsCloseNotifySkippable(update.closeReason) { + continue + } + filteredUpdates = append(filteredUpdates, update.pi) + } + + var fullUpdates []*livekit.ParticipantInfo + for _, update := range updates { + fullUpdates = append(fullUpdates, update.pi) + } + for _, op := range r.GetParticipants() { - err := op.SendParticipantUpdate(updates) + var err error + if op.ProtocolVersion().SupportsIdentityBasedReconnection() { + err = op.SendParticipantUpdate(filteredUpdates) + } else { + err = op.SendParticipantUpdate(fullUpdates) + } if err != nil { - r.Logger.Errorw("could not send update to participant", err, - "participant", op.Identity(), "pID", op.ID()) + op.GetLogger().Errorw("could not send update to participant", err) } } } @@ -1170,29 +1199,34 @@ func (r *Room) sendSpeakerChanges(speakers []*livekit.SpeakerInfo) { // * subscriber-only updates will be queued for batch updates // * publisher & immediate updates will be returned without queuing // * when the SID changes, it will return both updates, with the earlier participant set to disconnected -func (r *Room) pushAndDequeueUpdates(pi *livekit.ParticipantInfo, isImmediate bool) []*livekit.ParticipantInfo { +func (r *Room) pushAndDequeueUpdates( + pi *livekit.ParticipantInfo, + closeReason types.ParticipantCloseReason, + isImmediate bool, +) []*participantUpdate { r.batchedUpdatesMu.Lock() defer r.batchedUpdatesMu.Unlock() - var updates []*livekit.ParticipantInfo + var updates []*participantUpdate identity := livekit.ParticipantIdentity(pi.Identity) existing := r.batchedUpdates[identity] shouldSend := isImmediate || pi.IsPublisher if existing != nil { - if pi.Sid == existing.Sid { + if pi.Sid == existing.pi.Sid { // same participant session - if pi.Version < existing.Version { + if pi.Version < existing.pi.Version { // out of order update return nil } } else { // different participant sessions - if existing.JoinedAt < pi.JoinedAt { + if existing.pi.JoinedAt < pi.JoinedAt { // existing is older, synthesize a DISCONNECT for older and // send immediately along with newer session to signal switch shouldSend = true - existing.State = livekit.ParticipantInfo_DISCONNECTED + existing.pi.State = livekit.ParticipantInfo_DISCONNECTED + existing.isSynthesizedDisconnect = true updates = append(updates, existing) } else { // older session update, newer session has already become active, so nothing to do @@ -1213,10 +1247,10 @@ func (r *Room) pushAndDequeueUpdates(pi *livekit.ParticipantInfo, isImmediate bo if shouldSend { // include any queued update, and return delete(r.batchedUpdates, identity) - updates = append(updates, pi) + updates = append(updates, &participantUpdate{pi: pi, closeReason: closeReason}) } else { // enqueue for batch - r.batchedUpdates[identity] = pi + r.batchedUpdates[identity] = &participantUpdate{pi: pi, closeReason: closeReason} } return updates @@ -1257,18 +1291,14 @@ func (r *Room) changeUpdateWorker() { case <-subTicker.C: r.batchedUpdatesMu.Lock() updatesMap := r.batchedUpdates - r.batchedUpdates = make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo) + r.batchedUpdates = make(map[livekit.ParticipantIdentity]*participantUpdate) r.batchedUpdatesMu.Unlock() if len(updatesMap) == 0 { continue } - updates := make([]*livekit.ParticipantInfo, 0, len(updatesMap)) - for _, pi := range updatesMap { - updates = append(updates, pi) - } - r.sendParticipantUpdates(updates) + r.sendParticipantUpdates(maps.Values(updatesMap)) } } } @@ -1509,6 +1539,10 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp }) } +func IsCloseNotifySkippable(closeReason types.ParticipantCloseReason) bool { + return closeReason == types.ParticipantCloseReasonDuplicateIdentity +} + func connectionDetailsFields(cds []*types.ICEConnectionDetails) []interface{} { var fields []interface{} connectionType := types.ICEConnectionTypeUnknown diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 4bf6ff6b6..0febf2989 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -135,7 +135,7 @@ func TestRoomJoin(t *testing.T) { disconnectedParticipant := participants[1].(*typesfakes.FakeLocalParticipant) disconnectedParticipant.StateReturns(livekit.ParticipantInfo_DISCONNECTED) - rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) + rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave) time.Sleep(defaultDelay) require.Equal(t, p, changedParticipant) @@ -265,17 +265,18 @@ func TestPushAndDequeueUpdates(t *testing.T) { require.Equal(t, a.Version, b.Version) } testCases := []struct { - name string - pi *livekit.ParticipantInfo - immediate bool - existing *livekit.ParticipantInfo - expected []*livekit.ParticipantInfo - validate func(t *testing.T, rm *Room, updates []*livekit.ParticipantInfo) + name string + pi *livekit.ParticipantInfo + closeReason types.ParticipantCloseReason + immediate bool + existing *participantUpdate + expected []*participantUpdate + validate func(t *testing.T, rm *Room, updates []*participantUpdate) }{ { name: "publisher updates are immediate", pi: publisher1v1, - expected: []*livekit.ParticipantInfo{publisher1v1}, + expected: []*participantUpdate{{pi: publisher1v1}}, }, { name: "subscriber updates are queued", @@ -284,20 +285,20 @@ func TestPushAndDequeueUpdates(t *testing.T) { { name: "last version is enqueued", pi: subscriber1v2, - existing: subscriber1v1, - validate: func(t *testing.T, rm *Room, _ []*livekit.ParticipantInfo) { + existing: &participantUpdate{pi: proto.Clone(subscriber1v1).(*livekit.ParticipantInfo)}, // clone the existing value since it can be modified when setting to disconnected + validate: func(t *testing.T, rm *Room, _ []*participantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] require.NotNil(t, queued) - requirePIEquals(t, subscriber1v2, queued) + requirePIEquals(t, subscriber1v2, queued.pi) }, }, { name: "latest version when immediate", pi: subscriber1v2, - existing: subscriber1v1, + existing: &participantUpdate{pi: proto.Clone(subscriber1v1).(*livekit.ParticipantInfo)}, immediate: true, - expected: []*livekit.ParticipantInfo{subscriber1v2}, - validate: func(t *testing.T, rm *Room, _ []*livekit.ParticipantInfo) { + expected: []*participantUpdate{{pi: subscriber1v2}}, + validate: func(t *testing.T, rm *Room, _ []*participantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] require.Nil(t, queued) }, @@ -305,32 +306,37 @@ func TestPushAndDequeueUpdates(t *testing.T) { { name: "out of order updates are rejected", pi: subscriber1v1, - existing: subscriber1v2, - validate: func(t *testing.T, rm *Room, updates []*livekit.ParticipantInfo) { + existing: &participantUpdate{pi: proto.Clone(subscriber1v2).(*livekit.ParticipantInfo)}, + validate: func(t *testing.T, rm *Room, updates []*participantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] - requirePIEquals(t, subscriber1v2, queued) + requirePIEquals(t, subscriber1v2, queued.pi) }, }, { - name: "sid change is broadcasted immediately", - pi: publisher2, - existing: subscriber1v2, - expected: []*livekit.ParticipantInfo{ + name: "sid change is broadcasted immediately with synthsized disconnect", + pi: publisher2, + closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant, // just to test if update contain the close reason + existing: &participantUpdate{pi: proto.Clone(subscriber1v2).(*livekit.ParticipantInfo), closeReason: types.ParticipantCloseReasonStale}, + expected: []*participantUpdate{ { - Identity: identity, - Sid: "1", - Version: 2, - State: livekit.ParticipantInfo_DISCONNECTED, + pi: &livekit.ParticipantInfo{ + Identity: identity, + Sid: "1", + Version: 2, + State: livekit.ParticipantInfo_DISCONNECTED, + }, + isSynthesizedDisconnect: true, + closeReason: types.ParticipantCloseReasonStale, }, - publisher2, + {pi: publisher2, closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant}, }, }, { name: "when switching to publisher, queue is cleared", pi: publisher1v2, - existing: subscriber1v1, - expected: []*livekit.ParticipantInfo{publisher1v2}, - validate: func(t *testing.T, rm *Room, updates []*livekit.ParticipantInfo) { + existing: &participantUpdate{pi: proto.Clone(subscriber1v1).(*livekit.ParticipantInfo)}, + expected: []*participantUpdate{{pi: publisher1v2}}, + validate: func(t *testing.T, rm *Room, updates []*participantUpdate) { require.Empty(t, rm.batchedUpdates) }, }, @@ -340,13 +346,14 @@ func TestPushAndDequeueUpdates(t *testing.T) { t.Run(tc.name, func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) if tc.existing != nil { - // clone the existing value since it can be modified when setting to disconnected - rm.batchedUpdates[livekit.ParticipantIdentity(tc.existing.Identity)] = proto.Clone(tc.existing).(*livekit.ParticipantInfo) + rm.batchedUpdates[livekit.ParticipantIdentity(tc.existing.pi.Identity)] = tc.existing } - updates := rm.pushAndDequeueUpdates(tc.pi, tc.immediate) + updates := rm.pushAndDequeueUpdates(tc.pi, tc.closeReason, tc.immediate) require.Equal(t, len(tc.expected), len(updates)) for i, item := range tc.expected { - requirePIEquals(t, item, updates[i]) + requirePIEquals(t, item.pi, updates[i].pi) + require.Equal(t, item.isSynthesizedDisconnect, updates[i].isSynthesizedDisconnect) + require.Equal(t, item.closeReason, updates[i].closeReason) } if tc.validate != nil { @@ -443,7 +450,7 @@ func TestActiveSpeakers(t *testing.T) { audioUpdateDuration := (audioUpdateInterval + 10) * time.Millisecond t.Run("participant should not be getting audio updates (protocol 2)", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1, protocol: 2}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) p := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) require.Empty(t, rm.GetActiveSpeakers()) @@ -455,7 +462,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("speakers should be sorted by loudness", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p2 := participants[1].(*typesfakes.FakeLocalParticipant) @@ -470,7 +477,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("participants are getting audio updates (protocol 3+)", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) time.Sleep(time.Millisecond) // let the first update cycle run @@ -509,7 +516,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("audio level is smoothed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3, audioSmoothIntervals: 3}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) op := participants[1].(*typesfakes.FakeLocalParticipant) @@ -566,7 +573,7 @@ func TestDataChannel(t *testing.T) { t.Run("participants should receive data", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) @@ -596,7 +603,7 @@ func TestDataChannel(t *testing.T) { t.Run("only one participant should receive the data", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 4}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p1 := participants[1].(*typesfakes.FakeLocalParticipant) @@ -627,7 +634,7 @@ func TestDataChannel(t *testing.T) { t.Run("publishing disallowed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p.CanPublishDataReturns(false) @@ -655,7 +662,7 @@ func TestDataChannel(t *testing.T) { func TestHiddenParticipants(t *testing.T) { t.Run("other participants don't receive hidden updates", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, numHidden: 1}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) pNew := NewMockParticipant("new", types.CurrentProtocol, false, false) rm.Join(pNew, nil, nil, iceServersForRoom) @@ -687,7 +694,7 @@ func TestHiddenParticipants(t *testing.T) { func TestRoomUpdate(t *testing.T) { t.Run("updates are sent when participant joined", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) require.Equal(t, 0, p1.SendRoomUpdateCallCount()) @@ -703,7 +710,7 @@ func TestRoomUpdate(t *testing.T) { t.Run("participants should receive metadata update", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - defer rm.Close() + defer rm.Close(types.ParticipantCloseReasonNone) rm.SetMetadata("test metadata...") diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index af18021d0..32eb1e35c 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -81,14 +81,13 @@ type SubscribedCodecQuality struct { type ParticipantCloseReason int const ( - ParticipantCloseReasonClientRequestLeave ParticipantCloseReason = iota + ParticipantCloseReasonNone ParticipantCloseReason = iota + ParticipantCloseReasonClientRequestLeave ParticipantCloseReasonRoomManagerStop - ParticipantCloseReasonRoomClose ParticipantCloseReasonVerifyFailed ParticipantCloseReasonJoinFailed ParticipantCloseReasonJoinTimeout ParticipantCloseReasonMessageBusFailed - ParticipantCloseReasonStateDisconnected ParticipantCloseReasonPeerConnectionDisconnected ParticipantCloseReasonDuplicateIdentity ParticipantCloseReasonMigrationComplete @@ -100,7 +99,6 @@ const ( ParticipantCloseReasonSimulateServerLeave ParticipantCloseReasonNegotiateFailed ParticipantCloseReasonMigrationRequested - ParticipantCloseReasonOvercommitted ParticipantCloseReasonPublicationError ParticipantCloseReasonSubscriptionError ParticipantCloseReasonDataChannelError @@ -110,12 +108,12 @@ const ( func (p ParticipantCloseReason) String() string { switch p { + case ParticipantCloseReasonNone: + return "NONE" case ParticipantCloseReasonClientRequestLeave: return "CLIENT_REQUEST_LEAVE" case ParticipantCloseReasonRoomManagerStop: return "ROOM_MANAGER_STOP" - case ParticipantCloseReasonRoomClose: - return "ROOM_CLOSE" case ParticipantCloseReasonVerifyFailed: return "VERIFY_FAILED" case ParticipantCloseReasonJoinFailed: @@ -124,8 +122,6 @@ func (p ParticipantCloseReason) String() string { return "JOIN_TIMEOUT" case ParticipantCloseReasonMessageBusFailed: return "MESSAGE_BUS_FAILED" - case ParticipantCloseReasonStateDisconnected: - return "STATE_DISCONNECTED" case ParticipantCloseReasonPeerConnectionDisconnected: return "PEER_CONNECTION_DISCONNECTED" case ParticipantCloseReasonDuplicateIdentity: @@ -148,8 +144,6 @@ func (p ParticipantCloseReason) String() string { return "NEGOTIATE_FAILED" case ParticipantCloseReasonMigrationRequested: return "MIGRATION_REQUESTED" - case ParticipantCloseReasonOvercommitted: - return "OVERCOMMITTED" case ParticipantCloseReasonPublicationError: return "PUBLICATION_ERROR" case ParticipantCloseReasonSubscriptionError: @@ -184,7 +178,7 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_PARTICIPANT_REMOVED case ParticipantCloseReasonServiceRequestDeleteRoom: return livekit.DisconnectReason_ROOM_DELETED - case ParticipantCloseReasonSimulateNodeFailure, ParticipantCloseReasonSimulateServerLeave, ParticipantCloseReasonOvercommitted: + case ParticipantCloseReasonSimulateNodeFailure, ParticipantCloseReasonSimulateServerLeave: return livekit.DisconnectReason_SERVER_SHUTDOWN case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError, ParticipantCloseReasonDataChannelError, ParticipantCloseReasonMigrateCodecMismatch: return livekit.DisconnectReason_STATE_MISMATCH @@ -250,6 +244,7 @@ type Participant interface { ID() livekit.ParticipantID Identity() livekit.ParticipantIdentity State() livekit.ParticipantInfo_State + CloseReason() ParticipantCloseReason CanSkipBroadcast() bool ToProto() *livekit.ParticipantInfo diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 08075184d..ae2282085 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -84,6 +84,10 @@ func (v ProtocolVersion) SupportsAsyncRoomID() bool { return v > 11 } +func (v ProtocolVersion) SupportsIdentityBasedReconnection() bool { + return v > 11 +} + func (v ProtocolVersion) SupportsRegionsInLeaveRequest() bool { return v > 12 } diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index f101e4398..c20a1a1d7 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -133,6 +133,16 @@ type FakeLocalParticipant struct { closeReturnsOnCall map[int]struct { result1 error } + CloseReasonStub func() types.ParticipantCloseReason + closeReasonMutex sync.RWMutex + closeReasonArgsForCall []struct { + } + closeReasonReturns struct { + result1 types.ParticipantCloseReason + } + closeReasonReturnsOnCall map[int]struct { + result1 types.ParticipantCloseReason + } CloseSignalConnectionStub func(types.SignallingCloseReason) closeSignalConnectionMutex sync.RWMutex closeSignalConnectionArgsForCall []struct { @@ -1545,6 +1555,59 @@ func (fake *FakeLocalParticipant) CloseReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeLocalParticipant) CloseReason() types.ParticipantCloseReason { + fake.closeReasonMutex.Lock() + ret, specificReturn := fake.closeReasonReturnsOnCall[len(fake.closeReasonArgsForCall)] + fake.closeReasonArgsForCall = append(fake.closeReasonArgsForCall, struct { + }{}) + stub := fake.CloseReasonStub + fakeReturns := fake.closeReasonReturns + fake.recordInvocation("CloseReason", []interface{}{}) + fake.closeReasonMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) CloseReasonCallCount() int { + fake.closeReasonMutex.RLock() + defer fake.closeReasonMutex.RUnlock() + return len(fake.closeReasonArgsForCall) +} + +func (fake *FakeLocalParticipant) CloseReasonCalls(stub func() types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = stub +} + +func (fake *FakeLocalParticipant) CloseReasonReturns(result1 types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = nil + fake.closeReasonReturns = struct { + result1 types.ParticipantCloseReason + }{result1} +} + +func (fake *FakeLocalParticipant) CloseReasonReturnsOnCall(i int, result1 types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = nil + if fake.closeReasonReturnsOnCall == nil { + fake.closeReasonReturnsOnCall = make(map[int]struct { + result1 types.ParticipantCloseReason + }) + } + fake.closeReasonReturnsOnCall[i] = struct { + result1 types.ParticipantCloseReason + }{result1} +} + func (fake *FakeLocalParticipant) CloseSignalConnection(arg1 types.SignallingCloseReason) { fake.closeSignalConnectionMutex.Lock() fake.closeSignalConnectionArgsForCall = append(fake.closeSignalConnectionArgsForCall, struct { @@ -6171,6 +6234,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.claimGrantsMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.closeReasonMutex.RLock() + defer fake.closeReasonMutex.RUnlock() fake.closeSignalConnectionMutex.RLock() defer fake.closeSignalConnectionMutex.RUnlock() fake.connectedAtMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 0c5c3929a..98fff9c80 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -33,6 +33,16 @@ type FakeParticipant struct { closeReturnsOnCall map[int]struct { result1 error } + CloseReasonStub func() types.ParticipantCloseReason + closeReasonMutex sync.RWMutex + closeReasonArgsForCall []struct { + } + closeReasonReturns struct { + result1 types.ParticipantCloseReason + } + closeReasonReturnsOnCall map[int]struct { + result1 types.ParticipantCloseReason + } DebugInfoStub func() map[string]interface{} debugInfoMutex sync.RWMutex debugInfoArgsForCall []struct { @@ -342,6 +352,59 @@ func (fake *FakeParticipant) CloseReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeParticipant) CloseReason() types.ParticipantCloseReason { + fake.closeReasonMutex.Lock() + ret, specificReturn := fake.closeReasonReturnsOnCall[len(fake.closeReasonArgsForCall)] + fake.closeReasonArgsForCall = append(fake.closeReasonArgsForCall, struct { + }{}) + stub := fake.CloseReasonStub + fakeReturns := fake.closeReasonReturns + fake.recordInvocation("CloseReason", []interface{}{}) + fake.closeReasonMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) CloseReasonCallCount() int { + fake.closeReasonMutex.RLock() + defer fake.closeReasonMutex.RUnlock() + return len(fake.closeReasonArgsForCall) +} + +func (fake *FakeParticipant) CloseReasonCalls(stub func() types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = stub +} + +func (fake *FakeParticipant) CloseReasonReturns(result1 types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = nil + fake.closeReasonReturns = struct { + result1 types.ParticipantCloseReason + }{result1} +} + +func (fake *FakeParticipant) CloseReasonReturnsOnCall(i int, result1 types.ParticipantCloseReason) { + fake.closeReasonMutex.Lock() + defer fake.closeReasonMutex.Unlock() + fake.CloseReasonStub = nil + if fake.closeReasonReturnsOnCall == nil { + fake.closeReasonReturnsOnCall = make(map[int]struct { + result1 types.ParticipantCloseReason + }) + } + fake.closeReasonReturnsOnCall[i] = struct { + result1 types.ParticipantCloseReason + }{result1} +} + func (fake *FakeParticipant) DebugInfo() map[string]interface{} { fake.debugInfoMutex.Lock() ret, specificReturn := fake.debugInfoReturnsOnCall[len(fake.debugInfoArgsForCall)] @@ -1337,6 +1400,8 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.canSkipBroadcastMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.closeReasonMutex.RLock() + defer fake.closeReasonMutex.RUnlock() fake.debugInfoMutex.RLock() defer fake.debugInfoMutex.RUnlock() fake.getAudioLevelMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index abfd29331..b09ec48c2 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -213,10 +213,7 @@ func (r *RoomManager) Stop() { r.lock.RUnlock() for _, room := range rooms { - for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomManagerStop, false) - } - room.Close() + room.Close(types.ParticipantCloseReasonRoomManagerStop) } r.roomServers.Kill() @@ -727,10 +724,7 @@ func (r *RoomManager) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq } } else { room.Logger.Infow("deleting room") - for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonServiceRequestDeleteRoom, false) - } - room.Close() + room.Close(types.ParticipantCloseReasonServiceRequestDeleteRoom) } return &livekit.DeleteRoomResponse{}, nil } diff --git a/test/webhook_test.go b/test/webhook_test.go index 678c48c80..0463eea1a 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -35,6 +35,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/service" "github.com/livekit/livekit-server/pkg/testutils" ) @@ -108,7 +109,7 @@ func TestWebhooks(t *testing.T) { // room closed rm := server.RoomManager().GetRoom(context.Background(), testRoom) - rm.Close() + rm.Close(types.ParticipantCloseReasonNone) testutils.WithTimeout(t, func() string { if ts.GetEvent(webhook.EventRoomFinished) == nil { return "did not receive RoomFinished" From ff69c2aa11768e9441b906a40156c269611127eb Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 31 Jan 2024 15:33:39 +0530 Subject: [PATCH 113/114] Add debug to understand VP9 freezes. (#2434) * Add debug to understand VP9 freezes. Have reports of VP9 freezing in some rooms. Some data indicates that NACKs are received by SFU, but cannot get RTP packet when that happens. It is possible that the NACKs are all from dropped packets. Adding some debug to understand drops/NACKs better. * enable DD debug * comment out DD debug * markers * add back log about diff length mismatch * add back key frame mismatch logging * log skipped drops also --- pkg/sfu/buffer/buffer.go | 4 + pkg/sfu/downtrack.go | 12 ++- pkg/sfu/forwarder.go | 32 +++++++- .../dependencydescriptor.go | 74 ++++++++++--------- 4 files changed, 83 insertions(+), 39 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index fcf5ff7b5..d0147bab6 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -457,6 +457,10 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) { b.logger.Errorw("could not exclude range", err, "sn", rtpPacket.SequenceNumber, "esn", flowState.ExtSequenceNumber) } } + // TODO-VP9-DEBUG-REMOVE-START + snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber) + b.logger.Debugw("dropping padding packet", "sn", rtpPacket.SequenceNumber, "osn", flowState.ExtSequenceNumber, "msn", flowState.ExtSequenceNumber-snAdjustment, "error", err) + // TODO-VP9-DEBUG-REMOVE-END return } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 22cc6fadc..f12859142 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1471,7 +1471,6 @@ func (d *DownTrack) handleRTCP(bytes []byte) { pliOnce := true sendPliOnce := func() { _, layer := d.forwarder.CheckSync() - d.params.Logger.Debugw("received PLI/FIR RTCP", "layer", layer) if pliOnce { if layer != buffer.InvalidLayerSpatial { d.params.Logger.Debugw("sending PLI RTCP", "layer", layer) @@ -1626,6 +1625,17 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { if err == io.EOF { break } + // TODO-VP9-DEBUG-REMOVE-START + d.params.Logger.Debugw( + "NACK miss", + "isn", epm.sourceSeqNo, + "osn", epm.targetSeqNo, + "ots", epm.timestamp, + "eosn", epm.extSequenceNumber, + "eots", epm.extTimestamp, + "sid", epm.layer, + ) + // TODO-VP9-DEBUG-REMOVE-END nackMisses++ continue } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index e0ac55742..44de9ba71 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1695,8 +1695,36 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.shouldDrop = true if f.started && result.IsRelevant { // call to update highest incoming sequence number and other internal structures - if tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, result.RTPMarker); err == nil && tpRTP.snOrdering == SequenceNumberOrderingContiguous { - f.rtpMunger.PacketDropped(extPkt) + if tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, result.RTPMarker); err == nil { + if tpRTP.snOrdering == SequenceNumberOrderingContiguous { + // TODO-VP9-DEBUG-REMOVE-START + f.logger.Debugw( + "dropping packet", + "isn", extPkt.ExtSequenceNumber, + "its", extPkt.ExtTimestamp, + "osn", tpRTP.extSequenceNumber, + "ots", tpRTP.extTimestamp, + "payloadLen", len(extPkt.Packet.Payload), + "sid", extPkt.Spatial, + "tid", extPkt.Temporal, + ) + // TODO-VP9-DEBUG-REMOVE-END + f.rtpMunger.PacketDropped(extPkt) + } else { + // TODO-VP9-DEBUG-REMOVE-START + f.logger.Debugw( + "dropping packet skipped as not contiguous", + "isn", extPkt.ExtSequenceNumber, + "its", extPkt.ExtTimestamp, + "osn", tpRTP.extSequenceNumber, + "ots", tpRTP.extTimestamp, + "payloadLen", len(extPkt.Packet.Payload), + "sid", extPkt.Spatial, + "tid", extPkt.Temporal, + "snOrdering", tpRTP.snOrdering, + ) + // TODO-VP9-DEBUG-REMOVE-END + } } } return tp, nil diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 7a29e5d95..15d9df1eb 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -86,6 +86,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } if !d.keyFrameValid && dd.AttachedStructure == nil { + // d.logger.Debugw(fmt.Sprintf("drop packet, no attached structure, incoming %v, sn: %d, isKeyFrame: %v", extPkt.VideoLayer, extPkt.Packet.SequenceNumber, extPkt.KeyFrame)) return } @@ -94,10 +95,10 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r if err != nil { // do not mark as dropped as only error is an old frame // d.logger.Debugw(fmt.Sprintf("drop packet on decision error, incoming %v, fn: %d/%d, sn: %d", - // incomingLayer, - // dd.FrameNumber, - // extFrameNum, - // extPkt.Packet.SequenceNumber, + // incomingLayer, + // dd.FrameNumber, + // extFrameNum, + // extPkt.Packet.SequenceNumber, // ), "err", err) return } @@ -105,10 +106,10 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r case selectorDecisionDropped: // a packet of an alreadty dropped frame, maintain decision // d.logger.Debugw(fmt.Sprintf("drop packet already dropped, incoming %v, fn: %d/%d, sn: %d", - // incomingLayer, - // dd.FrameNumber, - // extFrameNum, - // extPkt.Packet.SequenceNumber, + // incomingLayer, + // dd.FrameNumber, + // extFrameNum, + // extPkt.Packet.SequenceNumber, // )) return } @@ -147,7 +148,8 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r "chainDiffs", fd.ChainDiffs, "chains", len(d.chains), "requiredKeyFrame", ddwdt.ExtKeyFrameNum, - "structureKeyFrame", d.extKeyFrameNum) + "structureKeyFrame", d.extKeyFrameNum, + ) d.decisions.AddDropped(extFrameNum) return } @@ -190,15 +192,15 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r if highestDecodeTarget.Target < 0 { // no active decode target, do not select // d.logger.Debugw( - // "drop packet for no target found", - // "highestDecodeTarget", highestDecodeTarget, - // "decodeTargets", d.decodeTargets, - // "tagetLayer", d.targetLayer, - // "incoming", incomingLayer, - // "fn", dd.FrameNumber, - // "efn", extFrameNum, - // "sn", extPkt.Packet.SequenceNumber, - // "isKeyFrame", extPkt.KeyFrame, + // "drop packet for no target found", + // "highestDecodeTarget", highestDecodeTarget, + // "decodeTargets", d.decodeTargets, + // "tagetLayer", d.targetLayer, + // "incoming", incomingLayer, + // "fn", dd.FrameNumber, + // "efn", extFrameNum, + // "sn", extPkt.Packet.SequenceNumber, + // "isKeyFrame", extPkt.KeyFrame, // ) d.decisions.AddDropped(extFrameNum) return @@ -207,15 +209,15 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r // DD-TODO : if bandwidth in congest, could drop the 'Discardable' frame if dti == dede.DecodeTargetNotPresent { // d.logger.Debugw( - // "drop packet for decode target not present", - // "highestDecodeTarget", highestDecodeTarget, - // "decodeTargets", d.decodeTargets, - // "tagetLayer", d.targetLayer, - // "incoming", incomingLayer, - // "fn", dd.FrameNumber, - // "efn", extFrameNum, - // "sn", extPkt.Packet.SequenceNumber, - // "isKeyFrame", extPkt.KeyFrame, + // "drop packet for decode target not present", + // "highestDecodeTarget", highestDecodeTarget, + // "decodeTargets", d.decodeTargets, + // "tagetLayer", d.targetLayer, + // "incoming", incomingLayer, + // "fn", dd.FrameNumber, + // "efn", extFrameNum, + // "sn", extPkt.Packet.SequenceNumber, + // "isKeyFrame", extPkt.KeyFrame, // ) d.decisions.AddDropped(extFrameNum) return @@ -237,15 +239,15 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } if !isDecodable { // d.logger.Debugw( - // "drop packet for not decodable", - // "highestDecodeTarget", highestDecodeTarget, - // "decodeTargets", d.decodeTargets, - // "tagetLayer", d.targetLayer, - // "incoming", incomingLayer, - // "fn", dd.FrameNumber, - // "efn", extFrameNum, - // "sn", extPkt.Packet.SequenceNumber, - // "isKeyFrame", extPkt.KeyFrame, + // "drop packet for not decodable", + // "highestDecodeTarget", highestDecodeTarget, + // "decodeTargets", d.decodeTargets, + // "tagetLayer", d.targetLayer, + // "incoming", incomingLayer, + // "fn", dd.FrameNumber, + // "efn", extFrameNum, + // "sn", extPkt.Packet.SequenceNumber, + // "isKeyFrame", extPkt.KeyFrame, // ) d.decisions.AddDropped(extFrameNum) return From 174e69c81d10c7ee373425a9e81b8d2dec892d40 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 2 Feb 2024 08:52:52 +0530 Subject: [PATCH 114/114] Restore min score to 30. (#2435) Was at 20 when LOST was introduced, but was going to 20 even when under not LOST conditions. When there are packets, want the min to be at 30. Going down to 20 resulted in reporting LOST quality even when packets were flowing (although they were experiencing heavy loss and quality would have been very bad, yet they are not lost). Also, sample warning about adding packet to bucket even more. --- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/connectionquality/scorer.go | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index d0147bab6..f27946064 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -476,7 +476,7 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) { if err != nil { if errors.Is(err, bucket.ErrPacketTooOld) { packetTooOldCount := b.packetTooOldCount.Inc() - if packetTooOldCount%20 == 0 { + if (packetTooOldCount-1)%100 == 0 { b.logger.Warnw("could not add packet to bucket", err, "count", packetTooOldCount) } } else if err != bucket.ErrRTXPacket { diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index ffc3961da..d5c3b019c 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -29,9 +29,8 @@ const ( MaxMOS = float32(4.5) MinMOS = float32(1.0) - cMaxScore = float64(100.0) - cMinScore = float64(20.0) - cPausedPoorScore = float64(30.0) + cMaxScore = float64(100.0) + cMinScore = float64(30.0) increaseFactor = float64(0.4) // slower increase, i. e. when score is recovering move up slower -> conservative decreaseFactor = float64(0.7) // faster decrease, i. e. when score is dropping move down faster -> aggressive to be responsive to quality drops @@ -305,7 +304,7 @@ func (q *qualityScorer) updatePauseAtLocked(isPaused bool, at time.Time) { q.layerDistance.Reset() q.pausedAt = at - q.score = cPausedPoorScore + q.score = cMinScore } } else { if q.isPaused() { @@ -364,7 +363,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // considered (as long as enough time has passed since unmute). // // Similarly, when paused (possibly due to congestion), score is immediately - // set to cPausedPoorScore for responsiveness. The layer transision is reest. + // set to cMinScore for responsiveness. The layer transision is reest. // On a resume, quality climbs back up using normal operation. if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() { q.lastUpdateAt = at @@ -404,12 +403,12 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { factor = decreaseFactor } score = factor*score + (1.0-factor)*q.score - } - if score < cMinScore { - // lower bound to prevent score from becoming very small values due to extreme conditions. - // Without a lower bound, it can get so low that it takes a long time to climb back to - // better quality even under excellent conditions. - score = cMinScore + if score < cMinScore { + // lower bound to prevent score from becoming very small values due to extreme conditions. + // Without a lower bound, it can get so low that it takes a long time to climb back to + // better quality even under excellent conditions. + score = cMinScore + } } prevCQ := scoreToConnectionQuality(q.score) currCQ := scoreToConnectionQuality(score)