Add interface and ipfilter to udpmux option (#1270)

* Add interface and ipfilter to udpmux option * validate external ip is accessable by client * add context * use external ip only for firefox * fix mapping error * Update pion/ice and use external ip only for firefox * Use single external ip for NAT1To1Ips if validate failed * update pion/ice
2026-05-14 18:25:24 +00:00 · 2022-12-30 16:01:12 +08:00
parent 112d6fc18b
commit c393a5f8dd
5 changed files with 157 additions and 26 deletions
@@ -26,7 +26,7 @@ require (
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.5.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pion/ice/v2 v2.2.12
+	github.com/pion/ice/v2 v2.2.13
 	github.com/pion/interceptor v0.1.12
 	github.com/pion/logging v0.2.2
 	github.com/pion/rtcp v1.2.10
@@ -94,9 +94,9 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.2.0 // indirect
+	golang.org/x/crypto v0.4.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.3.0 // indirect
+	golang.org/x/net v0.4.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/tools v0.1.12 // indirect
@@ -304,8 +304,9 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew
 github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0=
 github.com/pion/dtls/v2 v2.1.5 h1:jlh2vtIyUBShchoTDqpCCqiYCyRFJ/lvf/gQ8TALs+c=
 github.com/pion/dtls/v2 v2.1.5/go.mod h1:BqCE7xPZbPSubGasRoDFJeTsyJtdD1FanJYL0JGheqY=
-github.com/pion/ice/v2 v2.2.12 h1:n3M3lUMKQM5IoofhJo73D3qVla+mJN2nVvbSPq32Nig=
 github.com/pion/ice/v2 v2.2.12/go.mod h1:z2KXVFyRkmjetRlaVRgjO9U3ShKwzhlUylvxKfHfd5A=
+github.com/pion/ice/v2 v2.2.13 h1:NvLtzwcyob6wXgFqLmVQbGB3s9zzWmOegNMKYig5l9M=
+github.com/pion/ice/v2 v2.2.13/go.mod h1:eFO4/1zCI+a3OFVt7l7kP+5jWCuZo8FwU2UwEa3+164=
 github.com/pion/interceptor v0.1.11/go.mod h1:tbtKjZY14awXd7Bq0mmWvgtHB5MDaRN7HV3OZ/uy7s8=
 github.com/pion/interceptor v0.1.12 h1:CslaNriCFUItiXS5o+hh5lpL0t0ytQkFnUcbbCs2Zq8=
 github.com/pion/interceptor v0.1.12/go.mod h1:bDtgAD9dRkBZpWHGKaoKb42FhDHTG2rX8Ii9LRALLVA=
@@ -440,8 +441,8 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
-golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
+golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -525,8 +526,10 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -618,6 +621,7 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -8,6 +8,8 @@ import (

 	"github.com/pion/stun"
 	"github.com/pkg/errors"
+
+	"github.com/livekit/protocol/logger"
 )

 func (conf *Config) determineIP() (string, error) {
@@ -19,7 +21,7 @@ func (conf *Config) determineIP() (string, error) {
 		var err error
 		for i := 0; i < 3; i++ {
 			var ip string
-			ip, err = GetExternalIP(stunServers, nil)
+			ip, err = GetExternalIP(context.Background(), stunServers, nil)
 			if err == nil {
 				return ip, nil
 			} else {
@@ -83,8 +85,9 @@ func GetLocalIPAddresses(includeLoopback bool) ([]string, error) {
 	return nil, fmt.Errorf("could not find local IP address")
 }

-// GetExternalIP return external IP for localAddr from stun server. If localAddr is nil, a local address is chosen automatically.
-func GetExternalIP(stunServers []string, localAddr net.Addr) (string, error) {
+// GetExternalIP return external IP for localAddr from stun server. If localAddr is nil, a local address is chosen automatically,
+// else the address will be used to validate the external IP is accessible from the outside.
+func GetExternalIP(ctx context.Context, stunServers []string, localAddr net.Addr) (string, error) {
 	if len(stunServers) == 0 {
 		return "", errors.New("STUN servers are required but not defined")
 	}
@@ -129,12 +132,16 @@ func GetExternalIP(stunServers []string, localAddr net.Addr) (string, error) {
 		return "", err
 	}

-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx1, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 	select {
 	case nodeIP := <-ipChan:
-		return nodeIP, nil
-	case <-ctx.Done():
+		if localAddr == nil {
+			return nodeIP, nil
+		}
+		_ = c.Close()
+		return nodeIP, validateExternalIP(ctx1, nodeIP, localAddr.(*net.UDPAddr))
+	case <-ctx1.Done():
 		msg := "could not determine public IP"
 		if stunErr != nil {
 			return "", errors.Wrap(stunErr, msg)
@@ -143,3 +150,51 @@ func GetExternalIP(stunServers []string, localAddr net.Addr) (string, error) {
 		}
 	}
 }
+
+// validateExternalIP validates that the external IP is accessible from the outside by listen the local address,
+// it will send a magic string to the external IP and check the string is received by the local address.
+func validateExternalIP(ctx context.Context, nodeIP string, addr *net.UDPAddr) error {
+	srv, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		return err
+	}
+	defer srv.Close()
+
+	magicString := "9#B8D2Nvg2xg5P$ZRwJ+f)*^Nne6*W3WamGY"
+
+	validCh := make(chan struct{})
+	go func() {
+		buf := make([]byte, 1024)
+		for {
+			n, err := srv.Read(buf)
+			if err != nil {
+				logger.Debugw("error reading from UDP socket", "err", err)
+				return
+			}
+			if string(buf[:n]) == magicString {
+				close(validCh)
+				return
+			}
+		}
+	}()
+
+	cli, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP(nodeIP), Port: srv.LocalAddr().(*net.UDPAddr).Port})
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	if _, err = cli.Write([]byte(magicString)); err != nil {
+		return err
+	}
+
+	ctx1, cancel := context.WithTimeout(ctx, 1*time.Second)
+	defer cancel()
+	select {
+	case <-validCh:
+		return nil
+	case <-ctx1.Done():
+		break
+	}
+	return fmt.Errorf("could not validate external IP")
+}
@@ -1,9 +1,13 @@
 package rtc

 import (
+	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
+	"strings"
+	"sync"
 	"time"

 	"github.com/pion/ice/v2"
@@ -32,7 +36,7 @@ type WebRTCConfig struct {
 	TCPMuxListener *net.TCPListener
 	Publisher      DirectionConfig
 	Subscriber     DirectionConfig
-	ExternalIP     string
+	NAT1To1IPs     []string
 }

 type ReceiverConfig struct {
@@ -70,8 +74,10 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 		LoggerFactory: logging.NewLoggerFactory(logger.GetLogger()),
 	}

+	var ifFilter func(string) bool
 	if len(rtcConf.Interfaces.Includes) != 0 || len(rtcConf.Interfaces.Excludes) != 0 {
-		s.SetInterfaceFilter(InterfaceFilterFromConf(rtcConf.Interfaces))
+		ifFilter = InterfaceFilterFromConf(rtcConf.Interfaces)
+		s.SetInterfaceFilter(ifFilter)
 	}

 	var ipFilter func(net.IP) bool
@@ -84,7 +90,7 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 		s.SetIPFilter(filter)
 	}

-	var confExternalIP string
+	var nat1to1IPs []string
 	// force it to the node IPs that the user has set
 	if externalIP != "" && (conf.RTC.UseExternalIP || (conf.RTC.NodeIP != "" && !conf.RTC.NodeIPAutoGenerated)) {
 		if conf.RTC.UseExternalIP {
@@ -92,9 +98,14 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 			if err != nil {
 				return nil, err
 			}
-			logger.Debugw("using external IPs", "ips", ips)
-			s.SetNAT1To1IPs(ips, webrtc.ICECandidateTypeHost)
-			confExternalIP = externalIP
+			if len(ips) == 0 {
+				logger.Infow("no external IPs found, using node IP for NAT1To1Ips", "ip", externalIP)
+				s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost)
+			} else {
+				logger.Infow("using external IPs", "ips", ips)
+				s.SetNAT1To1IPs(ips, webrtc.ICECandidateTypeHost)
+			}
+			nat1to1IPs = ips
 		} else {
 			s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost)
 		}
@@ -125,6 +136,12 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 			if rtcConf.EnableLoopbackCandidate {
 				opts = append(opts, ice.UDPMuxFromPortWithLoopback())
 			}
+			if ipFilter != nil {
+				opts = append(opts, ice.UDPMuxFromPortWithIPFilter(ipFilter))
+			}
+			if ifFilter != nil {
+				opts = append(opts, ice.UDPMuxFromPortWithInterfaceFilter(ifFilter))
+			}
 			udpMux, err := ice.NewMultiUDPMuxFromPort(int(rtcConf.UDPPort), opts...)
 			if err != nil {
 				return nil, err
@@ -240,7 +257,7 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err
 		TCPMuxListener: tcpListener,
 		Publisher:      publisherConfig,
 		Subscriber:     subscriberConfig,
-		ExternalIP:     confExternalIP,
+		NAT1To1IPs:     nat1to1IPs,
 	}, nil
 }

@@ -271,18 +288,43 @@ func getNAT1to1IPsForConf(conf *config.Config, ipFilter func(net.IP) bool) ([]st
 		localIP    string
 	}
 	addrCh := make(chan ipmapping, len(localIPs))
+
+	var udpPorts []int
+	if conf.RTC.ICEPortRangeStart != 0 && conf.RTC.ICEPortRangeEnd != 0 {
+		portRangeStart, portRangeEnd := uint16(conf.RTC.ICEPortRangeStart), uint16(conf.RTC.ICEPortRangeEnd)
+		for i := 0; i < 5; i++ {
+			udpPorts = append(udpPorts, rand.Intn(int(portRangeEnd-portRangeStart))+int(portRangeStart))
+		}
+	} else if conf.RTC.UDPPort != 0 {
+		udpPorts = append(udpPorts, int(conf.RTC.UDPPort))
+	} else {
+		udpPorts = append(udpPorts, 0)
+	}
+
+	var wg sync.WaitGroup
+	ctx, cancel := context.WithCancel(context.Background())
 	for _, ip := range localIPs {
 		if ipFilter != nil && !ipFilter(net.ParseIP(ip)) {
 			continue
 		}

+		wg.Add(1)
 		go func(localIP string) {
-			addr, err := config.GetExternalIP(stunServers, &net.UDPAddr{IP: net.ParseIP(localIP)})
-			if err != nil {
-				logger.Infow("failed to get external ip", "local", localIP, "err", err)
+			defer wg.Done()
+			for _, port := range udpPorts {
+				addr, err := config.GetExternalIP(ctx, stunServers, &net.UDPAddr{IP: net.ParseIP(localIP), Port: port})
+				if err != nil {
+					if strings.Contains(err.Error(), "address already in use") {
+						logger.Debugw("failed to get external ip, address already in use", "local", localIP, "port", port)
+						continue
+					}
+					logger.Infow("failed to get external ip", "local", localIP, "err", err)
+					return
+				}
+				addrCh <- ipmapping{externalIP: addr, localIP: localIP}
 				return
 			}
-			addrCh <- ipmapping{externalIP: addr, localIP: localIP}
+			logger.Infow("failed to get external ip after all ports tried", "local", localIP, "ports", udpPorts)
 		}(ip)
 	}

@@ -312,6 +354,13 @@ done:
 			break done
 		}
 	}
+	cancel()
+	wg.Wait()
+
+	if len(natMapping) == 0 {
+		// no external ip resolved
+		return nil, nil
+	}

 	// mapping unresolved local ip to itself
 	for _, local := range localIPs {
@@ -2,6 +2,7 @@ package rtc

 import (
 	"fmt"
+	"net"
 	"strings"
 	"sync"
 	"time"
@@ -258,8 +259,30 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat
 	se.SetICETimeouts(iceDisconnectedTimeout, iceFailedTimeout, iceKeepaliveInterval)

 	// if client don't support prflx over relay, we should not expose private address to it, use single external ip as host candidate
-	if !params.ClientInfo.SupportPrflxOverRelay() && params.Config.ExternalIP != "" {
-		se.SetNAT1To1IPs([]string{params.Config.ExternalIP}, webrtc.ICECandidateTypeHost)
+	if !params.ClientInfo.SupportPrflxOverRelay() && len(params.Config.NAT1To1IPs) > 0 {
+		var nat1to1Ips []string
+		var includeIps []string
+		for _, mapping := range params.Config.NAT1To1IPs {
+			if ips := strings.Split(mapping, "/"); len(ips) == 2 {
+				if ips[0] != ips[1] {
+					nat1to1Ips = append(nat1to1Ips, mapping)
+					includeIps = append(includeIps, ips[1])
+				}
+			}
+		}
+		if len(nat1to1Ips) > 0 {
+			params.Logger.Infow("client doesn't support prflx over relay, use external ip only as host candidate", "ips", nat1to1Ips)
+			se.SetNAT1To1IPs(nat1to1Ips, webrtc.ICECandidateTypeHost)
+			se.SetIPFilter(func (ip net.IP) bool {
+				ipstr := ip.String()
+				for _, inc := range includeIps {
+					if inc == ipstr {
+						return true
+					}
+				}
+				return false
+			})
+		}
 	}

 	lf := serverlogger.NewLoggerFactory(params.Logger)