Release 2.1.0

According to Apple: Swift compiler no longer supports statically linking the Swift libraries. They're included in the OS by default starting with macOS Mojave 10.14.4. For macOS Mojave 10.14.3 and earlier, there's an optional Swift library package that can be downloaded from "More Downloads" for Apple Developers at https://developer.apple.com/download/more/

.gitignore

@@ -12,3 +12,5 @@
 
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
+
+.idea/

Changelog.md

@@ -4,12 +4,34 @@ This Changelog records major changes between versions.
 
 Not all changes are recorded. Please check git log for details.
 
+## Version 2.1.0
+
+- Add `local_addr` configuration for doh-server (#39)
+- Fix a problem when compiling on macOS 10.14.4 or newer
+- Add Quad9 DoH server to the example `doh-client.conf`
+- Use TCP when appropriate for the given query type/response (AXFR/IXFR)
+
+## Version 2.0.1
+
+- Fix a crash with the random load balancing algorithm.
+
+## Version 2.0.0
+
+**This is a breaking change!** Please update the configuration file after upgrading.
+
+- Implemented two upstream server selector algorithms: `weighted_round_robin` and `lvs_weighted_round_robin`.
+- Add a configuration option for doh-server: `log_guessed_client_ip`.
+
+## Version 1.4.2
+
+- Add PID file feature for systems which lacks a cgroup-based process tracker.
+- Remove dns.ErrTruncated according to <https://github.com/miekg/dns/pull/815>.
+
 ## Version 1.4.1
 
-- Add a configuration option: `debug_http_headers` (e.g. Add `CF-Ray` to diagnose CloudFlare's resolver)
+- Add a configuration option: `debug_http_headers` (e.g. Add `CF-Ray` to diagnose Cloudflare's resolver)
 - Add a configuration option: `passrthrough`
 - macOS logger is rebuilt with static libswiftCore
-- Fix a segfault when `no_cookies=true`
 - Fix HTTP stream leaking problem, which may cause massive half-open connections if HTTP/1 is in use
 - Utilize Go's cancelable context to detect timeouts more reliably.
 - Fix interoperation problems with gDNS

Makefile

@@ -62,7 +62,7 @@ deps:
 	$(GOGET_UPDATE) github.com/m13253/dns-over-https/json-dns
 	$(GOGET) ./doh-client ./doh-server
 
-doh-client/doh-client: deps doh-client/client.go doh-client/config.go doh-client/google.go doh-client/ietf.go doh-client/main.go doh-client/version.go json-dns/error.go json-dns/globalip.go json-dns/marshal.go json-dns/response.go json-dns/unmarshal.go
+doh-client/doh-client: deps doh-client/client.go doh-client/config/config.go doh-client/google.go doh-client/ietf.go doh-client/main.go doh-client/version.go json-dns/error.go json-dns/globalip.go json-dns/marshal.go json-dns/response.go json-dns/unmarshal.go
 	cd doh-client && $(GOBUILD)
 
 doh-server/doh-server: deps doh-server/config.go doh-server/google.go doh-server/ietf.go doh-server/main.go doh-server/server.go doh-server/version.go json-dns/error.go json-dns/globalip.go json-dns/marshal.go json-dns/response.go json-dns/unmarshal.go

darwin-wrapper/Makefile

@@ -6,7 +6,7 @@ PREFIX = /usr/local
 all: doh-logger
 
 doh-logger: doh-logger.swift
-	$(SWIFTC) -o $@ -O -static-stdlib $<
+	$(SWIFTC) -o $@ -O $<
 
 clean:
 	rm -f doh-logger

doh-client/client.go

@@ -31,11 +31,14 @@ import (
 	"net"
 	"net/http"
 	"net/http/cookiejar"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/m13253/dns-over-https/doh-client/config"
+	"github.com/m13253/dns-over-https/doh-client/selector"
 	"github.com/m13253/dns-over-https/json-dns"
 	"github.com/miekg/dns"
 	"golang.org/x/net/http2"
@@ -43,7 +46,7 @@ import (
 )
 
 type Client struct {
-	conf                 *config
+	conf                 *config.Config
 	bootstrap            []string
 	passthrough          []string
 	udpClient            *dns.Client
@@ -56,6 +59,7 @@ type Client struct {
 	httpTransport        *http.Transport
 	httpClient           *http.Client
 	httpClientLastCreate time.Time
+	selector             selector.Selector
 }
 
 type DNSRequest struct {
@@ -68,7 +72,7 @@ type DNSRequest struct {
 	err               error
 }
 
-func NewClient(conf *config) (c *Client, err error) {
+func NewClient(conf *config.Config) (c *Client, err error) {
 	c = &Client{
 		conf: conf,
 	}
@@ -78,11 +82,11 @@ func NewClient(conf *config) (c *Client, err error) {
 	c.udpClient = &dns.Client{
 		Net:     "udp",
 		UDPSize: dns.DefaultMsgSize,
-		Timeout: time.Duration(conf.Timeout) * time.Second,
+		Timeout: time.Duration(conf.Other.Timeout) * time.Second,
 	}
 	c.tcpClient = &dns.Client{
 		Net:     "tcp",
-		Timeout: time.Duration(conf.Timeout) * time.Second,
+		Timeout: time.Duration(conf.Other.Timeout) * time.Second,
 	}
 	for _, addr := range conf.Listen {
 		c.udpServers = append(c.udpServers, &dns.Server{
@@ -98,9 +102,9 @@ func NewClient(conf *config) (c *Client, err error) {
 		})
 	}
 	c.bootstrapResolver = net.DefaultResolver
-	if len(conf.Bootstrap) != 0 {
-		c.bootstrap = make([]string, len(conf.Bootstrap))
-		for i, bootstrap := range conf.Bootstrap {
+	if len(conf.Other.Bootstrap) != 0 {
+		c.bootstrap = make([]string, len(conf.Other.Bootstrap))
+		for i, bootstrap := range conf.Other.Bootstrap {
 			bootstrapAddr, err := net.ResolveUDPAddr("udp", bootstrap)
 			if err != nil {
 				bootstrapAddr, err = net.ResolveUDPAddr("udp", "["+bootstrap+"]:53")
@@ -120,9 +124,9 @@ func NewClient(conf *config) (c *Client, err error) {
 				return conn, err
 			},
 		}
-		if len(conf.Passthrough) != 0 {
-			c.passthrough = make([]string, len(conf.Passthrough))
-			for i, passthrough := range conf.Passthrough {
+		if len(conf.Other.Passthrough) != 0 {
+			c.passthrough = make([]string, len(conf.Other.Passthrough))
+			for i, passthrough := range conf.Other.Passthrough {
 				if punycode, err := idna.ToASCII(passthrough); err != nil {
 					passthrough = punycode
 				}
@@ -133,7 +137,7 @@ func NewClient(conf *config) (c *Client, err error) {
 	// Most CDNs require Cookie support to prevent DDoS attack.
 	// Disabling Cookie does not effectively prevent tracking,
 	// so I will leave it on to make anti-DDoS services happy.
-	if !c.conf.NoCookies {
+	if !c.conf.Other.NoCookies {
 		c.cookieJar, err = cookiejar.New(nil)
 		if err != nil {
 			return nil, err
@@ -147,23 +151,93 @@ func NewClient(conf *config) (c *Client, err error) {
 	if err != nil {
 		return nil, err
 	}
+
+	switch c.conf.Upstream.UpstreamSelector {
+	case config.NginxWRR:
+		if c.conf.Other.Verbose {
+			log.Println(config.NginxWRR, "mode start")
+		}
+
+		s := selector.NewNginxWRRSelector(time.Duration(c.conf.Other.Timeout) * time.Second)
+		for _, u := range c.conf.Upstream.UpstreamGoogle {
+			if err := s.Add(u.URL, selector.Google, u.Weight); err != nil {
+				return nil, err
+			}
+		}
+
+		for _, u := range c.conf.Upstream.UpstreamIETF {
+			if err := s.Add(u.URL, selector.IETF, u.Weight); err != nil {
+				return nil, err
+			}
+		}
+
+		c.selector = s
+
+	case config.LVSWRR:
+		if c.conf.Other.Verbose {
+			log.Println(config.LVSWRR, "mode start")
+		}
+
+		s := selector.NewLVSWRRSelector(time.Duration(c.conf.Other.Timeout) * time.Second)
+		for _, u := range c.conf.Upstream.UpstreamGoogle {
+			if err := s.Add(u.URL, selector.Google, u.Weight); err != nil {
+				return nil, err
+			}
+		}
+
+		for _, u := range c.conf.Upstream.UpstreamIETF {
+			if err := s.Add(u.URL, selector.IETF, u.Weight); err != nil {
+				return nil, err
+			}
+		}
+
+		c.selector = s
+
+	default:
+		if c.conf.Other.Verbose {
+			log.Println(config.Random, "mode start")
+		}
+
+		// if selector is invalid or random, use random selector, or should we stop program and let user knows he is wrong?
+		s := selector.NewRandomSelector()
+		for _, u := range c.conf.Upstream.UpstreamGoogle {
+			if err := s.Add(u.URL, selector.Google); err != nil {
+				return nil, err
+			}
+		}
+
+		for _, u := range c.conf.Upstream.UpstreamIETF {
+			if err := s.Add(u.URL, selector.IETF); err != nil {
+				return nil, err
+			}
+		}
+
+		c.selector = s
+	}
+
+	if c.conf.Other.Verbose {
+		if reporter, ok := c.selector.(selector.DebugReporter); ok {
+			reporter.ReportWeights()
+		}
+	}
+
 	return c, nil
 }
 
 func (c *Client) newHTTPClient() error {
 	c.httpClientMux.Lock()
 	defer c.httpClientMux.Unlock()
-	if !c.httpClientLastCreate.IsZero() && time.Now().Sub(c.httpClientLastCreate) < time.Duration(c.conf.Timeout)*time.Second {
+	if !c.httpClientLastCreate.IsZero() && time.Since(c.httpClientLastCreate) < time.Duration(c.conf.Other.Timeout)*time.Second {
 		return nil
 	}
 	if c.httpTransport != nil {
 		c.httpTransport.CloseIdleConnections()
 	}
 	dialer := &net.Dialer{
-		Timeout:   time.Duration(c.conf.Timeout) * time.Second,
+		Timeout:   time.Duration(c.conf.Other.Timeout) * time.Second,
 		KeepAlive: 30 * time.Second,
-		DualStack: true,
-		Resolver:  c.bootstrapResolver,
+		// DualStack: true,
+		Resolver: c.bootstrapResolver,
 	}
 	c.httpTransport = &http.Transport{
 		DialContext:           dialer.DialContext,
@@ -172,9 +246,9 @@ func (c *Client) newHTTPClient() error {
 		MaxIdleConns:          100,
 		MaxIdleConnsPerHost:   10,
 		Proxy:                 http.ProxyFromEnvironment,
-		TLSHandshakeTimeout:   time.Duration(c.conf.Timeout) * time.Second,
+		TLSHandshakeTimeout:   time.Duration(c.conf.Other.Timeout) * time.Second,
 	}
-	if c.conf.NoIPv6 {
+	if c.conf.Other.NoIPv6 {
 		c.httpTransport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
 			if strings.HasPrefix(network, "tcp") {
 				network = "tcp4"
@@ -206,6 +280,9 @@ func (c *Client) Start() error {
 		}(srv)
 	}
 
+	// start evaluation loop
+	c.selector.StartEvaluate()
+
 	for i := 0; i < cap(results); i++ {
 		err := <-results
 		if err != nil {
@@ -213,14 +290,15 @@ func (c *Client) Start() error {
 		}
 	}
 	close(results)
+
 	return nil
 }
 
 func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.conf.Timeout)*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.conf.Other.Timeout)*time.Second)
 	defer cancel()
 
-	if r.Response == true {
+	if r.Response {
 		log.Println("Received a response packet")
 		return
 	}
@@ -246,7 +324,7 @@ func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
 	} else {
 		questionType = strconv.FormatUint(uint64(question.Qtype), 10)
 	}
-	if c.conf.Verbose {
+	if c.conf.Other.Verbose {
 		fmt.Printf("%s - - [%s] \"%s %s %s\"\n", w.RemoteAddr(), time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
 	}
 
@@ -273,7 +351,7 @@ func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
 		} else {
 			reply, _, err = c.tcpClient.Exchange(r, upstream)
 		}
-		if err == nil || err == dns.ErrTruncated {
+		if err == nil {
 			w.WriteMsg(reply)
 			return
 		}
@@ -284,64 +362,80 @@ func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
 		return
 	}
 
-	requestType := ""
-	if len(c.conf.UpstreamIETF) == 0 {
-		requestType = "application/dns-json"
-	} else if len(c.conf.UpstreamGoogle) == 0 {
-		requestType = "application/dns-message"
-	} else {
-		numServers := len(c.conf.UpstreamGoogle) + len(c.conf.UpstreamIETF)
-		random := rand.Intn(numServers)
-		if random < len(c.conf.UpstreamGoogle) {
-			requestType = "application/dns-json"
-		} else {
-			requestType = "application/dns-message"
-		}
+	upstream := c.selector.Get()
+	requestType := upstream.RequestType
+
+	if c.conf.Other.Verbose {
+		log.Println("choose upstream:", upstream)
 	}
 
 	var req *DNSRequest
-	if requestType == "application/dns-json" {
-		req = c.generateRequestGoogle(ctx, w, r, isTCP)
-	} else if requestType == "application/dns-message" {
-		req = c.generateRequestIETF(ctx, w, r, isTCP)
-	} else {
+	switch requestType {
+	case "application/dns-json":
+		req = c.generateRequestGoogle(ctx, w, r, isTCP, upstream)
+
+	case "application/dns-message":
+		req = c.generateRequestIETF(ctx, w, r, isTCP, upstream)
+
+	default:
 		panic("Unknown request Content-Type")
 	}
 
-	if req.response != nil {
-		defer req.response.Body.Close()
-		for _, header := range c.conf.DebugHTTPHeaders {
-			if value := req.response.Header.Get(header); value != "" {
-				log.Printf("%s: %s\n", header, value)
+	if req.err != nil {
+		if urlErr, ok := req.err.(*url.Error); ok {
+			// should we only check timeout?
+			if urlErr.Timeout() {
+				c.selector.ReportUpstreamStatus(upstream, selector.Timeout)
 			}
 		}
-	}
-	if req.err != nil {
+
 		return
 	}
 
-	contentType := ""
-	candidateType := strings.SplitN(req.response.Header.Get("Content-Type"), ";", 2)[0]
-	if candidateType == "application/json" {
-		contentType = "application/json"
-	} else if candidateType == "application/dns-message" {
-		contentType = "application/dns-message"
-	} else if candidateType == "application/dns-udpwireformat" {
-		contentType = "application/dns-message"
-	} else {
-		if requestType == "application/dns-json" {
-			contentType = "application/json"
-		} else if requestType == "application/dns-message" {
-			contentType = "application/dns-message"
+	// if req.err == nil, req.response != nil
+	defer req.response.Body.Close()
+
+	for _, header := range c.conf.Other.DebugHTTPHeaders {
+		if value := req.response.Header.Get(header); value != "" {
+			log.Printf("%s: %s\n", header, value)
 		}
 	}
 
-	if contentType == "application/json" {
+	candidateType := strings.SplitN(req.response.Header.Get("Content-Type"), ";", 2)[0]
+
+	switch candidateType {
+	case "application/json":
 		c.parseResponseGoogle(ctx, w, r, isTCP, req)
-	} else if contentType == "application/dns-message" {
+
+	case "application/dns-message", "application/dns-udpwireformat":
 		c.parseResponseIETF(ctx, w, r, isTCP, req)
-	} else {
-		panic("Unknown response Content-Type")
+
+	default:
+		switch requestType {
+		case "application/dns-json":
+			c.parseResponseGoogle(ctx, w, r, isTCP, req)
+
+		case "application/dns-message":
+			c.parseResponseIETF(ctx, w, r, isTCP, req)
+
+		default:
+			panic("Unknown response Content-Type")
+		}
+	}
+
+	// https://developers.cloudflare.com/1.1.1.1/dns-over-https/request-structure/ says
+	// returns code will be 200 / 400 / 413 / 415 / 504, some server will return 503, so
+	// I think if status code is 5xx, upstream must has some problems
+	/*if req.response.StatusCode/100 == 5 {
+		c.selector.ReportUpstreamStatus(upstream, selector.Medium)
+	}*/
+
+	switch req.response.StatusCode / 100 {
+	case 5:
+		c.selector.ReportUpstreamStatus(upstream, selector.Error)
+
+	case 2:
+		c.selector.ReportUpstreamStatus(upstream, selector.OK)
 	}
 }
 
@@ -360,7 +454,7 @@ var (
 
 func (c *Client) findClientIP(w dns.ResponseWriter, r *dns.Msg) (ednsClientAddress net.IP, ednsClientNetmask uint8) {
 	ednsClientNetmask = 255
-	if c.conf.NoECS {
+	if c.conf.Other.NoECS {
 		return net.IPv4(0, 0, 0, 0), 0
 	}
 	if opt := r.IsEdns0(); opt != nil {

doh-client/config/config.go

@@ -21,7 +21,7 @@
    DEALINGS IN THE SOFTWARE.
 */
 
-package main
+package config
 
 import (
 	"fmt"
@@ -29,10 +29,24 @@ import (
 	"github.com/BurntSushi/toml"
 )
 
-type config struct {
-	Listen           []string `toml:"listen"`
-	UpstreamGoogle   []string `toml:"upstream_google"`
-	UpstreamIETF     []string `toml:"upstream_ietf"`
+const (
+	Random   = "random"
+	NginxWRR = "weighted_round_robin"
+	LVSWRR   = "lvs_weighted_round_robin"
+)
+
+type upstreamDetail struct {
+	URL    string `toml:"url"`
+	Weight int32  `toml:"weight"`
+}
+
+type upstream struct {
+	UpstreamGoogle   []upstreamDetail `toml:"upstream_google"`
+	UpstreamIETF     []upstreamDetail `toml:"upstream_ietf"`
+	UpstreamSelector string           `toml:"upstream_selector"` // usable: random or weighted_random
+}
+
+type others struct {
 	Bootstrap        []string `toml:"bootstrap"`
 	Passthrough      []string `toml:"passthrough"`
 	Timeout          uint     `toml:"timeout"`
@@ -43,8 +57,14 @@ type config struct {
 	DebugHTTPHeaders []string `toml:"debug_http_headers"`
 }
 
-func loadConfig(path string) (*config, error) {
-	conf := &config{}
+type Config struct {
+	Listen   []string `toml:"listen"`
+	Upstream upstream `toml:"upstream"`
+	Other    others   `toml:"others"`
+}
+
+func LoadConfig(path string) (*Config, error) {
+	conf := &Config{}
 	metaData, err := toml.DecodeFile(path, conf)
 	if err != nil {
 		return nil, err
@@ -56,11 +76,15 @@ func loadConfig(path string) (*config, error) {
 	if len(conf.Listen) == 0 {
 		conf.Listen = []string{"127.0.0.1:53", "[::1]:53"}
 	}
-	if len(conf.UpstreamGoogle) == 0 && len(conf.UpstreamIETF) == 0 {
-		conf.UpstreamGoogle = []string{"https://dns.google.com/resolve"}
+	if len(conf.Upstream.UpstreamGoogle) == 0 && len(conf.Upstream.UpstreamIETF) == 0 {
+		conf.Upstream.UpstreamGoogle = []upstreamDetail{{URL: "https://dns.google.com/resolve", Weight: 50}}
 	}
-	if conf.Timeout == 0 {
-		conf.Timeout = 10
+	if conf.Other.Timeout == 0 {
+		conf.Other.Timeout = 10
+	}
+
+	if conf.Upstream.UpstreamSelector == "" {
+		conf.Upstream.UpstreamSelector = Random
 	}
 
 	return conf, nil

doh-client/doh-client.conf

@@ -7,40 +7,52 @@ listen = [
 ]
 
 # HTTP path for upstream resolver
-# If multiple servers are specified, a random one will be chosen each time.
-upstream_google = [
 
-    # Google's productive resolver, good ECS, bad DNSSEC
-    "https://dns.google.com/resolve",
+[upstream]
 
-    # CloudFlare's resolver, bad ECS, good DNSSEC
-    #"https://cloudflare-dns.com/dns-query",
-    #"https://1.1.1.1/dns-query",
-    #"https://1.0.0.1/dns-query",
+# available selector: random or weighted_round_robin or lvs_weighted_round_robin
+upstream_selector = "random"
 
-    # CloudFlare's resolver for Tor, available only with Tor
-    # Remember to disable ECS below when using Tor!
-    # Blog: https://blog.cloudflare.com/welcome-hidden-resolver/
-    #"https://dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion/dns-query",
+# weight should in (0, 100], if upstream_selector is random, weight will be ignored
 
-]
-upstream_ietf = [
+## Google's productive resolver, good ECS, bad DNSSEC
+#[[upstream.upstream_google]]
+#    url = "https://dns.google.com/resolve"
+#    weight = 50
 
-    # Google's experimental resolver, good ECS, good DNSSEC
-    #"https://dns.google.com/experimental",
+## CloudFlare's resolver, bad ECS, good DNSSEC
+## ECS is disabled for privacy by design: https://developers.cloudflare.com/1.1.1.1/nitty-gritty-details/#edns-client-subnet
+[[upstream.upstream_ietf]]
+    url = "https://cloudflare-dns.com/dns-query"
+    weight = 50
 
-    # CloudFlare's resolver, bad ECS, good DNSSEC
-    #"https://cloudflare-dns.com/dns-query",
-    #"https://1.1.1.1/dns-query",
-    #"https://1.0.0.1/dns-query",
+## CloudFlare's resolver, bad ECS, good DNSSEC
+## ECS is disabled for privacy by design: https://developers.cloudflare.com/1.1.1.1/nitty-gritty-details/#edns-client-subnet
+## Note that some ISPs have problems connecting to 1.1.1.1, try 1.0.0.1 if problems happen.
+#[[upstream.upstream_ietf]]
+#    url = "https://1.1.1.1/dns-query"
+#    weight = 50
 
-    # CloudFlare's resolver for Tor, available only with Tor
-    # Remember to disable ECS below when using Tor!
-    # Blog: https://blog.cloudflare.com/welcome-hidden-resolver/
-    #"https://dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion/dns-query",
+## Quad9's resolver, bad ECS, good DNSSEC
+## ECS is disabled for privacy by design: https://www.quad9.net/faq/#What_is_EDNS_Client-Subnet
+#[[upstream.upstream_ietf]]
+#    url = "https://9.9.9.9/dns-query"
+#    weight = 50
 
-]
+## Google's experimental resolver, good ECS, good DNSSEC
+#[[upstream.upstream_ietf]]
+#    url = "https://dns.google.com/experimental"
+#    weight = 50
 
+## CloudFlare's resolver for Tor, available only with Tor
+## Remember to disable ECS below when using Tor!
+## Blog: https://blog.cloudflare.com/welcome-hidden-resolver/
+#[[upstream.upstream_ietf]]
+#    url = "https://dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion/dns-query"
+#    weight = 50
+
+
+[others]
 # Bootstrap DNS server to resolve the address of the upstream resolver
 # If multiple servers are specified, a random one will be chosen each time.
 # If empty, use the system DNS settings.
@@ -76,7 +88,7 @@ passthrough = [
     "time.windows.com",
 ]
 
-# Timeout for upstream request
+# Timeout for upstream request in seconds
 timeout = 30
 
 # Disable HTTP Cookies

doh-client/google.go

@@ -29,17 +29,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 
+	"github.com/m13253/dns-over-https/doh-client/selector"
 	"github.com/m13253/dns-over-https/json-dns"
 	"github.com/miekg/dns"
 )
 
-func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool) *DNSRequest {
+func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool, upstream *selector.Upstream) *DNSRequest {
 	question := &r.Question[0]
 	questionName := question.Name
 	questionClass := question.Qclass
@@ -58,9 +58,7 @@ func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter
 		questionType = strconv.FormatUint(uint64(question.Qtype), 10)
 	}
 
-	numServers := len(c.conf.UpstreamGoogle)
-	upstream := c.conf.UpstreamGoogle[rand.Intn(numServers)]
-	requestURL := fmt.Sprintf("%s?ct=application/dns-json&name=%s&type=%s", upstream, url.QueryEscape(questionName), url.QueryEscape(questionType))
+	requestURL := fmt.Sprintf("%s?ct=application/dns-json&name=%s&type=%s", upstream.URL, url.QueryEscape(questionName), url.QueryEscape(questionType))
 
 	if r.CheckingDisabled {
 		requestURL += "&cd=1"
@@ -76,7 +74,7 @@ func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter
 		requestURL += fmt.Sprintf("&edns_client_subnet=%s/%d", ednsClientAddress.String(), ednsClientNetmask)
 	}
 
-	req, err := http.NewRequest("GET", requestURL, nil)
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
 	if err != nil {
 		log.Println(err)
 		reply := jsonDNS.PrepareReply(r)
@@ -86,19 +84,24 @@ func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter
 			err: err,
 		}
 	}
+
 	req.Header.Set("Accept", "application/json, application/dns-message, application/dns-udpwireformat")
 	req.Header.Set("User-Agent", USER_AGENT)
 	req = req.WithContext(ctx)
+
 	c.httpClientMux.RLock()
 	resp, err := c.httpClient.Do(req)
 	c.httpClientMux.RUnlock()
-	if err == context.DeadlineExceeded {
+
+	// if http Client.Do returns non-nil error, it always *url.Error
+	/*if err == context.DeadlineExceeded {
 		// Do not respond, silently fail to prevent caching of SERVFAIL
 		log.Println(err)
 		return &DNSRequest{
 			err: err,
 		}
-	}
+	}*/
+
 	if err != nil {
 		log.Println(err)
 		reply := jsonDNS.PrepareReply(r)
@@ -115,12 +118,12 @@ func (c *Client) generateRequestGoogle(ctx context.Context, w dns.ResponseWriter
 		udpSize:           udpSize,
 		ednsClientAddress: ednsClientAddress,
 		ednsClientNetmask: ednsClientNetmask,
-		currentUpstream:   upstream,
+		currentUpstream:   upstream.URL,
 	}
 }
 
 func (c *Client) parseResponseGoogle(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool, req *DNSRequest) {
-	if req.response.StatusCode != 200 {
+	if req.response.StatusCode != http.StatusOK {
 		log.Printf("HTTP error from upstream %s: %s\n", req.currentUpstream, req.response.Status)
 		req.reply.Rcode = dns.RcodeServerFailure
 		contentType := req.response.Header.Get("Content-Type")

doh-client/ietf.go

@@ -30,17 +30,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"strings"
 	"time"
 
+	"github.com/m13253/dns-over-https/doh-client/selector"
 	"github.com/m13253/dns-over-https/json-dns"
 	"github.com/miekg/dns"
 )
 
-func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool) *DNSRequest {
+func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool, upstream *selector.Upstream) *DNSRequest {
 	opt := r.IsEdns0()
 	udpSize := uint16(512)
 	if opt == nil {
@@ -100,13 +100,11 @@ func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter,
 	r.Id = requestID
 	requestBase64 := base64.RawURLEncoding.EncodeToString(requestBinary)
 
-	numServers := len(c.conf.UpstreamIETF)
-	upstream := c.conf.UpstreamIETF[rand.Intn(numServers)]
-	requestURL := fmt.Sprintf("%s?ct=application/dns-message&dns=%s", upstream, requestBase64)
+	requestURL := fmt.Sprintf("%s?ct=application/dns-message&dns=%s", upstream.URL, requestBase64)
 
 	var req *http.Request
 	if len(requestURL) < 2048 {
-		req, err = http.NewRequest("GET", requestURL, nil)
+		req, err = http.NewRequest(http.MethodGet, requestURL, nil)
 		if err != nil {
 			log.Println(err)
 			reply := jsonDNS.PrepareReply(r)
@@ -117,7 +115,7 @@ func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter,
 			}
 		}
 	} else {
-		req, err = http.NewRequest("POST", upstream, bytes.NewReader(requestBinary))
+		req, err = http.NewRequest(http.MethodPost, upstream.URL, bytes.NewReader(requestBinary))
 		if err != nil {
 			log.Println(err)
 			reply := jsonDNS.PrepareReply(r)
@@ -135,13 +133,16 @@ func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter,
 	c.httpClientMux.RLock()
 	resp, err := c.httpClient.Do(req)
 	c.httpClientMux.RUnlock()
-	if err == context.DeadlineExceeded {
+
+	// if http Client.Do returns non-nil error, it always *url.Error
+	/*if err == context.DeadlineExceeded {
 		// Do not respond, silently fail to prevent caching of SERVFAIL
 		log.Println(err)
 		return &DNSRequest{
 			err: err,
 		}
-	}
+	}*/
+
 	if err != nil {
 		log.Println(err)
 		reply := jsonDNS.PrepareReply(r)
@@ -158,12 +159,12 @@ func (c *Client) generateRequestIETF(ctx context.Context, w dns.ResponseWriter,
 		udpSize:           udpSize,
 		ednsClientAddress: ednsClientAddress,
 		ednsClientNetmask: ednsClientNetmask,
-		currentUpstream:   upstream,
+		currentUpstream:   upstream.URL,
 	}
 }
 
 func (c *Client) parseResponseIETF(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, isTCP bool, req *DNSRequest) {
-	if req.response.StatusCode != 200 {
+	if req.response.StatusCode != http.StatusOK {
 		log.Printf("HTTP error from upstream %s: %s\n", req.currentUpstream, req.response.Status)
 		req.reply.Rcode = dns.RcodeServerFailure
 		contentType := req.response.Header.Get("Content-Type")
@@ -205,7 +206,7 @@ func (c *Client) parseResponseIETF(ctx context.Context, w dns.ResponseWriter, r
 
 	fullReply := new(dns.Msg)
 	err = fullReply.Unpack(body)
-	if err != nil && err != dns.ErrTruncated {
+	if err != nil {
 		log.Println(err)
 		req.reply.Rcode = dns.RcodeServerFailure
 		w.WriteMsg(req.reply)

doh-client/main.go

@@ -25,21 +25,91 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"os"
+	"runtime"
+	"strconv"
+
+	"github.com/m13253/dns-over-https/doh-client/config"
 )
 
+func checkPIDFile(pidFile string) (bool, error) {
+retry:
+	f, err := os.OpenFile(pidFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if os.IsExist(err) {
+		pidStr, err := ioutil.ReadFile(pidFile)
+		if err != nil {
+			return false, err
+		}
+		pid, err := strconv.ParseUint(string(pidStr), 10, 0)
+		if err != nil {
+			return false, err
+		}
+		_, err = os.Stat(fmt.Sprintf("/proc/%d", pid))
+		if os.IsNotExist(err) {
+			err = os.Remove(pidFile)
+			if err != nil {
+				return false, err
+			}
+			goto retry
+		} else if err != nil {
+			return false, err
+		}
+		log.Printf("Already running on PID %d, exiting.\n", pid)
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	_, err = io.WriteString(f, strconv.FormatInt(int64(os.Getpid()), 10))
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 func main() {
 	confPath := flag.String("conf", "doh-client.conf", "Configuration file")
 	verbose := flag.Bool("verbose", false, "Enable logging")
+	showVersion := flag.Bool("version", false, "Show software version and exit")
+	var pidFile *string
+
+	// I really want to push the technology forward by recommending cgroup-based
+	// process tracking. But I understand some cloud service providers have
+	// their own monitoring system. So this feature is only enabled on Linux and
+	// BSD series platforms which lacks functionality similar to cgroup.
+	switch runtime.GOOS {
+	case "dragonfly", "freebsd", "linux", "netbsd", "openbsd":
+		pidFile = flag.String("pid-file", "", "PID file for legacy supervision systems lacking support for reliable cgroup-based process tracking")
+	}
+
 	flag.Parse()
 
-	conf, err := loadConfig(*confPath)
+	if *showVersion {
+		fmt.Printf("doh-server %s\nHomepage: https://github.com/m13253/dns-over-https\n", VERSION)
+		return
+	}
+
+	if pidFile != nil && *pidFile != "" {
+		ok, err := checkPIDFile(*pidFile)
+		if err != nil {
+			log.Printf("Error checking PID file: %v\n", err)
+		}
+		if !ok {
+			return
+		}
+	}
+
+	conf, err := config.LoadConfig(*confPath)
 	if err != nil {
 		log.Fatalln(err)
 	}
 
 	if *verbose {
-		conf.Verbose = true
+		conf.Other.Verbose = true
 	}
 
 	client, err := NewClient(conf)

doh-client/selector/lvsWRRSelector.go

@@ -0,0 +1,262 @@
+package selector
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type LVSWRRSelector struct {
+	upstreams     []*Upstream // upstreamsInfo
+	client        http.Client // http client to check the upstream
+	lastChoose    int32
+	currentWeight int32
+}
+
+func NewLVSWRRSelector(timeout time.Duration) *LVSWRRSelector {
+	return &LVSWRRSelector{
+		client:     http.Client{Timeout: timeout},
+		lastChoose: -1,
+	}
+}
+
+func (ls *LVSWRRSelector) Add(url string, upstreamType UpstreamType, weight int32) (err error) {
+	if weight < 1 {
+		return errors.New("weight is 1")
+	}
+
+	switch upstreamType {
+	case Google:
+		ls.upstreams = append(ls.upstreams, &Upstream{
+			Type:            Google,
+			URL:             url,
+			RequestType:     "application/dns-json",
+			weight:          weight,
+			effectiveWeight: weight,
+		})
+
+	case IETF:
+		ls.upstreams = append(ls.upstreams, &Upstream{
+			Type:            IETF,
+			URL:             url,
+			RequestType:     "application/dns-message",
+			weight:          weight,
+			effectiveWeight: weight,
+		})
+
+	default:
+		return errors.New("unknown upstream type")
+	}
+
+	return nil
+}
+
+func (ls *LVSWRRSelector) StartEvaluate() {
+	go func() {
+		for {
+			wg := sync.WaitGroup{}
+
+			for i := range ls.upstreams {
+				wg.Add(1)
+
+				go func(i int) {
+					defer wg.Done()
+
+					upstreamURL := ls.upstreams[i].URL
+					var acceptType string
+
+					switch ls.upstreams[i].Type {
+					case Google:
+						upstreamURL += "?name=www.example.com&type=A"
+						acceptType = "application/dns-json"
+
+					case IETF:
+						// www.example.com
+						upstreamURL += "?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"
+						acceptType = "application/dns-message"
+					}
+
+					req, err := http.NewRequest(http.MethodGet, upstreamURL, nil)
+					if err != nil {
+						/*log.Println("upstream:", upstreamURL, "type:", typeMap[upstream.Type], "check failed:", err)
+						continue*/
+
+						// should I only log it? But if there is an error, I think when query the server will return error too
+						panic("upstream: " + upstreamURL + " type: " + typeMap[ls.upstreams[i].Type] + " check failed: " + err.Error())
+					}
+
+					req.Header.Set("accept", acceptType)
+
+					resp, err := ls.client.Do(req)
+					if err != nil {
+						// should I check error in detail?
+						if atomic.AddInt32(&ls.upstreams[i].effectiveWeight, -5) < 1 {
+							atomic.StoreInt32(&ls.upstreams[i].effectiveWeight, 1)
+						}
+						return
+					}
+
+					switch ls.upstreams[i].Type {
+					case Google:
+						ls.checkGoogleResponse(resp, ls.upstreams[i])
+
+					case IETF:
+						ls.checkIETFResponse(resp, ls.upstreams[i])
+					}
+				}(i)
+			}
+
+			wg.Wait()
+
+			time.Sleep(15 * time.Second)
+		}
+	}()
+}
+
+func (ls *LVSWRRSelector) Get() *Upstream {
+	if len(ls.upstreams) == 1 {
+		return ls.upstreams[0]
+	}
+
+	for {
+		atomic.StoreInt32(&ls.lastChoose, (atomic.LoadInt32(&ls.lastChoose)+1)%int32(len(ls.upstreams)))
+
+		if atomic.LoadInt32(&ls.lastChoose) == 0 {
+			atomic.AddInt32(&ls.currentWeight, -ls.gcdWeight())
+
+			if atomic.LoadInt32(&ls.currentWeight) <= 0 {
+				atomic.AddInt32(&ls.currentWeight, ls.maxWeight())
+
+				if atomic.LoadInt32(&ls.currentWeight) == 0 {
+					panic("current weight is 0")
+				}
+			}
+		}
+
+		if atomic.LoadInt32(&ls.upstreams[atomic.LoadInt32(&ls.lastChoose)].effectiveWeight) >= atomic.LoadInt32(&ls.currentWeight) {
+			return ls.upstreams[atomic.LoadInt32(&ls.lastChoose)]
+		}
+	}
+}
+
+func (ls *LVSWRRSelector) gcdWeight() (res int32) {
+	res = gcd(atomic.LoadInt32(&ls.upstreams[0].effectiveWeight), atomic.LoadInt32(&ls.upstreams[0].effectiveWeight))
+
+	for i := 1; i < len(ls.upstreams); i++ {
+		res = gcd(res, atomic.LoadInt32(&ls.upstreams[i].effectiveWeight))
+	}
+
+	return
+}
+
+func (ls *LVSWRRSelector) maxWeight() (res int32) {
+	for _, upstream := range ls.upstreams {
+		w := atomic.LoadInt32(&upstream.effectiveWeight)
+		if w > res {
+			res = w
+		}
+	}
+
+	return
+}
+
+func gcd(x, y int32) int32 {
+	for {
+		if x < y {
+			x, y = y, x
+		}
+
+		tmp := x % y
+		if tmp == 0 {
+			return y
+		}
+
+		x = tmp
+	}
+}
+
+func (ls *LVSWRRSelector) ReportUpstreamStatus(upstream *Upstream, upstreamStatus upstreamStatus) {
+	switch upstreamStatus {
+	case Timeout:
+		if atomic.AddInt32(&upstream.effectiveWeight, -5) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+
+	case Error:
+		if atomic.AddInt32(&upstream.effectiveWeight, -2) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+
+	case OK:
+		if atomic.AddInt32(&upstream.effectiveWeight, 1) > upstream.weight {
+			atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+		}
+	}
+}
+
+func (ls *LVSWRRSelector) checkGoogleResponse(resp *http.Response, upstream *Upstream) {
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// server error
+		if atomic.AddInt32(&upstream.effectiveWeight, -3) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	m := make(map[string]interface{})
+	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
+		// should I check error in detail?
+		if atomic.AddInt32(&upstream.effectiveWeight, -2) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	if status, ok := m["Status"]; ok {
+		if statusNum, ok := status.(float64); ok && statusNum == 0 {
+			if atomic.AddInt32(&upstream.effectiveWeight, 5) > upstream.weight {
+				atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+			}
+			return
+		}
+	}
+
+	// should I check error in detail?
+	if atomic.AddInt32(&upstream.effectiveWeight, -2) < 1 {
+		atomic.StoreInt32(&upstream.effectiveWeight, 1)
+	}
+}
+
+func (ls *LVSWRRSelector) checkIETFResponse(resp *http.Response, upstream *Upstream) {
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// server error
+		if atomic.AddInt32(&upstream.effectiveWeight, -3) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	if atomic.AddInt32(&upstream.effectiveWeight, 5) > upstream.weight {
+		atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+	}
+}
+
+func (ls *LVSWRRSelector) ReportWeights() {
+	go func() {
+		for {
+			time.Sleep(15 * time.Second)
+
+			for _, u := range ls.upstreams {
+				log.Printf("%s, effect weight: %d", u, atomic.LoadInt32(&u.effectiveWeight))
+			}
+		}
+	}()
+}

doh-client/selector/nginxWRRSelector.go

@@ -0,0 +1,215 @@
+package selector
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type NginxWRRSelector struct {
+	upstreams []*Upstream // upstreamsInfo
+	client    http.Client // http client to check the upstream
+}
+
+func NewNginxWRRSelector(timeout time.Duration) *NginxWRRSelector {
+	return &NginxWRRSelector{
+		client: http.Client{Timeout: timeout},
+	}
+}
+
+func (ws *NginxWRRSelector) Add(url string, upstreamType UpstreamType, weight int32) (err error) {
+	switch upstreamType {
+	case Google:
+		ws.upstreams = append(ws.upstreams, &Upstream{
+			Type:            Google,
+			URL:             url,
+			RequestType:     "application/dns-json",
+			weight:          weight,
+			effectiveWeight: weight,
+		})
+
+	case IETF:
+		ws.upstreams = append(ws.upstreams, &Upstream{
+			Type:            IETF,
+			URL:             url,
+			RequestType:     "application/dns-message",
+			weight:          weight,
+			effectiveWeight: weight,
+		})
+
+	default:
+		return errors.New("unknown upstream type")
+	}
+
+	return nil
+}
+
+func (ws *NginxWRRSelector) StartEvaluate() {
+	go func() {
+		for {
+			wg := sync.WaitGroup{}
+
+			for i := range ws.upstreams {
+				wg.Add(1)
+
+				go func(i int) {
+					defer wg.Done()
+
+					upstreamURL := ws.upstreams[i].URL
+					var acceptType string
+
+					switch ws.upstreams[i].Type {
+					case Google:
+						upstreamURL += "?name=www.example.com&type=A"
+						acceptType = "application/dns-json"
+
+					case IETF:
+						// www.example.com
+						upstreamURL += "?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"
+						acceptType = "application/dns-message"
+					}
+
+					req, err := http.NewRequest(http.MethodGet, upstreamURL, nil)
+					if err != nil {
+						/*log.Println("upstream:", upstreamURL, "type:", typeMap[upstream.Type], "check failed:", err)
+						continue*/
+
+						// should I only log it? But if there is an error, I think when query the server will return error too
+						panic("upstream: " + upstreamURL + " type: " + typeMap[ws.upstreams[i].Type] + " check failed: " + err.Error())
+					}
+
+					req.Header.Set("accept", acceptType)
+
+					resp, err := ws.client.Do(req)
+					if err != nil {
+						// should I check error in detail?
+						if atomic.AddInt32(&ws.upstreams[i].effectiveWeight, -10) < 1 {
+							atomic.StoreInt32(&ws.upstreams[i].effectiveWeight, 1)
+						}
+						return
+					}
+
+					switch ws.upstreams[i].Type {
+					case Google:
+						ws.checkGoogleResponse(resp, ws.upstreams[i])
+
+					case IETF:
+						ws.checkIETFResponse(resp, ws.upstreams[i])
+					}
+				}(i)
+			}
+
+			wg.Wait()
+
+			time.Sleep(15 * time.Second)
+		}
+	}()
+}
+
+// nginx wrr like
+func (ws *NginxWRRSelector) Get() *Upstream {
+	var (
+		total             int32
+		bestUpstreamIndex = -1
+	)
+
+	for i := range ws.upstreams {
+		effectiveWeight := atomic.LoadInt32(&ws.upstreams[i].effectiveWeight)
+		atomic.AddInt32(&ws.upstreams[i].currentWeight, effectiveWeight)
+		total += effectiveWeight
+
+		if bestUpstreamIndex == -1 || atomic.LoadInt32(&ws.upstreams[i].currentWeight) > atomic.LoadInt32(&ws.upstreams[bestUpstreamIndex].currentWeight) {
+			bestUpstreamIndex = i
+		}
+	}
+
+	atomic.AddInt32(&ws.upstreams[bestUpstreamIndex].currentWeight, -total)
+
+	return ws.upstreams[bestUpstreamIndex]
+}
+
+func (ws *NginxWRRSelector) ReportUpstreamStatus(upstream *Upstream, upstreamStatus upstreamStatus) {
+	switch upstreamStatus {
+	case Timeout:
+		if atomic.AddInt32(&upstream.effectiveWeight, -5) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+
+	case Error:
+		if atomic.AddInt32(&upstream.effectiveWeight, -3) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+
+	case OK:
+		if atomic.AddInt32(&upstream.effectiveWeight, 1) > upstream.weight {
+			atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+		}
+	}
+}
+
+func (ws *NginxWRRSelector) checkGoogleResponse(resp *http.Response, upstream *Upstream) {
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// server error
+		if atomic.AddInt32(&upstream.effectiveWeight, -3) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	m := make(map[string]interface{})
+	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
+		// should I check error in detail?
+		if atomic.AddInt32(&upstream.effectiveWeight, -2) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	if status, ok := m["Status"]; ok {
+		if statusNum, ok := status.(float64); ok && statusNum == 0 {
+			if atomic.AddInt32(&upstream.effectiveWeight, 5) > upstream.weight {
+				atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+			}
+			return
+		}
+	}
+
+	// should I check error in detail?
+	if atomic.AddInt32(&upstream.effectiveWeight, -2) < 1 {
+		atomic.StoreInt32(&upstream.effectiveWeight, 1)
+	}
+}
+
+func (ws *NginxWRRSelector) checkIETFResponse(resp *http.Response, upstream *Upstream) {
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// server error
+		if atomic.AddInt32(&upstream.effectiveWeight, -5) < 1 {
+			atomic.StoreInt32(&upstream.effectiveWeight, 1)
+		}
+		return
+	}
+
+	if atomic.AddInt32(&upstream.effectiveWeight, 5) > upstream.weight {
+		atomic.StoreInt32(&upstream.effectiveWeight, upstream.weight)
+	}
+}
+
+func (ws *NginxWRRSelector) ReportWeights() {
+	go func() {
+		for {
+			time.Sleep(15 * time.Second)
+
+			for _, u := range ws.upstreams {
+				log.Printf("%s, effect weight: %d", u, atomic.LoadInt32(&u.effectiveWeight))
+			}
+		}
+	}()
+}

doh-client/selector/randomSelector.go

@@ -0,0 +1,50 @@
+package selector
+
+import (
+	"errors"
+	"math/rand"
+	"time"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+type RandomSelector struct {
+	upstreams []*Upstream
+}
+
+func NewRandomSelector() *RandomSelector {
+	return new(RandomSelector)
+}
+
+func (rs *RandomSelector) Add(url string, upstreamType UpstreamType) (err error) {
+	switch upstreamType {
+	case Google:
+		rs.upstreams = append(rs.upstreams, &Upstream{
+			Type:        Google,
+			URL:         url,
+			RequestType: "application/dns-json",
+		})
+
+	case IETF:
+		rs.upstreams = append(rs.upstreams, &Upstream{
+			Type:        IETF,
+			URL:         url,
+			RequestType: "application/dns-message",
+		})
+
+	default:
+		return errors.New("unknown upstream type")
+	}
+
+	return nil
+}
+
+func (rs *RandomSelector) Get() *Upstream {
+	return rs.upstreams[rand.Intn(len(rs.upstreams))]
+}
+
+func (rs *RandomSelector) StartEvaluate() {}
+
+func (rs *RandomSelector) ReportUpstreamStatus(upstream *Upstream, upstreamStatus upstreamStatus) {}

doh-client/selector/selector.go

@@ -0,0 +1,17 @@
+package selector
+
+type Selector interface {
+	// Get returns a upstream
+	Get() *Upstream
+
+	// StartEvaluate start upstream evaluation loop
+	StartEvaluate()
+
+	// ReportUpstreamStatus report upstream status
+	ReportUpstreamStatus(upstream *Upstream, upstreamStatus upstreamStatus)
+}
+
+type DebugReporter interface {
+	// ReportWeights starts a goroutine to report all upstream weights, recommend interval is 15s
+	ReportWeights()
+}

doh-client/selector/upstream.go

@@ -0,0 +1,28 @@
+package selector
+
+import "fmt"
+
+type UpstreamType int
+
+const (
+	Google UpstreamType = iota
+	IETF
+)
+
+var typeMap = map[UpstreamType]string{
+	Google: "Google",
+	IETF:   "IETF",
+}
+
+type Upstream struct {
+	Type            UpstreamType
+	URL             string
+	RequestType     string
+	weight          int32
+	effectiveWeight int32
+	currentWeight   int32
+}
+
+func (u Upstream) String() string {
+	return fmt.Sprintf("upstream type: %s, upstream url: %s", typeMap[u.Type], u.URL)
+}

doh-client/selector/upstreamStatus.go

@@ -0,0 +1,14 @@
+package selector
+
+type upstreamStatus int
+
+const (
+	// when query upstream timeout, usually upstream is unavailable for a long time
+	Timeout upstreamStatus = iota
+
+	// when query upstream return 5xx response, upstream still alive, maybe just a lof of query for him
+	Error
+
+	// when query upstream ok, means upstream is available
+	OK
+)

doh-client/version.go

@@ -24,6 +24,6 @@
 package main
 
 const (
-	VERSION    = "1.4.1"
+	VERSION    = "2.1.0"
 	USER_AGENT = "DNS-over-HTTPS/" + VERSION + " (+https://github.com/m13253/dns-over-https)"
 )

doh-server/config.go

@@ -31,6 +31,7 @@ import (
 
 type config struct {
 	Listen           []string `toml:"listen"`
+	LocalAddr        string   `toml:"local_addr"`
 	Cert             string   `toml:"cert"`
 	Key              string   `toml:"key"`
 	Path             string   `toml:"path"`
@@ -40,6 +41,7 @@ type config struct {
 	TCPOnly          bool     `toml:"tcp_only"`
 	Verbose          bool     `toml:"verbose"`
 	DebugHTTPHeaders []string `toml:"debug_http_headers"`
+	LogGuessedIP     bool     `toml:"log_guessed_client_ip"`
 }
 
 func loadConfig(path string) (*config, error) {

doh-server/doh-server.conf

@@ -4,6 +4,10 @@ listen = [
     "[::1]:8053",
 ]
 
+# Local address and port for upstream DNS
+# If left empty, a local address is automatically chosen.
+local_addr = ""
+
 # TLS certification file
 # If left empty, plain-text HTTP will be used.
 # You are recommended to leave empty and to use a server load balancer (e.g.
@@ -38,3 +42,7 @@ tcp_only = false
 
 # Enable logging
 verbose = false
+
+# Enable log IP from HTTPS-reverse proxy header: X-Forwarded-For or X-Real-IP
+# Note: http uri/useragent log cannot be controlled by this config
+log_guessed_client_ip = false

doh-server/ietf.go

@@ -30,6 +30,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -94,7 +95,15 @@ func (s *Server) parseRequestIETF(ctx context.Context, w http.ResponseWriter, r
 		} else {
 			questionType = strconv.FormatUint(uint64(question.Qtype), 10)
 		}
-		fmt.Printf("%s - - [%s] \"%s %s %s\"\n", r.RemoteAddr, time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
+		var clientip net.IP = nil
+		if s.conf.LogGuessedIP {
+			clientip = s.findClientIP(r)
+		}
+		if clientip != nil {
+			fmt.Printf("%s - - [%s] \"%s %s %s\"\n", clientip, time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
+		} else {
+			fmt.Printf("%s - - [%s] \"%s %s %s\"\n", r.RemoteAddr, time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
+		}
 	}
 
 	transactionID := msg.Id

doh-server/main.go

@@ -25,14 +25,82 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"os"
+	"runtime"
+	"strconv"
 )
 
+func checkPIDFile(pidFile string) (bool, error) {
+retry:
+	f, err := os.OpenFile(pidFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if os.IsExist(err) {
+		pidStr, err := ioutil.ReadFile(pidFile)
+		if err != nil {
+			return false, err
+		}
+		pid, err := strconv.ParseUint(string(pidStr), 10, 0)
+		if err != nil {
+			return false, err
+		}
+		_, err = os.Stat(fmt.Sprintf("/proc/%d", pid))
+		if os.IsNotExist(err) {
+			err = os.Remove(pidFile)
+			if err != nil {
+				return false, err
+			}
+			goto retry
+		} else if err != nil {
+			return false, err
+		}
+		log.Printf("Already running on PID %d, exiting.\n", pid)
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	_, err = io.WriteString(f, strconv.FormatInt(int64(os.Getpid()), 10))
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 func main() {
 	confPath := flag.String("conf", "doh-server.conf", "Configuration file")
 	verbose := flag.Bool("verbose", false, "Enable logging")
+	showVersion := flag.Bool("version", false, "Show software version and exit")
+	var pidFile *string
+
+	// I really want to push the technology forward by recommending cgroup-based
+	// process tracking. But I understand some cloud service providers have
+	// their own monitoring system. So this feature is only enabled on Linux and
+	// BSD series platforms which lacks functionality similar to cgroup.
+	switch runtime.GOOS {
+	case "dragonfly", "freebsd", "linux", "netbsd", "openbsd":
+		pidFile = flag.String("pid-file", "", "PID file for legacy supervision systems lacking support for reliable cgroup-based process tracking")
+	}
+
 	flag.Parse()
 
+	if *showVersion {
+		fmt.Printf("doh-server %s\nHomepage: https://github.com/m13253/dns-over-https\n", VERSION)
+		return
+	}
+
+	if pidFile != nil && *pidFile != "" {
+		ok, err := checkPIDFile(*pidFile)
+		if err != nil {
+			log.Printf("Error checking PID file: %v\n", err)
+		}
+		if !ok {
+			return
+		}
+	}
+
 	conf, err := loadConfig(*confPath)
 	if err != nil {
 		log.Fatalln(err)
@@ -42,6 +110,9 @@ func main() {
 		conf.Verbose = true
 	}
 
-	server := NewServer(conf)
+	server, err := NewServer(conf)
+	if err != nil {
+		log.Fatalln(err)
+	}
 	_ = server.Start()
 }

doh-server/server.go

@@ -56,22 +56,41 @@ type DNSRequest struct {
 	errtext         string
 }
 
-func NewServer(conf *config) (s *Server) {
-	s = &Server{
+func NewServer(conf *config) (*Server, error) {
+	timeout := time.Duration(conf.Timeout) * time.Second
+	s := &Server{
 		conf: conf,
 		udpClient: &dns.Client{
 			Net:     "udp",
 			UDPSize: dns.DefaultMsgSize,
-			Timeout: time.Duration(conf.Timeout) * time.Second,
+			Timeout: timeout,
 		},
 		tcpClient: &dns.Client{
 			Net:     "tcp",
-			Timeout: time.Duration(conf.Timeout) * time.Second,
+			Timeout: timeout,
 		},
 		servemux: http.NewServeMux(),
 	}
+	if conf.LocalAddr != "" {
+		udpLocalAddr, err := net.ResolveUDPAddr("udp", conf.LocalAddr)
+		if err != nil {
+			return nil, err
+		}
+		tcpLocalAddr, err := net.ResolveTCPAddr("tcp", conf.LocalAddr)
+		if err != nil {
+			return nil, err
+		}
+		s.udpClient.Dialer = &net.Dialer{
+			Timeout:   timeout,
+			LocalAddr: udpLocalAddr,
+		}
+		s.tcpClient.Dialer = &net.Dialer{
+			Timeout:   timeout,
+			LocalAddr: tcpLocalAddr,
+		}
+	}
 	s.servemux.HandleFunc(conf.Path, s.handlerFunc)
-	return
+	return s, nil
 }
 
 func (s *Server) Start() error {
@@ -244,21 +263,40 @@ func (s *Server) patchRootRD(req *DNSRequest) *DNSRequest {
 	return req
 }
 
+// Return the position index for the question of qtype from a DNS msg, otherwise return -1
+func (s *Server) indexQuestionType(msg *dns.Msg, qtype uint16) int {
+	for i, question := range msg.Question {
+		if question.Qtype == qtype {
+			return i
+		}
+	}
+	return -1
+}
+
 func (s *Server) doDNSQuery(ctx context.Context, req *DNSRequest) (resp *DNSRequest, err error) {
 	// TODO(m13253): Make ctx work. Waiting for a patch for ExchangeContext from miekg/dns.
 	numServers := len(s.conf.Upstream)
 	for i := uint(0); i < s.conf.Tries; i++ {
 		req.currentUpstream = s.conf.Upstream[rand.Intn(numServers)]
-		if !s.conf.TCPOnly {
+
+		// Use TCP if always configured to or if the Query type dictates it (AXFR)
+		if s.conf.TCPOnly || (s.indexQuestionType(req.request, dns.TypeAXFR) > -1) {
+			req.response, _, err = s.tcpClient.Exchange(req.request, req.currentUpstream)
+		} else {
 			req.response, _, err = s.udpClient.Exchange(req.request, req.currentUpstream)
-			if err == dns.ErrTruncated {
+			if err == nil && req.response != nil && req.response.Truncated {
 				log.Println(err)
 				req.response, _, err = s.tcpClient.Exchange(req.request, req.currentUpstream)
 			}
-		} else {
-			req.response, _, err = s.tcpClient.Exchange(req.request, req.currentUpstream)
+
+			// Retry with TCP if this was an IXFR request and we only received an SOA
+			if (s.indexQuestionType(req.request, dns.TypeIXFR) > -1) &&
+				(len(req.response.Answer) == 1) &&
+				(req.response.Answer[0].Header().Rrtype == dns.TypeSOA) {
+				req.response, _, err = s.tcpClient.Exchange(req.request, req.currentUpstream)
+			}
 		}
-		if err == nil || err == dns.ErrTruncated {
+		if err == nil {
 			return req, nil
 		}
 		log.Printf("DNS error from upstream %s: %s\n", req.currentUpstream, err.Error())

doh-server/version.go

@@ -24,6 +24,6 @@
 package main
 
 const (
-	VERSION    = "1.4.1"
+	VERSION    = "2.1.0"
 	USER_AGENT = "DNS-over-HTTPS/" + VERSION + " (+https://github.com/m13253/dns-over-https)"
 )

go.mod

@@ -0,0 +1,13 @@
+module github.com/m13253/dns-over-https
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/gorilla/handlers v1.4.0
+	github.com/miekg/dns v1.1.6
+	golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a // indirect
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect
+	golang.org/x/sys v0.0.0-20190312061237-fead79001313 // indirect
+)

go.sum

@@ -0,0 +1,25 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA=
+github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0=
+github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.6 h1:jVwb4GDwD65q/gtItR/lIZHjNH93QfeGxZUkzJcW9mc=
+github.com/miekg/dns v1.1.6/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a h1:YX8ljsm6wXlHZO+aRz9Exqr0evNhKRNe5K/gi+zKh4U=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU=
+golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190308023053-584f3b12f43e h1:K7CV15oJ823+HLXQ+M7MSMrUg8LjfqY7O3naO+8Pp/I=
+golang.org/x/sys v0.0.0-20190308023053-584f3b12f43e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=