From 22dfb140d0b3587dc9de4f2f15f93a9f319becf2 Mon Sep 17 00:00:00 2001 From: Jake Lucas Date: Wed, 31 Oct 2018 03:02:59 +0900 Subject: [PATCH] proxy: Add new fallback_delay sub-directive (#2309) * Updates the existing proxy and reverse proxy tests to include a new fallback delay value * Adds a new fallback_delay sub-directive to the proxy directive and uses it in the creation of single host reverse proxies --- caddyhttp/proxy/proxy.go | 7 ++++ caddyhttp/proxy/proxy_test.go | 60 +++++++++++++++------------- caddyhttp/proxy/reverseproxy.go | 5 ++- caddyhttp/proxy/reverseproxy_test.go | 2 +- caddyhttp/proxy/upstream.go | 17 +++++++- 5 files changed, 61 insertions(+), 30 deletions(-) diff --git a/caddyhttp/proxy/proxy.go b/caddyhttp/proxy/proxy.go index c47a1597c..5b386020e 100644 --- a/caddyhttp/proxy/proxy.go +++ b/caddyhttp/proxy/proxy.go @@ -47,6 +47,12 @@ type Upstream interface { // Checks if subpath is not an ignored path AllowedPath(string) bool + // Gets the duration of the headstart the first + // connection is given in the Go standard library's + // implementation of "Happy Eyeballs" when DualStack + // is enabled in net.Dialer. + GetFallbackDelay() time.Duration + // Gets how long to try selecting upstream hosts // in the case of cascading failures. GetTryDuration() time.Duration @@ -195,6 +201,7 @@ func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { host.WithoutPathPrefix, http.DefaultMaxIdleConnsPerHost, upstream.GetTimeout(), + upstream.GetFallbackDelay(), ) } diff --git a/caddyhttp/proxy/proxy_test.go b/caddyhttp/proxy/proxy_test.go index 79c238879..caf2e6c99 100644 --- a/caddyhttp/proxy/proxy_test.go +++ b/caddyhttp/proxy/proxy_test.go @@ -122,7 +122,7 @@ func TestReverseProxy(t *testing.T) { // set up proxy p := &Proxy{ Next: httpserver.EmptyNext, // prevents panic in some cases when test fails - Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)}, + Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)}, } // Create the fake request body. @@ -202,7 +202,7 @@ func TestReverseProxyInsecureSkipVerify(t *testing.T) { // set up proxy p := &Proxy{ Next: httpserver.EmptyNext, // prevents panic in some cases when test fails - Upstreams: []Upstream{newFakeUpstream(backend.URL, true, 30*time.Second)}, + Upstreams: []Upstream{newFakeUpstream(backend.URL, true, 30*time.Second, 300*time.Millisecond)}, } // create request and response recorder @@ -289,6 +289,7 @@ func TestReverseProxyMaxConnLimit(t *testing.T) { func TestReverseProxyTimeout(t *testing.T) { timeout := 2 * time.Second + fallbackDelay := 300 * time.Millisecond errorMargin := 100 * time.Millisecond log.SetOutput(ioutil.Discard) defer log.SetOutput(os.Stderr) @@ -296,7 +297,7 @@ func TestReverseProxyTimeout(t *testing.T) { // set up proxy p := &Proxy{ Next: httpserver.EmptyNext, // prevents panic in some cases when test fails - Upstreams: []Upstream{newFakeUpstream("https://8.8.8.8", true, timeout)}, + Upstreams: []Upstream{newFakeUpstream("https://8.8.8.8", true, timeout, fallbackDelay)}, } // create request and response recorder @@ -711,7 +712,7 @@ func TestUpstreamHeadersUpdate(t *testing.T) { })) defer backend.Close() - upstream := newFakeUpstream(backend.URL, false, 30*time.Second) + upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond) upstream.host.UpstreamHeaders = http.Header{ "Connection": {"{>Connection}"}, "Upgrade": {"{>Upgrade}"}, @@ -778,7 +779,7 @@ func TestDownstreamHeadersUpdate(t *testing.T) { })) defer backend.Close() - upstream := newFakeUpstream(backend.URL, false, 30*time.Second) + upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond) upstream.host.DownstreamHeaders = http.Header{ "+Merge-Me": {"Merge-Value"}, "+Add-Me": {"Add-Value"}, @@ -918,7 +919,7 @@ func TestHostSimpleProxyNoHeaderForward(t *testing.T) { // set up proxy p := &Proxy{ Next: httpserver.EmptyNext, // prevents panic in some cases when test fails - Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)}, + Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)}, } r := httptest.NewRequest("GET", "/", nil) @@ -1007,7 +1008,7 @@ func TestHostHeaderReplacedUsingForward(t *testing.T) { })) defer backend.Close() - upstream := newFakeUpstream(backend.URL, false, 30*time.Second) + upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond) proxyHostHeader := "test2.com" upstream.host.UpstreamHeaders = http.Header{"Host": []string{proxyHostHeader}} // set up proxy @@ -1069,7 +1070,7 @@ func basicAuthTestcase(t *testing.T, upstreamUser, clientUser *url.Userinfo) { p := &Proxy{ Next: httpserver.EmptyNext, - Upstreams: []Upstream{newFakeUpstream(backURL.String(), false, 30*time.Second)}, + Upstreams: []Upstream{newFakeUpstream(backURL.String(), false, 30*time.Second, 300*time.Millisecond)}, } r, err := http.NewRequest("GET", "/foo", nil) if err != nil { @@ -1204,7 +1205,7 @@ func TestProxyDirectorURL(t *testing.T) { continue } - NewSingleHostReverseProxy(targetURL, c.without, 0, 30*time.Second).Director(req) + NewSingleHostReverseProxy(targetURL, c.without, 0, 30*time.Second, 300*time.Millisecond).Director(req) if expect, got := c.expectURL, req.URL.String(); expect != got { t.Errorf("case %d url not equal: expect %q, but got %q", i, expect, got) @@ -1351,7 +1352,7 @@ func TestCancelRequest(t *testing.T) { // set up proxy p := &Proxy{ Next: httpserver.EmptyNext, // prevents panic in some cases when test fails - Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)}, + Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)}, } // setup request with cancel ctx @@ -1400,15 +1401,16 @@ func (r *noopReader) Read(b []byte) (int, error) { return n, nil } -func newFakeUpstream(name string, insecure bool, timeout time.Duration) *fakeUpstream { +func newFakeUpstream(name string, insecure bool, timeout, fallbackDelay time.Duration) *fakeUpstream { uri, _ := url.Parse(name) u := &fakeUpstream{ - name: name, - from: "/", - timeout: timeout, + name: name, + from: "/", + timeout: timeout, + fallbackDelay: fallbackDelay, host: &UpstreamHost{ Name: name, - ReverseProxy: NewSingleHostReverseProxy(uri, "", http.DefaultMaxIdleConnsPerHost, timeout), + ReverseProxy: NewSingleHostReverseProxy(uri, "", http.DefaultMaxIdleConnsPerHost, timeout, fallbackDelay), }, } if insecure { @@ -1418,11 +1420,12 @@ func newFakeUpstream(name string, insecure bool, timeout time.Duration) *fakeUps } type fakeUpstream struct { - name string - host *UpstreamHost - from string - without string - timeout time.Duration + name string + host *UpstreamHost + from string + without string + timeout time.Duration + fallbackDelay time.Duration } func (u *fakeUpstream) From() string { @@ -1437,13 +1440,14 @@ func (u *fakeUpstream) Select(r *http.Request) *UpstreamHost { } u.host = &UpstreamHost{ Name: u.name, - ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout()), + ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout(), u.GetFallbackDelay()), } } return u.host } func (u *fakeUpstream) AllowedPath(requestPath string) bool { return true } +func (u *fakeUpstream) GetFallbackDelay() time.Duration { return 300 * time.Millisecond } func (u *fakeUpstream) GetTryDuration() time.Duration { return 1 * time.Second } func (u *fakeUpstream) GetTryInterval() time.Duration { return 250 * time.Millisecond } func (u *fakeUpstream) GetTimeout() time.Duration { return u.timeout } @@ -1474,10 +1478,11 @@ func newPrefixedWebSocketTestProxy(backendAddr string, prefix string) *Proxy { } type fakeWsUpstream struct { - name string - without string - insecure bool - timeout time.Duration + name string + without string + insecure bool + timeout time.Duration + fallbackDelay time.Duration } func (u *fakeWsUpstream) From() string { @@ -1488,7 +1493,7 @@ func (u *fakeWsUpstream) Select(r *http.Request) *UpstreamHost { uri, _ := url.Parse(u.name) host := &UpstreamHost{ Name: u.name, - ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout()), + ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout(), u.GetFallbackDelay()), UpstreamHeaders: http.Header{ "Connection": {"{>Connection}"}, "Upgrade": {"{>Upgrade}"}}, @@ -1500,6 +1505,7 @@ func (u *fakeWsUpstream) Select(r *http.Request) *UpstreamHost { } func (u *fakeWsUpstream) AllowedPath(requestPath string) bool { return true } +func (u *fakeWsUpstream) GetFallbackDelay() time.Duration { return 300 * time.Millisecond } func (u *fakeWsUpstream) GetTryDuration() time.Duration { return 1 * time.Second } func (u *fakeWsUpstream) GetTryInterval() time.Duration { return 250 * time.Millisecond } func (u *fakeWsUpstream) GetTimeout() time.Duration { return u.timeout } @@ -1548,7 +1554,7 @@ func BenchmarkProxy(b *testing.B) { })) defer backend.Close() - upstream := newFakeUpstream(backend.URL, false, 30*time.Second) + upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond) upstream.host.UpstreamHeaders = http.Header{ "Hostname": {"{hostname}"}, "Host": {"{host}"}, diff --git a/caddyhttp/proxy/reverseproxy.go b/caddyhttp/proxy/reverseproxy.go index b638bd23d..279902408 100644 --- a/caddyhttp/proxy/reverseproxy.go +++ b/caddyhttp/proxy/reverseproxy.go @@ -148,7 +148,7 @@ func singleJoiningSlash(a, b string) string { // the target request will be for /base/dir. // Without logic: target's path is "/", incoming is "/api/messages", // without is "/api", then the target request will be for /messages. -func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, timeout time.Duration) *ReverseProxy { +func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, timeout, fallbackDelay time.Duration) *ReverseProxy { targetQuery := target.RawQuery director := func(req *http.Request) { if target.Scheme == "unix" { @@ -234,6 +234,9 @@ func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, t if timeout != defaultDialer.Timeout { dialer.Timeout = timeout } + if fallbackDelay != defaultDialer.FallbackDelay { + dialer.FallbackDelay = fallbackDelay + } rp := &ReverseProxy{ Director: director, diff --git a/caddyhttp/proxy/reverseproxy_test.go b/caddyhttp/proxy/reverseproxy_test.go index 8b01054e5..57c335b0a 100644 --- a/caddyhttp/proxy/reverseproxy_test.go +++ b/caddyhttp/proxy/reverseproxy_test.go @@ -67,7 +67,7 @@ func TestSingleSRVHostReverseProxy(t *testing.T) { } port := uint16(pp) - rp := NewSingleHostReverseProxy(target, "", http.DefaultMaxIdleConnsPerHost, 30*time.Second) + rp := NewSingleHostReverseProxy(target, "", http.DefaultMaxIdleConnsPerHost, 30*time.Second, 300*time.Millisecond) rp.srvResolver = testResolver{ result: []*net.SRV{ {Target: upstream.Hostname(), Port: port, Priority: 1, Weight: 1}, diff --git a/caddyhttp/proxy/upstream.go b/caddyhttp/proxy/upstream.go index 8e5395c6b..033fbecfc 100644 --- a/caddyhttp/proxy/upstream.go +++ b/caddyhttp/proxy/upstream.go @@ -49,6 +49,7 @@ type staticUpstream struct { Hosts HostPool Policy Policy KeepAlive int + FallbackDelay time.Duration Timeout time.Duration FailTimeout time.Duration TryDuration time.Duration @@ -227,7 +228,7 @@ func (u *staticUpstream) NewHost(host string) (*UpstreamHost, error) { return nil, err } - uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix, u.KeepAlive, u.Timeout) + uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix, u.KeepAlive, u.Timeout, u.FallbackDelay) if u.insecureSkipVerify { uh.ReverseProxy.UseInsecureTransport() } @@ -309,6 +310,15 @@ func parseBlock(c *caddyfile.Dispenser, u *staticUpstream, hasSrv bool) error { arg = c.Val() } u.Policy = policyCreateFunc(arg) + case "fallback_delay": + if !c.NextArg() { + return c.ArgErr() + } + dur, err := time.ParseDuration(c.Val()) + if err != nil { + return err + } + u.FallbackDelay = dur case "fail_timeout": if !c.NextArg() { return c.ArgErr() @@ -620,6 +630,11 @@ func (u *staticUpstream) AllowedPath(requestPath string) bool { return true } +// GetFallbackDelay returns u.FallbackDelay. +func (u *staticUpstream) GetFallbackDelay() time.Duration { + return u.FallbackDelay +} + // GetTryDuration returns u.TryDuration. func (u *staticUpstream) GetTryDuration() time.Duration { return u.TryDuration