diff --git a/go.mod b/go.mod index 236a14bd7..41248b79b 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mholt/acmez v1.2.0 github.com/prometheus/client_golang v1.18.0 - github.com/quic-go/quic-go v0.41.0 + github.com/quic-go/quic-go v0.42.0 github.com/smallstep/certificates v0.25.0 github.com/smallstep/nosql v0.6.0 github.com/smallstep/truststore v0.12.1 @@ -41,7 +41,7 @@ require ( golang.org/x/net v0.21.0 golang.org/x/sync v0.5.0 golang.org/x/term v0.17.0 - google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f + golang.org/x/time v0.5.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -69,7 +69,7 @@ require ( go.opentelemetry.io/contrib/propagators/b3 v1.17.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.17.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.17.0 // indirect - go.uber.org/mock v0.3.0 // indirect + go.uber.org/mock v0.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect ) diff --git a/go.sum b/go.sum index 579eadc7d..3a7cd85dd 100644 --- a/go.sum +++ b/go.sum @@ -487,8 +487,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= -github.com/quic-go/quic-go v0.41.0/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -653,8 +653,8 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= -go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= diff --git a/listeners.go b/listeners.go index 84a32e45a..dd02844dd 100644 --- a/listeners.go +++ b/listeners.go @@ -28,11 +28,11 @@ import ( "strings" "sync" "sync/atomic" - "time" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" "go.uber.org/zap" + "golang.org/x/time/rate" "github.com/caddyserver/caddy/v2/internal" ) @@ -406,48 +406,12 @@ func JoinNetworkAddress(network, host, port string) string { return a } -// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future. -func Listen(network, addr string) (net.Listener, error) { - // a 0 timeout means Go uses its default - return ListenTimeout(network, addr, 0) -} - -// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future. -func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) { - netAddr, err := ParseNetworkAddress(JoinNetworkAddress(network, addr, "")) - if err != nil { - return nil, err - } - - ln, err := netAddr.Listen(context.TODO(), 0, net.ListenConfig{KeepAlive: keepalivePeriod}) - if err != nil { - return nil, err - } - - return ln.(net.Listener), nil -} - -// DEPRECATED: Use NetworkAddress.Listen instead. This function will likely be changed or removed in the future. -func ListenPacket(network, addr string) (net.PacketConn, error) { - netAddr, err := ParseNetworkAddress(JoinNetworkAddress(network, addr, "")) - if err != nil { - return nil, err - } - - ln, err := netAddr.Listen(context.TODO(), 0, net.ListenConfig{}) - if err != nil { - return nil, err - } - - return ln.(net.PacketConn), nil -} - // ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module. // The network will be transformed into a QUIC-compatible type (if unix, then // unixgram will be used; otherwise, udp will be used). // // NOTE: This API is EXPERIMENTAL and may be changed or removed. -func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) { +func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config) (http3.QUICEarlyListener, error) { lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset)) sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) { @@ -468,20 +432,22 @@ func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config } } - sqs := newSharedQUICState(tlsConf, activeRequests) + sqs := newSharedQUICState(tlsConf) // http3.ConfigureTLSConfig only uses this field and tls App sets this field as well //nolint:gosec quicTlsConfig := &tls.Config{GetConfigForClient: sqs.getConfigForClient} - earlyLn, err := quic.ListenEarly(h3ln, http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{ - Allow0RTT: true, - RequireAddressValidation: func(clientAddr net.Addr) bool { - // TODO: make tunable? - return sqs.getActiveRequests() > 1000 - }, - }) + // Require clients to verify their source address when we're handling more than 1000 handshakes per second. + // TODO: make tunable? + limiter := rate.NewLimiter(1000, 1000) + tr := &quic.Transport{ + Conn: h3ln, + VerifySourceAddress: func(addr net.Addr) bool { return !limiter.Allow() }, + } + earlyLn, err := tr.ListenEarly(http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{Allow0RTT: true}) if err != nil { return nil, err } + // TODO: figure out when to close the listener and the transport // using the original net.PacketConn to close them properly return &sharedQuicListener{EarlyListener: earlyLn, packetConn: ln, sqs: sqs, key: lnKey}, nil }) @@ -490,47 +456,8 @@ func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config } sql := sharedEarlyListener.(*sharedQuicListener) - // add current tls.Config to sqs, so GetConfigForClient will always return the latest tls.Config in case of context cancellation, - // and the request counter will reflect current http server - ctx, cancel := sql.sqs.addState(tlsConf, activeRequests) - - return &fakeCloseQuicListener{ - sharedQuicListener: sql, - context: ctx, - contextCancel: cancel, - }, nil -} - -// DEPRECATED: Use NetworkAddress.ListenQUIC instead. This function will likely be changed or removed in the future. -// TODO: See if we can find a more elegant solution closer to the new NetworkAddress.Listen API. -func ListenQUIC(ln net.PacketConn, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) { - lnKey := listenerKey("quic+"+ln.LocalAddr().Network(), ln.LocalAddr().String()) - - sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) { - sqs := newSharedQUICState(tlsConf, activeRequests) - // http3.ConfigureTLSConfig only uses this field and tls App sets this field as well - //nolint:gosec - quicTlsConfig := &tls.Config{GetConfigForClient: sqs.getConfigForClient} - earlyLn, err := quic.ListenEarly(ln, http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{ - Allow0RTT: true, - RequireAddressValidation: func(clientAddr net.Addr) bool { - // TODO: make tunable? - return sqs.getActiveRequests() > 1000 - }, - }) - if err != nil { - return nil, err - } - return &sharedQuicListener{EarlyListener: earlyLn, sqs: sqs, key: lnKey}, nil - }) - if err != nil { - return nil, err - } - - sql := sharedEarlyListener.(*sharedQuicListener) - // add current tls.Config and request counter to sqs, so GetConfigForClient will always return the latest tls.Config in case of context cancellation, - // and the request counter will reflect current http server - ctx, cancel := sql.sqs.addState(tlsConf, activeRequests) + // add current tls.Config to sqs, so GetConfigForClient will always return the latest tls.Config in case of context cancellation + ctx, cancel := sql.sqs.addState(tlsConf) return &fakeCloseQuicListener{ sharedQuicListener: sql, @@ -551,25 +478,21 @@ type contextAndCancelFunc struct { context.CancelFunc } -// sharedQUICState manages GetConfigForClient and current number of active requests +// sharedQUICState manages GetConfigForClient // see issue: https://github.com/caddyserver/caddy/pull/4849 type sharedQUICState struct { - rmu sync.RWMutex - tlsConfs map[*tls.Config]contextAndCancelFunc - requestCounters map[*tls.Config]*int64 - activeTlsConf *tls.Config - activeRequestsCounter *int64 + rmu sync.RWMutex + tlsConfs map[*tls.Config]contextAndCancelFunc + activeTlsConf *tls.Config } // newSharedQUICState creates a new sharedQUICState -func newSharedQUICState(tlsConfig *tls.Config, activeRequests *int64) *sharedQUICState { +func newSharedQUICState(tlsConfig *tls.Config) *sharedQUICState { sqtc := &sharedQUICState{ - tlsConfs: make(map[*tls.Config]contextAndCancelFunc), - requestCounters: make(map[*tls.Config]*int64), - activeTlsConf: tlsConfig, - activeRequestsCounter: activeRequests, + tlsConfs: make(map[*tls.Config]contextAndCancelFunc), + activeTlsConf: tlsConfig, } - sqtc.addState(tlsConfig, activeRequests) + sqtc.addState(tlsConfig) return sqtc } @@ -580,17 +503,9 @@ func (sqs *sharedQUICState) getConfigForClient(ch *tls.ClientHelloInfo) (*tls.Co return sqs.activeTlsConf.GetConfigForClient(ch) } -// getActiveRequests returns the number of active requests -func (sqs *sharedQUICState) getActiveRequests() int64 { - // Prevent a race when a context is cancelled and active request counter is being changed - sqs.rmu.RLock() - defer sqs.rmu.RUnlock() - return atomic.LoadInt64(sqs.activeRequestsCounter) -} - // addState adds tls.Config and activeRequests to the map if not present and returns the corresponding context and its cancelFunc -// so that when cancelled, the active tls.Config and request counter will change -func (sqs *sharedQUICState) addState(tlsConfig *tls.Config, activeRequests *int64) (context.Context, context.CancelFunc) { +// so that when cancelled, the active tls.Config will change +func (sqs *sharedQUICState) addState(tlsConfig *tls.Config) (context.Context, context.CancelFunc) { sqs.rmu.Lock() defer sqs.rmu.Unlock() @@ -606,19 +521,16 @@ func (sqs *sharedQUICState) addState(tlsConfig *tls.Config, activeRequests *int6 defer sqs.rmu.Unlock() delete(sqs.tlsConfs, tlsConfig) - delete(sqs.requestCounters, tlsConfig) if sqs.activeTlsConf == tlsConfig { - // select another tls.Config and request counter, if there is none, + // select another tls.Config, if there is none, // related sharedQuicListener will be destroyed anyway - for tc, counter := range sqs.requestCounters { + for tc := range sqs.tlsConfs { sqs.activeTlsConf = tc - sqs.activeRequestsCounter = counter break } } } sqs.tlsConfs[tlsConfig] = contextAndCancelFunc{ctx, wrappedCancel} - sqs.requestCounters[tlsConfig] = activeRequests // there should be at most 2 tls.Configs if len(sqs.tlsConfs) > 2 { Log().Warn("quic listener tls configs are more than 2", zap.Int("number of configs", len(sqs.tlsConfs))) diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 1ebfde61f..ea748bc12 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -27,7 +27,6 @@ import ( "runtime" "strings" "sync" - "sync/atomic" "time" "github.com/caddyserver/certmagic" @@ -43,8 +42,6 @@ import ( // Server describes an HTTP server. type Server struct { - activeRequests int64 // accessed atomically - // Socket addresses to which to bind listeners. Accepts // [network addresses](/docs/conventions#network-addresses) // that may include port ranges. Listener addresses must @@ -274,10 +271,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // advertise HTTP/3, if enabled if s.h3server != nil { - // keep track of active requests for QUIC transport purposes - atomic.AddInt64(&s.activeRequests, 1) - defer atomic.AddInt64(&s.activeRequests, -1) - if r.ProtoMajor < 3 { err := s.h3server.SetQuicHeaders(w.Header()) if err != nil { @@ -567,7 +560,7 @@ func (s *Server) findLastRouteWithHostMatcher() int { // the listener, with Server s as the handler. func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error { addr.Network = getHTTP3Network(addr.Network) - h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg, &s.activeRequests) + h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg) if err != nil { return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err) }