From f5b4f268dc73078467f19faf6068cc8047cabc7f Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Mon, 10 Jun 2019 10:21:25 -0600 Subject: [PATCH] Implement encode middleware (#2) * Implement encode middleware * Add missing break; and add missing JSON struct field tag --- cmd/caddy2/main.go | 2 + modules/caddyhttp/encode/encode.go | 286 ++++++++++++++++++++++++++ modules/caddyhttp/encode/gzip/gzip.go | 36 ++++ modules/caddyhttp/encode/zstd/zstd.go | 23 +++ 4 files changed, 347 insertions(+) create mode 100644 modules/caddyhttp/encode/encode.go create mode 100644 modules/caddyhttp/encode/gzip/gzip.go create mode 100644 modules/caddyhttp/encode/zstd/zstd.go diff --git a/cmd/caddy2/main.go b/cmd/caddy2/main.go index d0289ecf4..9e1d47f3e 100644 --- a/cmd/caddy2/main.go +++ b/cmd/caddy2/main.go @@ -6,6 +6,8 @@ import ( // this is where modules get plugged in _ "github.com/caddyserver/caddy2/modules/caddyhttp" _ "github.com/caddyserver/caddy2/modules/caddyhttp/caddylog" + _ "github.com/caddyserver/caddy2/modules/caddyhttp/encode" + _ "github.com/caddyserver/caddy2/modules/caddyhttp/encode/gzip" _ "github.com/caddyserver/caddy2/modules/caddyhttp/fileserver" _ "github.com/caddyserver/caddy2/modules/caddyhttp/headers" _ "github.com/caddyserver/caddy2/modules/caddyhttp/markdown" diff --git a/modules/caddyhttp/encode/encode.go b/modules/caddyhttp/encode/encode.go new file mode 100644 index 000000000..2d6f5242f --- /dev/null +++ b/modules/caddyhttp/encode/encode.go @@ -0,0 +1,286 @@ +// Package encode implements an encoder middleware for Caddy. The initial +// enhancements related to Accept-Encoding, minimum content length, and +// buffer/writer pools were adapted from https://github.com/xi2/httpgzip +// then modified heavily to accommodate modular encoders. Code borrowed +// from that repository is Copyright (c) 2015 The Httpgzip Authors. +package encode + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "sync" + + "github.com/caddyserver/caddy2" + "github.com/caddyserver/caddy2/modules/caddyhttp" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.middleware.encode", + New: func() interface{} { return new(Encode) }, + }) +} + +// Encode is a middleware which can encode responses. +type Encode struct { + EncodingsRaw map[string]json.RawMessage `json:"encodings,omitempty"` + Prefer []string `json:"prefer,omitempty"` + MinLength int `json:"minimum_length,omitempty"` + + Encodings map[string]Encoding `json:"-"` + + writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads... +} + +// Provision provisions enc. +func (enc *Encode) Provision(ctx caddy2.Context) error { + enc.Encodings = make(map[string]Encoding) + enc.writerPools = make(map[string]*sync.Pool) + + for modName, rawMsg := range enc.EncodingsRaw { + val, err := ctx.LoadModule("http.encoders."+modName, rawMsg) + if err != nil { + return fmt.Errorf("loading encoder module '%s': %v", modName, err) + } + encoder := val.(Encoding) + enc.Encodings[modName] = encoder + + enc.writerPools[modName] = &sync.Pool{ + New: func() interface{} { + return encoder.NewEncoder() + }, + } + } + enc.EncodingsRaw = nil // allow GC to deallocate - TODO: Does this help? + + if enc.MinLength == 0 { + enc.MinLength = defaultMinLength + } + + return nil +} + +func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + for _, encName := range acceptedEncodings(r) { + if _, ok := enc.writerPools[encName]; !ok { + continue // encoding not offered + } + w = enc.openResponseWriter(encName, w) + defer w.(*responseWriter).Close() + break + } + + return next.ServeHTTP(w, r) +} + +// openResponseWriter creates a new response writer that may (or may not) +// encode the response with encodingName. The returned response writer MUST +// be closed after the handler completes. +func (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter) *responseWriter { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + return &responseWriter{ + ResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w}, + encodingName: encodingName, + buf: buf, + config: enc, + } +} + +// responseWriter writes to an underlying response writer +// using the encoding represented by encodingName and +// configured by config. +type responseWriter struct { + *caddyhttp.ResponseWriterWrapper + encodingName string + w Encoder + buf *bytes.Buffer + config *Encode + statusCode int +} + +// WriteHeader stores the status to write when the time comes +// to actually write the header. +func (rw *responseWriter) WriteHeader(status int) { + rw.statusCode = status +} + +// Write writes to the response. If the response qualifies, +// it is encoded using the encoder, which is initialized +// if not done so already. +func (rw *responseWriter) Write(p []byte) (int, error) { + var n, written int + var err error + + if rw.buf != nil && rw.config.MinLength > 0 { + written = rw.buf.Len() + _, err := rw.buf.Write(p) + if err != nil { + return 0, err + } + if rw.buf.Len() < rw.config.MinLength { + return len(p), nil + } + rw.init() + p = rw.buf.Bytes() + defer func() { + bufPool.Put(rw.buf) + rw.buf = nil + }() + } + + switch { + case rw.w != nil: + n, err = rw.w.Write(p) + default: + n, err = rw.ResponseWriter.Write(p) + } + n -= written + if n < 0 { + n = 0 + } + return n, err +} + +// init should be called once we know we are writing an encoded response. +func (rw *responseWriter) init() { + if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength { + rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder) + rw.w.Reset(rw.ResponseWriter) + rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975 + rw.Header().Set("Content-Encoding", rw.encodingName) + } + rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content + rw.ResponseWriter.WriteHeader(rw.statusCode) +} + +// Close writes any remaining buffered response and +// deallocates any active resources. +func (rw *responseWriter) Close() error { + var err error + if rw.buf != nil { + rw.init() + p := rw.buf.Bytes() + defer func() { + bufPool.Put(rw.buf) + rw.buf = nil + }() + switch { + case rw.w != nil: + _, err = rw.w.Write(p) + default: + _, err = rw.ResponseWriter.Write(p) + } + } + if rw.w != nil { + err2 := rw.w.Close() + if err2 != nil && err == nil { + err = err2 + } + rw.config.writerPools[rw.encodingName].Put(rw.w) + rw.w = nil + } + return err +} + +// acceptedEncodings returns the list of encodings that the +// client supports, in descending order of preference. If +// the Sec-WebSocket-Key header is present then non-identity +// encodings are not considered. See +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html. +func acceptedEncodings(r *http.Request) []string { + acceptEncHeader := r.Header.Get("Accept-Encoding") + websocketKey := r.Header.Get("Sec-WebSocket-Key") + if acceptEncHeader == "" { + return []string{} + } + + var prefs []encodingPreference + + for _, accepted := range strings.Split(acceptEncHeader, ",") { + parts := strings.Split(accepted, ";") + encName := strings.ToLower(strings.TrimSpace(parts[0])) + + // determine q-factor + qFactor := 1.0 + if len(parts) > 1 { + qFactorStr := strings.ToLower(strings.TrimSpace(parts[1])) + if strings.HasPrefix(qFactorStr, "q=") { + if qFactorFloat, err := strconv.ParseFloat(qFactorStr[2:], 32); err == nil { + if qFactorFloat >= 0 && qFactorFloat <= 1 { + qFactor = qFactorFloat + } + } + } + } + + // encodings with q-factor of 0 are not accepted; + // use a small theshold to account for float precision + if qFactor < 0.00001 { + continue + } + + // don't encode WebSocket handshakes + if websocketKey != "" && encName != "identity" { + continue + } + + prefs = append(prefs, encodingPreference{ + encoding: encName, + q: qFactor, + }) + } + + // sort preferences by descending q-factor + sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q }) + + // TODO: If no preference, or same pref for all encodings, + // and not websocket, use default encoding ordering (enc.Prefer) + // for those which are accepted by the client + + prefEncNames := make([]string, len(prefs)) + for i := range prefs { + prefEncNames[i] = prefs[i].encoding + } + + return prefEncNames +} + +// encodingPreference pairs an encoding with its q-factor. +type encodingPreference struct { + encoding string + q float64 +} + +// Encoder is a type which can encode a stream of data. +type Encoder interface { + io.WriteCloser + Reset(io.Writer) +} + +// Encoding is a type which can create encoders of its kind. +type Encoding interface { + NewEncoder() Encoder +} + +var bufPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, +} + +// defaultMinLength is the minimum length at which to compress content. +const defaultMinLength = 512 + +// Interface guards +var ( + _ caddy2.Provisioner = (*Encode)(nil) + _ caddyhttp.MiddlewareHandler = (*Encode)(nil) + _ caddyhttp.HTTPInterfaces = (*responseWriter)(nil) +) diff --git a/modules/caddyhttp/encode/gzip/gzip.go b/modules/caddyhttp/encode/gzip/gzip.go new file mode 100644 index 000000000..0da5fc346 --- /dev/null +++ b/modules/caddyhttp/encode/gzip/gzip.go @@ -0,0 +1,36 @@ +package caddygzip + +import ( + "compress/flate" + "compress/gzip" // TODO: consider using https://github.com/klauspost/compress/gzip + + "github.com/caddyserver/caddy2" + "github.com/caddyserver/caddy2/modules/caddyhttp/encode" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.encoders.gzip", + New: func() interface{} { return new(Gzip) }, + }) +} + +// Gzip can create gzip encoders. +type Gzip struct { + Level int `json:"level,omitempty"` +} + +// NewEncoder returns a new gzip writer. +func (g Gzip) NewEncoder() encode.Encoder { + if g.Level <= flate.NoCompression { + g.Level = defaultGzipLevel + } + if g.Level > flate.BestCompression { + g.Level = flate.BestCompression + } + writer, _ := gzip.NewWriterLevel(nil, g.Level) + return writer +} + +// Informed from http://blog.klauspost.com/gzip-performance-for-go-webservers/ +var defaultGzipLevel = 5 diff --git a/modules/caddyhttp/encode/zstd/zstd.go b/modules/caddyhttp/encode/zstd/zstd.go new file mode 100644 index 000000000..6afe53ef4 --- /dev/null +++ b/modules/caddyhttp/encode/zstd/zstd.go @@ -0,0 +1,23 @@ +package caddystd + +import ( + "github.com/caddyserver/caddy2" + "github.com/caddyserver/caddy2/modules/caddyhttp/encode" + "github.com/klauspost/compress/zstd" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.encoders.zstd", + New: func() interface{} { return new(Zstd) }, + }) +} + +// Zstd can create zstd encoders. +type Zstd struct{} + +// NewEncoder returns a new gzip writer. +func (z Zstd) NewEncoder() encode.Encoder { + writer, _ := zstd.NewWriter(nil) + return writer +}