diff --git a/caddyhttp/gzip/gzip.go b/caddyhttp/gzip/gzip.go index ddaac4fb3..29fccac7f 100644 --- a/caddyhttp/gzip/gzip.go +++ b/caddyhttp/gzip/gzip.go @@ -53,9 +53,6 @@ outer: } } - // Delete this header so gzipping is not repeated later in the chain - r.Header.Del("Accept-Encoding") - // gzipWriter modifies underlying writer at init, // use a discard writer instead to leave ResponseWriter in // original form. diff --git a/caddyhttp/gzip/gzip_test.go b/caddyhttp/gzip/gzip_test.go index 9c93a2742..5c57d9313 100644 --- a/caddyhttp/gzip/gzip_test.go +++ b/caddyhttp/gzip/gzip_test.go @@ -91,9 +91,6 @@ func nextFunc(shouldGzip bool) httpserver.Handler { } if shouldGzip { - if r.Header.Get("Accept-Encoding") != "" { - return 0, fmt.Errorf("Accept-Encoding header not expected") - } if w.Header().Get("Content-Encoding") != "gzip" { return 0, fmt.Errorf("Content-Encoding must be gzip, found %v", r.Header.Get("Content-Encoding")) } diff --git a/caddyhttp/gzip/responsefilter.go b/caddyhttp/gzip/responsefilter.go index 3039eb9e6..b62350511 100644 --- a/caddyhttp/gzip/responsefilter.go +++ b/caddyhttp/gzip/responsefilter.go @@ -25,6 +25,20 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { return l != 0 && int64(l) <= length } +// SkipCompressedFilter is ResponseFilter that will discard already compressed responses +type SkipCompressedFilter struct{} + +// ShouldCompress returns true if served file is not already compressed +// encodings via https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding +func (n SkipCompressedFilter) ShouldCompress(w http.ResponseWriter) bool { + switch w.Header().Get("Content-Encoding") { + case "gzip", "compress", "deflate", "br": + return false + default: + return true + } +} + // ResponseFilterWriter validates ResponseFilters. It writes // gzip compressed data if ResponseFilters are satisfied or // uncompressed data otherwise. diff --git a/caddyhttp/gzip/responsefilter_test.go b/caddyhttp/gzip/responsefilter_test.go index a34f58cd3..206826f2f 100644 --- a/caddyhttp/gzip/responsefilter_test.go +++ b/caddyhttp/gzip/responsefilter_test.go @@ -87,3 +87,26 @@ func TestResponseFilterWriter(t *testing.T) { } } } + +func TestResponseGzippedOutput(t *testing.T) { + server := Gzip{Configs: []Config{ + {ResponseFilters: []ResponseFilter{SkipCompressedFilter{}}}, + }} + + server.Next = httpserver.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + w.Header().Set("Content-Encoding", "gzip") + w.Write([]byte("gzipped")) + return 200, nil + }) + + r := urlRequest("/") + r.Header.Set("Accept-Encoding", "gzip") + + w := httptest.NewRecorder() + server.ServeHTTP(w, r) + resp := w.Body.String() + + if resp != "gzipped" { + t.Errorf("Expected output not to be gzipped") + } +} diff --git a/caddyhttp/gzip/setup.go b/caddyhttp/gzip/setup.go index 0a22e23c6..613b6497f 100644 --- a/caddyhttp/gzip/setup.go +++ b/caddyhttp/gzip/setup.go @@ -106,6 +106,8 @@ func gzipParse(c *caddy.Controller) ([]Config, error) { config.RequestFilters = append(config.RequestFilters, DefaultExtFilter()) } + config.ResponseFilters = append(config.ResponseFilters, SkipCompressedFilter{}) + // Response Filters // If min_length is specified, use it. if int64(lengthFilter) != 0 { diff --git a/caddyhttp/gzip/setup_test.go b/caddyhttp/gzip/setup_test.go index f57da0220..31c69e041 100644 --- a/caddyhttp/gzip/setup_test.go +++ b/caddyhttp/gzip/setup_test.go @@ -99,3 +99,31 @@ func TestSetup(t *testing.T) { } } } + +func TestShouldAddResponseFilters(t *testing.T) { + configs, err := gzipParse(caddy.NewTestController("http", `gzip { min_length 654 }`)) + + if err != nil { + t.Errorf("Test expected no error but found: %v", err) + } + filters := 0 + + for _, config := range configs { + for _, filter := range config.ResponseFilters { + switch filter.(type) { + case SkipCompressedFilter: + filters++ + case LengthFilter: + filters++ + + if filter != LengthFilter(654) { + t.Errorf("Expected LengthFilter to have length 654, got: %v", filter) + } + } + } + + if filters != 2 { + t.Errorf("Expected 2 response filters to be registered, got: %v", filters) + } + } +} diff --git a/caddyhttp/staticfiles/fileserver.go b/caddyhttp/staticfiles/fileserver.go index 7c0caa521..be4da60a8 100644 --- a/caddyhttp/staticfiles/fileserver.go +++ b/caddyhttp/staticfiles/fileserver.go @@ -43,6 +43,9 @@ func (fs FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, err // serveFile writes the specified file to the HTTP response. // name is '/'-separated, not filepath.Separator. func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name string) (int, error) { + + location := name + // Prevent absolute path access on Windows. // TODO remove when stdlib http.Dir fixes this. if runtime.GOOS == "windows" { @@ -97,17 +100,27 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri for _, indexPage := range IndexPages { index := strings.TrimSuffix(name, "/") + "/" + indexPage ff, err := fs.Root.Open(index) - if err == nil { - // this defer does not leak fds because previous iterations - // of the loop must have had an err, so nothing to close - defer ff.Close() - dd, err := ff.Stat() - if err == nil { - d = dd - f = ff - break - } + if err != nil { + continue } + + // this defer does not leak fds because previous iterations + // of the loop must have had an err, so nothing to close + defer ff.Close() + + dd, err := ff.Stat() + if err != nil { + ff.Close() + continue + } + + // Close previous file - release fd immediately + f.Close() + + d = dd + f = ff + location = index + break } } @@ -121,13 +134,48 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri return http.StatusNotFound, nil } + filename := d.Name() + + for _, encoding := range staticEncodingPriority { + if !strings.Contains(r.Header.Get("Accept-Encoding"), encoding) { + continue + } + + encodedFile, err := fs.Root.Open(location + staticEncoding[encoding]) + if err != nil { + continue + } + + encodedFileInfo, err := encodedFile.Stat() + if err != nil { + encodedFile.Close() + continue + } + + // Close previous file - release fd + f.Close() + + // Stat is needed for generating valid ETag + d = encodedFileInfo + + // Encoded file will be served + f = encodedFile + + w.Header().Add("Vary", "Accept-Encoding") + w.Header().Set("Content-Encoding", encoding) + + defer f.Close() + break + + } + // Experimental ETag header e := fmt.Sprintf(`W/"%x-%x"`, d.ModTime().Unix(), d.Size()) w.Header().Set("ETag", e) // Note: Errors generated by ServeContent are written immediately // to the response. This usually only happens if seeking fails (rare). - http.ServeContent(w, r, d.Name(), d.ModTime(), f) + http.ServeContent(w, r, filename, d.ModTime(), f) return http.StatusOK, nil } @@ -168,3 +216,17 @@ var IndexPages = []string{ "default.htm", "default.txt", } + +// staticEncoding is a map of content-encoding to a file extension. +// If client accepts given encoding (via Accept-Encoding header) and compressed file with given extensions exists +// it will be served to the client instead of original one. +var staticEncoding = map[string]string{ + "gzip": ".gz", + "br": ".br", +} + +// staticEncodingPriority is a list of preferred static encodings (most efficient compression to least one). +var staticEncodingPriority = []string{ + "br", + "gzip", +} diff --git a/caddyhttp/staticfiles/fileserver_test.go b/caddyhttp/staticfiles/fileserver_test.go index 28855e66b..346a1d152 100644 --- a/caddyhttp/staticfiles/fileserver_test.go +++ b/caddyhttp/staticfiles/fileserver_test.go @@ -33,6 +33,12 @@ var ( var testFiles = map[string]string{ "unreachable.html": "

must not leak

", filepath.Join("webroot", "file1.html"): "

file1.html

", + filepath.Join("webroot", "sub", "gzipped.html"): "

gzipped.html

", + filepath.Join("webroot", "sub", "gzipped.html.gz"): "gzipped.html.gz", + filepath.Join("webroot", "sub", "gzipped.html.gz"): "gzipped.html.gz", + filepath.Join("webroot", "sub", "brotli.html"): "brotli.html", + filepath.Join("webroot", "sub", "brotli.html.gz"): "brotli.html.gz", + filepath.Join("webroot", "sub", "brotli.html.br"): "brotli.html.br", filepath.Join("webroot", "dirwithindex", "index.html"): "

dirwithindex/index.html

", filepath.Join("webroot", "dir", "file2.html"): "

dir/file2.html

", filepath.Join("webroot", "dir", "hidden.html"): "

dir/hidden.html

", @@ -72,14 +78,14 @@ func TestServeHTTP(t *testing.T) { { url: "https://foo/file1.html", expectedStatus: http.StatusOK, - expectedBodyContent: testFiles["file1.html"], + expectedBodyContent: testFiles[filepath.Join("webroot", "file1.html")], expectedEtag: `W/"1e240-13"`, }, // Test 3 - access folder with index file with trailing slash { url: "https://foo/dirwithindex/", expectedStatus: http.StatusOK, - expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")], + expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")], expectedEtag: `W/"1e240-20"`, }, // Test 4 - access folder with index file without trailing slash @@ -119,7 +125,7 @@ func TestServeHTTP(t *testing.T) { { url: "https://foo/dirwithindex/index.html", expectedStatus: http.StatusOK, - expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")], + expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")], expectedEtag: `W/"1e240-20"`, }, // Test 11 - send a request with query params @@ -158,11 +164,28 @@ func TestServeHTTP(t *testing.T) { url: "https://foo/%2f..%2funreachable.html", expectedStatus: http.StatusNotFound, }, + // Test 18 - try to get pre-gzipped file. + { + url: "https://foo/sub/gzipped.html", + expectedStatus: http.StatusOK, + expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "gzipped.html.gz")], + expectedEtag: `W/"1e240-f"`, + }, + // Test 19 - try to get pre-brotli encoded file. + { + url: "https://foo/sub/brotli.html", + expectedStatus: http.StatusOK, + expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "brotli.html.br")], + expectedEtag: `W/"1e240-e"`, + }, } for i, test := range tests { responseRecorder := httptest.NewRecorder() request, err := http.NewRequest("GET", test.url, nil) + + request.Header.Add("Accept-Encoding", "br,gzip") + if err != nil { t.Errorf("Test %d: Error making request: %v", i, err) }