serve http: support unix sockets and multiple listners

- add support for unix sockets (which skip the auth).
- add support for multiple listeners
- collapse unnecessary internal structure of lib/http so it can all be
  imported together
- moves files in sub directories of lib/http into the main lib/http
  directory and reworks the code that uses them.

See: https://forum.rclone.org/t/wip-rc-rcd-over-unix-socket/33619
Fixes: #6605

cmd/serve/http/http.go

@@ -2,7 +2,8 @@
 package http
 
 import (
-	"html/template"
+	"context"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -10,16 +11,15 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/http/data"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	httplib "github.com/rclone/rclone/lib/http"
+	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/http/auth"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -28,20 +28,27 @@ import (
 
 // Options required for http server
 type Options struct {
-	data.Options
+	Auth     libhttp.AuthConfig
+	HTTP     libhttp.HTTPConfig
+	Template libhttp.TemplateConfig
 }
 
 // DefaultOpt is the default values used for Options
-var DefaultOpt = Options{}
+var DefaultOpt = Options{
+	Auth:     libhttp.DefaultAuthCfg(),
+	HTTP:     libhttp.DefaultHTTPCfg(),
+	Template: libhttp.DefaultTemplateCfg(),
+}
 
 // Opt is options set by command line flags
 var Opt = DefaultOpt
 
 func init() {
-	data.AddFlags(Command.Flags(), "", &Opt.Options)
+	flagSet := Command.Flags()
-	httplib.AddFlags(Command.Flags())
+	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
-	auth.AddFlags(Command.Flags())
+	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
-	vfsflags.AddFlags(Command.Flags())
+	libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
+	vfsflags.AddFlags(flagSet)
 }
 
 // Command definition for cobra
@@ -59,60 +66,78 @@ The server will log errors.  Use ` + "`-v`" + ` to see access logs.
 
 ` + "`--bwlimit`" + ` will be respected for file transfers.  Use ` + "`--stats`" + ` to
 control the stats printing.
-` + httplib.Help + data.Help + auth.Help + vfs.Help,
+` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
+
 		cmd.Run(false, true, command, func() error {
-			s := newServer(f, Opt.Template)
+			ctx := context.Background()
-			router, err := httplib.Router()
+
+			s, err := run(ctx, f, Opt)
 			if err != nil {
-				return err
+				log.Fatal(err)
 			}
-			s.Bind(router)
+
-			httplib.Wait()
+			var finaliseOnce sync.Once
+			finalise := func() {
+				finaliseOnce.Do(func() {
+					if err := s.server.Shutdown(); err != nil {
+						log.Printf("error shutting down server: %v", err)
+					}
+				})
+			}
+			fnHandle := atexit.Register(finalise)
+			defer atexit.Unregister(fnHandle)
+
+			s.server.Wait()
 			return nil
 		})
 	},
 }
 
 // server contains everything to run the server
-type server struct {
+type serveCmd struct {
-	f            fs.Fs
+	f      fs.Fs
-	vfs          *vfs.VFS
+	vfs    *vfs.VFS
-	HTMLTemplate *template.Template // HTML template for web interface
+	server libhttp.Server
 }
 
-func newServer(f fs.Fs, templatePath string) *server {
+func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
-	htmlTemplate, templateErr := data.GetTemplate(templatePath)
+	var err error
-	if templateErr != nil {
-		log.Fatalf(templateErr.Error())
-	}
-	s := &server{
-		f:            f,
-		vfs:          vfs.New(f, &vfsflags.Opt),
-		HTMLTemplate: htmlTemplate,
-	}
-	return s
-}
 
-func (s *server) Bind(router chi.Router) {
+	s := &serveCmd{
-	if m := auth.Auth(auth.Opt); m != nil {
+		f:   f,
-		router.Use(m)
+		vfs: vfs.New(f, &vfsflags.Opt),
 	}
+
+	s.server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.HTTP),
+		libhttp.WithAuth(opt.Auth),
+		libhttp.WithTemplate(opt.Template),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
+	}
+
+	router := s.server.Router()
 	router.Use(
 		middleware.SetHeader("Accept-Ranges", "bytes"),
 		middleware.SetHeader("Server", "rclone/"+fs.Version),
 	)
 	router.Get("/*", s.handler)
 	router.Head("/*", s.handler)
+
+	s.server.Serve()
+
+	return s, nil
 }
 
 // handler reads incoming requests and dispatches them
-func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) {
 	isDir := strings.HasSuffix(r.URL.Path, "/")
 	remote := strings.Trim(r.URL.Path, "/")
 	if isDir {
@@ -123,7 +148,7 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
 }
 
 // serveDir serves a directory index at dirRemote
-func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
+func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
 	// List the directory
 	node, err := s.vfs.Stat(dirRemote)
 	if err == vfs.ENOENT {
@@ -145,7 +170,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
 	}
 
 	// Make the entries for display
-	directory := serve.NewDirectory(dirRemote, s.HTMLTemplate)
+	directory := serve.NewDirectory(dirRemote, s.server.HTMLTemplate())
 	for _, node := range dirEntries {
 		if vfsflags.Opt.NoModTime {
 			directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{})
@@ -165,7 +190,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
 }
 
 // serveFile serves a file object at remote
-func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
+func (s *serveCmd) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
 	node, err := s.vfs.Stat(remote)
 	if err == vfs.ENOENT {
 		fs.Infof(remote, "%s: File not found", r.RemoteAddr)

cmd/serve/http/http_test.go

@@ -14,14 +14,14 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/filter"
-	httplib "github.com/rclone/rclone/lib/http"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 var (
 	updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
-	httpServer   *server
+	sc           *serveCmd
 	testURL      string
 )
 
@@ -30,16 +30,25 @@ const (
 	testTemplate    = "testdata/golden/testindex.html"
 )
 
-func startServer(t *testing.T, f fs.Fs) {
+func start(t *testing.T, f fs.Fs) {
-	opt := httplib.DefaultOpt
+	ctx := context.Background()
-	opt.ListenAddr = testBindAddress
+
-	httpServer = newServer(f, testTemplate)
+	opts := Options{
-	router, err := httplib.Router()
+		HTTP: libhttp.DefaultHTTPCfg(),
-	if err != nil {
+		Template: libhttp.TemplateConfig{
-		t.Fatal(err.Error())
+			Path: testTemplate,
+		},
 	}
-	httpServer.Bind(router)
+	opts.HTTP.ListenAddr = []string{testBindAddress}
-	testURL = httplib.URL()
+
+	s, err := run(ctx, f, opts)
+	require.NoError(t, err, "failed to start server")
+	sc = s
+
+	urls := s.server.URLs()
+	require.Len(t, urls, 1, "expected one URL")
+
+	testURL = urls[0]
 
 	// try to connect to the test server
 	pause := time.Millisecond
@@ -54,7 +63,6 @@ func startServer(t *testing.T, f fs.Fs) {
 		pause *= 2
 	}
 	t.Fatal("couldn't connect to server")
-
 }
 
 var (
@@ -84,7 +92,7 @@ func TestInit(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, obj.SetModTime(context.Background(), expectedTime))
 
-	startServer(t, f)
+	start(t, f)
 }
 
 // check body against the file, or re-write body if -updategolden is
@@ -229,7 +237,3 @@ func TestGET(t *testing.T) {
 		checkGolden(t, test.Golden, body)
 	}
 }
-
-func TestFinalise(t *testing.T) {
-	_ = httplib.Shutdown()
-}

lib/http/auth.go

@@ -0,0 +1,70 @@
+package http
+
+import (
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Help contains text describing the http authentication to add to the command
+// help.
+var AuthHelp = `
+#### Authentication
+
+By default this will serve files without needing a login.
+
+You can either use an htpasswd file which can take lots of users, or
+set a single username and password with the ` + "`--user` and `--pass`" + ` flags.
+
+Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file.  This is
+in standard apache format and supports MD5, SHA1 and BCrypt for basic
+authentication.  Bcrypt is recommended.
+
+To create an htpasswd file:
+
+    touch htpasswd
+    htpasswd -B htpasswd user
+    htpasswd -B htpasswd anotherUser
+
+The password file can be updated while rclone is running.
+
+Use ` + "`--realm`" + ` to set the authentication realm.
+
+Use ` + "`--salt`" + ` to change the password hashing salt from the default.
+`
+
+// CustomAuthFn if used will be used to authenticate user, pass. If an error
+// is returned then the user is not authenticated.
+//
+// If a non nil value is returned then it is added to the context under the key
+type CustomAuthFn func(user, pass string) (value interface{}, err error)
+
+// AuthConfig contains options for the http authentication
+type AuthConfig struct {
+	HtPasswd     string       // htpasswd file - if not provided no authentication is done
+	Realm        string       // realm for authentication
+	BasicUser    string       // single username for basic auth if not using Htpasswd
+	BasicPass    string       // password for BasicUser
+	Salt         string       // password hashing salt
+	CustomAuthFn CustomAuthFn `json:"-"` // custom Auth (not set by command line flags)
+}
+
+// AddFlagsPrefix adds flags to the flag set for AuthConfig
+func (cfg *AuthConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringVarP(flagSet, &cfg.HtPasswd, prefix+"htpasswd", "", cfg.HtPasswd, "A htpasswd file - if not provided no authentication is done")
+	flags.StringVarP(flagSet, &cfg.Realm, prefix+"realm", "", cfg.Realm, "Realm for authentication")
+	flags.StringVarP(flagSet, &cfg.BasicUser, prefix+"user", "", cfg.BasicUser, "User name for authentication")
+	flags.StringVarP(flagSet, &cfg.BasicPass, prefix+"pass", "", cfg.BasicPass, "Password for authentication")
+	flags.StringVarP(flagSet, &cfg.Salt, prefix+"salt", "", cfg.Salt, "Password hashing salt")
+}
+
+// AddAuthFlagsPrefix adds flags to the flag set for AuthConfig
+func AddAuthFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *AuthConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultAuthCfg returns a new config which can be customized by command line flags
+func DefaultAuthCfg() AuthConfig {
+	return AuthConfig{
+		Salt: "dlPL2MqE",
+	}
+}

lib/http/auth/auth.go

@@ -1,84 +0,0 @@
-// Package auth provides authentication for http.
-package auth
-
-import (
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/lib/http"
-	"github.com/spf13/pflag"
-)
-
-// Help contains text describing the http authentication to add to the command
-// help.
-var Help = `
-#### Authentication
-
-By default this will serve files without needing a login.
-
-You can either use an htpasswd file which can take lots of users, or
-set a single username and password with the ` + "`--user` and `--pass`" + ` flags.
-
-Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file.  This is
-in standard apache format and supports MD5, SHA1 and BCrypt for basic
-authentication.  Bcrypt is recommended.
-
-To create an htpasswd file:
-
-    touch htpasswd
-    htpasswd -B htpasswd user
-    htpasswd -B htpasswd anotherUser
-
-The password file can be updated while rclone is running.
-
-Use ` + "`--realm`" + ` to set the authentication realm.
-
-Use ` + "`--salt`" + ` to change the password hashing salt from the default.
-`
-
-// CustomAuthFn if used will be used to authenticate user, pass. If an error
-// is returned then the user is not authenticated.
-//
-// If a non nil value is returned then it is added to the context under the key
-type CustomAuthFn func(user, pass string) (value interface{}, err error)
-
-// Options contains options for the http authentication
-type Options struct {
-	HtPasswd  string       // htpasswd file - if not provided no authentication is done
-	Realm     string       // realm for authentication
-	BasicUser string       // single username for basic auth if not using Htpasswd
-	BasicPass string       // password for BasicUser
-	Salt      string       // password hashing salt
-	Auth      CustomAuthFn `json:"-"` // custom Auth (not set by command line flags)
-}
-
-// Auth instantiates middleware that authenticates users based on the configuration
-func Auth(opt Options) http.Middleware {
-	if opt.Auth != nil {
-		return CustomAuth(opt.Auth, opt.Realm)
-	} else if opt.HtPasswd != "" {
-		return HtPasswdAuth(opt.HtPasswd, opt.Realm)
-	} else if opt.BasicUser != "" {
-		return SingleAuth(opt.BasicUser, opt.BasicPass, opt.Realm, opt.Salt)
-	}
-	return nil
-}
-
-// Options set by command line flags
-var (
-	Opt = Options{
-		Salt: "dlPL2MqE",
-	}
-)
-
-// AddFlagsPrefix adds flags for http/auth
-func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
-	flags.StringVarP(flagSet, &Opt.HtPasswd, prefix+"htpasswd", "", Opt.HtPasswd, "A htpasswd file - if not provided no authentication is done")
-	flags.StringVarP(flagSet, &Opt.Realm, prefix+"realm", "", Opt.Realm, "Realm for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication")
-	flags.StringVarP(flagSet, &Opt.Salt, prefix+"salt", "", Opt.Salt, "Password hashing salt")
-}
-
-// AddFlags adds flags for the http/auth
-func AddFlags(flagSet *pflag.FlagSet) {
-	AddFlagsPrefix(flagSet, "", &Opt)
-}

lib/http/auth/basic.go

@@ -1,120 +0,0 @@
-package auth
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"strings"
-
-	auth "github.com/abbot/go-http-auth"
-	"github.com/rclone/rclone/fs"
-	httplib "github.com/rclone/rclone/lib/http"
-)
-
-// parseAuthorization parses the Authorization header into user, pass
-// it returns a boolean as to whether the parse was successful
-func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
-	authHeader := r.Header.Get("Authorization")
-	if authHeader != "" {
-		s := strings.SplitN(authHeader, " ", 2)
-		if len(s) == 2 && s[0] == "Basic" {
-			b, err := base64.StdEncoding.DecodeString(s[1])
-			if err == nil {
-				parts := strings.SplitN(string(b), ":", 2)
-				user = parts[0]
-				if len(parts) > 1 {
-					pass = parts[1]
-					ok = true
-				}
-			}
-		}
-	}
-	return
-}
-
-type contextUserType struct{}
-
-// ContextUserKey is a simple context key for storing the username of the request
-var ContextUserKey = &contextUserType{}
-
-type contextAuthType struct{}
-
-// ContextAuthKey is a simple context key for storing info returned by CustomAuthFn
-var ContextAuthKey = &contextAuthType{}
-
-// LoggedBasicAuth extends BasicAuth to include access logging
-type LoggedBasicAuth struct {
-	auth.BasicAuth
-}
-
-// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests
-func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string {
-	username := a.BasicAuth.CheckAuth(r)
-	if username == "" {
-		user, _, _ := parseAuthorization(r)
-		fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user)
-	}
-	return username
-}
-
-// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator
-func NewLoggedBasicAuthenticator(realm string, secrets auth.SecretProvider) *LoggedBasicAuth {
-	return &LoggedBasicAuth{BasicAuth: auth.BasicAuth{Realm: realm, Secrets: secrets}}
-}
-
-// Helper to generate required interface for middleware
-func basicAuth(authenticator *LoggedBasicAuth) httplib.Middleware {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if username := authenticator.CheckAuth(r); username == "" {
-				authenticator.RequireAuth(w, r)
-			} else {
-				r = r.WithContext(context.WithValue(r.Context(), ContextUserKey, username))
-				next.ServeHTTP(w, r)
-			}
-		})
-	}
-}
-
-// HtPasswdAuth instantiates middleware that authenticates against the passed htpasswd file
-func HtPasswdAuth(path, realm string) httplib.Middleware {
-	fs.Infof(nil, "Using %q as htpasswd storage", path)
-	secretProvider := auth.HtpasswdFileProvider(path)
-	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
-	return basicAuth(authenticator)
-}
-
-// SingleAuth instantiates middleware that authenticates for a single user
-func SingleAuth(user, pass, realm, salt string) httplib.Middleware {
-	fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user)
-	pass = string(auth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$")))
-	secretProvider := func(u, r string) string {
-		if user == u {
-			return pass
-		}
-		return ""
-	}
-	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
-	return basicAuth(authenticator)
-}
-
-// CustomAuth instantiates middleware that authenticates using a custom function
-func CustomAuth(fn CustomAuthFn, realm string) httplib.Middleware {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			user, pass, ok := parseAuthorization(r)
-			if ok {
-				value, err := fn(user, pass)
-				if err != nil {
-					fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err)
-					auth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting
-					return
-				}
-				if value != nil {
-					r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value))
-				}
-				next.ServeHTTP(w, r)
-			}
-		})
-	}
-}

lib/http/context.go

@@ -0,0 +1,68 @@
+package http
+
+import (
+	"context"
+	"net"
+	"net/http"
+)
+
+type ctxKey int
+
+const (
+	ctxKeyAuth ctxKey = iota
+	ctxKeyPublicURL
+	ctxKeyUnixSock
+	ctxKeyUser
+)
+
+// NewBaseContext initializes the context for all requests, adding info for use in middleware and handlers
+func NewBaseContext(ctx context.Context, url string) func(l net.Listener) context.Context {
+	return func(l net.Listener) context.Context {
+		if l.Addr().Network() == "unix" {
+			ctx = context.WithValue(ctx, ctxKeyUnixSock, true)
+			return ctx
+		}
+
+		ctx = context.WithValue(ctx, ctxKeyPublicURL, url)
+		return ctx
+	}
+}
+
+// IsAuthenticated checks if this request was authenticated via a middleware
+func IsAuthenticated(r *http.Request) bool {
+	if v := r.Context().Value(ctxKeyAuth); v != nil {
+		return true
+	}
+	if v := r.Context().Value(ctxKeyUser); v != nil {
+		return true
+	}
+	return false
+}
+
+// IsUnixSocket checks if the request was received on a unix socket, used to skip auth & CORS
+func IsUnixSocket(r *http.Request) bool {
+	v, _ := r.Context().Value(ctxKeyUnixSock).(bool)
+	return v
+}
+
+// PublicURL returns the URL defined in NewBaseContext, used for logging & CORS
+func PublicURL(r *http.Request) string {
+	v, _ := r.Context().Value(ctxKeyPublicURL).(string)
+	return v
+}
+
+// CtxGetAuth is a wrapper over the private Auth context key
+func CtxGetAuth(ctx context.Context) interface{} {
+	return ctx.Value(ctxKeyAuth)
+}
+
+// CtxGetUser is a wrapper over the private User context key
+func CtxGetUser(ctx context.Context) (string, bool) {
+	v, ok := ctx.Value(ctxKeyUser).(string)
+	return v, ok
+}
+
+// CtxSetUser is a test helper that injects a User value into context
+func CtxSetUser(ctx context.Context, value string) context.Context {
+	return context.WithValue(ctx, ctxKeyUser, value)
+}

lib/http/http.go

@@ -1,441 +0,0 @@
-// Package http provides a registration interface for http services
-package http
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/spf13/pflag"
-)
-
-// Help contains text describing the http server to add to the command
-// help.
-var Help = `
-### Server options
-
-Use ` + "`--addr`" + ` to specify which IP address and port the server should
-listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all
-IPs.  By default it only listens on localhost.  You can use port
-:0 to let the OS choose an available port.
-
-If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address
-then using Authentication is advised - see the next section for info.
-
-` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to
-control the timeouts on the server.  Note that this is the total time
-for a transfer.
-
-` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will
-accept in the HTTP header.
-
-` + "`--baseurl`" + ` controls the URL prefix that rclone serves from.  By default
-rclone will serve from the root.  If you used ` + "`--baseurl \"/rclone\"`" + ` then
-rclone would serve from a URL starting with "/rclone/".  This is
-useful if you wish to proxy rclone serve.  Rclone automatically
-inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `,
-` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated
-identically.
-
-#### SSL/TLS
-
-By default this will serve over http.  If you want you can serve over
-https.  You will need to supply the ` + "`--cert` and `--key`" + ` flags.
-If you wish to do client side certificate validation then you will need to
-supply ` + "`--client-ca`" + ` also.
-
-` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation
-of that with the CA certificate.  ` + "`--key`" + ` should be the PEM encoded
-private key and ` + "`--client-ca`" + ` should be the PEM encoded client
-certificate authority certificate.
-
---min-tls-version is minimum TLS version that is acceptable. Valid
-  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
-  "tls1.0").
-`
-
-// Middleware function signature required by chi.Router.Use()
-type Middleware func(http.Handler) http.Handler
-
-// Options contains options for the http Server
-type Options struct {
-	ListenAddr         string        // Port to listen on
-	BaseURL            string        // prefix to strip from URLs
-	ServerReadTimeout  time.Duration // Timeout for server reading data
-	ServerWriteTimeout time.Duration // Timeout for server writing data
-	MaxHeaderBytes     int           // Maximum size of request header
-	SslCert            string        // Path to SSL PEM key (concatenation of certificate and CA certificate)
-	SslKey             string        // Path to SSL PEM Private key
-	SslCertBody        []byte        // SSL PEM key (concatenation of certificate and CA certificate) body, ignores SslCert
-	SslKeyBody         []byte        // SSL PEM Private key body, ignores SslKey
-	ClientCA           string        // Client certificate authority to verify clients with
-	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable.
-}
-
-// DefaultOpt is the default values used for Options
-var DefaultOpt = Options{
-	ListenAddr:         "127.0.0.1:8080",
-	ServerReadTimeout:  1 * time.Hour,
-	ServerWriteTimeout: 1 * time.Hour,
-	MaxHeaderBytes:     4096,
-	MinTLSVersion:      "tls1.0",
-}
-
-// Server interface of http server
-type Server interface {
-	Router() chi.Router
-	Route(pattern string, fn func(r chi.Router)) chi.Router
-	Mount(pattern string, h http.Handler)
-	Shutdown() error
-}
-
-type server struct {
-	addrs        []net.Addr
-	tlsAddrs     []net.Addr
-	listeners    []net.Listener
-	tlsListeners []net.Listener
-	httpServer   *http.Server
-	baseRouter   chi.Router
-	closing      *sync.WaitGroup
-	useSSL       bool
-}
-
-var (
-	defaultServer        *server
-	defaultServerOptions = DefaultOpt
-	defaultServerMutex   sync.Mutex
-)
-
-func useSSL(opt Options) bool {
-	return opt.SslKey != "" || len(opt.SslKeyBody) > 0
-}
-
-// NewServer instantiates a new http server using provided listeners and options
-// This function is provided if the default http server does not meet a services requirements and should not generally be used
-// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
-// tlsListeners are ignored if opt.SslKey is not provided
-func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, error) {
-	// Validate input
-	if len(listeners) == 0 && len(tlsListeners) == 0 {
-		return nil, errors.New("can't create server without listeners")
-	}
-
-	// Prepare TLS config
-	var tlsConfig *tls.Config
-
-	useSSL := useSSL(opt)
-	if (len(opt.SslCertBody) > 0) != (len(opt.SslKeyBody) > 0) {
-		err := errors.New("need both SslCertBody and SslKeyBody to use SSL")
-		log.Fatalf(err.Error())
-		return nil, err
-	}
-	if (opt.SslCert != "") != (opt.SslKey != "") {
-		err := errors.New("need both -cert and -key to use SSL")
-		log.Fatalf(err.Error())
-		return nil, err
-	}
-
-	if useSSL {
-		var cert tls.Certificate
-		var err error
-		if len(opt.SslCertBody) > 0 {
-			cert, err = tls.X509KeyPair(opt.SslCertBody, opt.SslKeyBody)
-		} else {
-			cert, err = tls.LoadX509KeyPair(opt.SslCert, opt.SslKey)
-		}
-		if err != nil {
-			log.Fatal(err)
-		}
-		var minTLSVersion uint16
-		switch opt.MinTLSVersion {
-		case "tls1.0":
-			minTLSVersion = tls.VersionTLS10
-		case "tls1.1":
-			minTLSVersion = tls.VersionTLS11
-		case "tls1.2":
-			minTLSVersion = tls.VersionTLS12
-		case "tls1.3":
-			minTLSVersion = tls.VersionTLS13
-		default:
-			err = errors.New("Invalid value for --min-tls-version")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		tlsConfig = &tls.Config{
-			MinVersion:   minTLSVersion,
-			Certificates: []tls.Certificate{cert},
-		}
-	} else if len(listeners) == 0 && len(tlsListeners) != 0 {
-		return nil, errors.New("no SslKey or non-tlsListeners")
-	}
-
-	if opt.ClientCA != "" {
-		if !useSSL {
-			err := errors.New("can't use --client-ca without --cert and --key")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		certpool := x509.NewCertPool()
-		pem, err := os.ReadFile(opt.ClientCA)
-		if err != nil {
-			log.Fatalf("Failed to read client certificate authority: %v", err)
-			return nil, err
-		}
-		if !certpool.AppendCertsFromPEM(pem) {
-			err := errors.New("can't parse client certificate authority")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		tlsConfig.ClientCAs = certpool
-		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
-	}
-
-	// Ignore passing "/" for BaseURL
-	opt.BaseURL = strings.Trim(opt.BaseURL, "/")
-	if opt.BaseURL != "" {
-		opt.BaseURL = "/" + opt.BaseURL
-	}
-
-	// Build base router
-	var router chi.Router = chi.NewRouter()
-	router.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) {
-		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
-	})
-	router.NotFound(func(w http.ResponseWriter, _ *http.Request) {
-		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
-	})
-
-	handler := router.(http.Handler)
-	if opt.BaseURL != "" {
-		handler = http.StripPrefix(opt.BaseURL, handler)
-	}
-
-	// Serve on listeners
-	httpServer := &http.Server{
-		Handler:           handler,
-		ReadTimeout:       opt.ServerReadTimeout,
-		WriteTimeout:      opt.ServerWriteTimeout,
-		MaxHeaderBytes:    opt.MaxHeaderBytes,
-		ReadHeaderTimeout: 10 * time.Second, // time to send the headers
-		IdleTimeout:       60 * time.Second, // time to keep idle connections open
-		TLSConfig:         tlsConfig,
-	}
-
-	addrs, tlsAddrs := make([]net.Addr, len(listeners)), make([]net.Addr, len(tlsListeners))
-
-	wg := &sync.WaitGroup{}
-
-	for i, l := range listeners {
-		addrs[i] = l.Addr()
-	}
-
-	if useSSL {
-		for i, l := range tlsListeners {
-			tlsAddrs[i] = l.Addr()
-		}
-	}
-
-	return &server{addrs, tlsAddrs, listeners, tlsListeners, httpServer, router, wg, useSSL}, nil
-}
-
-func (s *server) Serve() {
-	serve := func(l net.Listener, tls bool) {
-		defer s.closing.Done()
-		var err error
-		if tls {
-			err = s.httpServer.ServeTLS(l, "", "")
-		} else {
-			err = s.httpServer.Serve(l)
-		}
-		if err != http.ErrServerClosed && err != nil {
-			log.Fatalf(err.Error())
-		}
-	}
-
-	s.closing.Add(len(s.listeners))
-	for _, l := range s.listeners {
-		go serve(l, false)
-	}
-
-	if s.useSSL {
-		s.closing.Add(len(s.tlsListeners))
-		for _, l := range s.tlsListeners {
-			go serve(l, true)
-		}
-	}
-}
-
-// Wait blocks while the server is serving requests
-func (s *server) Wait() {
-	s.closing.Wait()
-}
-
-// Router returns the server base router
-func (s *server) Router() chi.Router {
-	return s.baseRouter
-}
-
-// Route mounts a sub-Router along a `pattern` string.
-func (s *server) Route(pattern string, fn func(r chi.Router)) chi.Router {
-	return s.baseRouter.Route(pattern, fn)
-}
-
-// Mount attaches another http.Handler along ./pattern/*
-func (s *server) Mount(pattern string, h http.Handler) {
-	s.baseRouter.Mount(pattern, h)
-}
-
-// Shutdown gracefully shuts down the server
-func (s *server) Shutdown() error {
-	if err := s.httpServer.Shutdown(context.Background()); err != nil {
-		return err
-	}
-	s.closing.Wait()
-	return nil
-}
-
-//---- Default HTTP server convenience functions ----
-
-// Router returns the server base router
-func Router() (chi.Router, error) {
-	if err := start(); err != nil {
-		return nil, err
-	}
-	return defaultServer.baseRouter, nil
-}
-
-// Route mounts a sub-Router along a `pattern` string.
-func Route(pattern string, fn func(r chi.Router)) (chi.Router, error) {
-	if err := start(); err != nil {
-		return nil, err
-	}
-	return defaultServer.Route(pattern, fn), nil
-}
-
-// Mount attaches another http.Handler along ./pattern/*
-func Mount(pattern string, h http.Handler) error {
-	if err := start(); err != nil {
-		return err
-	}
-	defaultServer.Mount(pattern, h)
-	return nil
-}
-
-// Restart or start the default http server using the default options and no handlers
-func Restart() error {
-	if e := Shutdown(); e != nil {
-		return e
-	}
-
-	return start()
-}
-
-// Wait blocks while the default http server is serving requests
-func Wait() {
-	defaultServer.Wait()
-}
-
-// Start the default server
-func start() error {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-
-	if defaultServer != nil {
-		// Server already started, do nothing
-		return nil
-	}
-
-	var err error
-	var l net.Listener
-	l, err = net.Listen("tcp", defaultServerOptions.ListenAddr)
-	if err != nil {
-		return err
-	}
-
-	var s Server
-	if useSSL(defaultServerOptions) {
-		s, err = NewServer([]net.Listener{}, []net.Listener{l}, defaultServerOptions)
-	} else {
-		s, err = NewServer([]net.Listener{l}, []net.Listener{}, defaultServerOptions)
-	}
-	if err != nil {
-		return err
-	}
-	defaultServer = s.(*server)
-	defaultServer.Serve()
-	return nil
-}
-
-// Shutdown gracefully shuts down the default http server
-func Shutdown() error {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	if defaultServer != nil {
-		s := defaultServer
-		defaultServer = nil
-		return s.Shutdown()
-	}
-	return nil
-}
-
-// GetOptions thread safe getter for the default server options
-func GetOptions() Options {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	return defaultServerOptions
-}
-
-// SetOptions thread safe setter for the default server options
-func SetOptions(opt Options) {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	defaultServerOptions = opt
-}
-
-//---- Utility functions ----
-
-// URL of default http server
-func URL() string {
-	if defaultServer == nil {
-		panic("Server not running")
-	}
-	for _, a := range defaultServer.addrs {
-		return fmt.Sprintf("http://%s%s/", a.String(), defaultServerOptions.BaseURL)
-	}
-	for _, a := range defaultServer.tlsAddrs {
-		return fmt.Sprintf("https://%s%s/", a.String(), defaultServerOptions.BaseURL)
-	}
-	panic("Server is running with no listener")
-}
-
-//---- Command line flags ----
-
-// AddFlagsPrefix adds flags for the httplib
-func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
-	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to")
-	flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, prefix+"server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data")
-	flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, prefix+"server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data")
-	flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, prefix+"max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header")
-	flags.StringVarP(flagSet, &Opt.SslCert, prefix+"cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)")
-	flags.StringVarP(flagSet, &Opt.SslKey, prefix+"key", "", Opt.SslKey, "SSL PEM Private key")
-	flags.StringVarP(flagSet, &Opt.ClientCA, prefix+"client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with")
-	flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root")
-	flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable")
-
-}
-
-// AddFlags adds flags for the httplib
-func AddFlags(flagSet *pflag.FlagSet) {
-	AddFlagsPrefix(flagSet, "", &defaultServerOptions)
-}

lib/http/http_test.go

@@ -1,515 +0,0 @@
-package http
-
-import (
-	"crypto/tls"
-	"net"
-	"net/http"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	"golang.org/x/net/nettest"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetOptions(t *testing.T) {
-	tests := []struct {
-		name string
-		want Options
-	}{
-		{name: "basic", want: defaultServerOptions},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := GetOptions(); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("GetOptions() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestMount(t *testing.T) {
-	type args struct {
-		pattern string
-		h       http.Handler
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{name: "basic", args: args{
-			pattern: "/",
-			h:       http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}),
-		}, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.wantErr {
-				require.Error(t, Mount(tt.args.pattern, tt.args.h))
-			} else {
-				require.NoError(t, Mount(tt.args.pattern, tt.args.h))
-			}
-			assert.NotNil(t, defaultServer)
-			assert.True(t, defaultServer.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to match route after registering")
-		})
-		if err := Shutdown(); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestNewServer(t *testing.T) {
-	type args struct {
-		listeners    []net.Listener
-		tlsListeners []net.Listener
-		opt          Options
-	}
-	listener, err := nettest.NewLocalListener("tcp")
-	if err != nil {
-		t.Fatal(err)
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{name: "default http", args: args{
-			listeners:    []net.Listener{listener},
-			tlsListeners: []net.Listener{},
-			opt:          defaultServerOptions,
-		}, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewServer(tt.args.listeners, tt.args.tlsListeners, tt.args.opt)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("NewServer() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			s, ok := got.(*server)
-			require.True(t, ok, "NewServer returned unexpected type")
-			if len(tt.args.listeners) > 0 {
-				assert.Equal(t, listener.Addr(), s.addrs[0])
-			} else {
-				assert.Empty(t, s.addrs)
-			}
-			if len(tt.args.tlsListeners) > 0 {
-				assert.Equal(t, listener.Addr(), s.tlsAddrs[0])
-			} else {
-				assert.Empty(t, s.tlsAddrs)
-			}
-			if tt.args.opt.BaseURL != "" {
-				assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter")
-			} else {
-				assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter")
-			}
-			if useSSL(tt.args.opt) {
-				assert.NotNil(t, s.httpServer.TLSConfig, "missing SSL config")
-			} else {
-				assert.Nil(t, s.httpServer.TLSConfig, "unexpectedly has SSL config")
-			}
-		})
-	}
-}
-
-func TestRestart(t *testing.T) {
-	tests := []struct {
-		name    string
-		started bool
-		wantErr bool
-	}{
-		{name: "started", started: true, wantErr: false},
-		{name: "stopped", started: false, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.started {
-				require.NoError(t, Restart()) // Call it twice basically
-			} else {
-				require.NoError(t, Shutdown())
-			}
-			current := defaultServer
-			if err := Restart(); (err != nil) != tt.wantErr {
-				t.Errorf("Restart() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.NotNil(t, defaultServer, "failed to start default server")
-			assert.NotSame(t, current, defaultServer, "same server instance as before restart")
-		})
-	}
-}
-
-func TestRoute(t *testing.T) {
-	type args struct {
-		pattern string
-		fn      func(r chi.Router)
-	}
-	tests := []struct {
-		name string
-		args args
-		test func(t *testing.T, r chi.Router)
-	}{
-		{
-			name: "basic",
-			args: args{
-				pattern: "/basic",
-				fn:      func(r chi.Router) {},
-			},
-			test: func(t *testing.T, r chi.Router) {
-				require.Len(t, r.Routes(), 1)
-				assert.Equal(t, r.Routes()[0].Pattern, "/basic/*")
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			require.NoError(t, Restart())
-			_, err := Route(tt.args.pattern, tt.args.fn)
-			require.NoError(t, err)
-			tt.test(t, defaultServer.baseRouter)
-		})
-
-		if err := Shutdown(); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestSetOptions(t *testing.T) {
-	type args struct {
-		opt Options
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			name: "basic",
-			args: args{opt: Options{
-				ListenAddr:         "127.0.0.1:9999",
-				BaseURL:            "/basic",
-				ServerReadTimeout:  1,
-				ServerWriteTimeout: 1,
-				MaxHeaderBytes:     1,
-			}},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			SetOptions(tt.args.opt)
-			require.Equal(t, tt.args.opt, defaultServerOptions)
-			require.NoError(t, Restart())
-			if useSSL(tt.args.opt) {
-				assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.tlsAddrs[0].String())
-			} else {
-				assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.addrs[0].String())
-			}
-			assert.Equal(t, tt.args.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout)
-			assert.Equal(t, tt.args.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout)
-			assert.Equal(t, tt.args.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes)
-			if tt.args.opt.BaseURL != "" && tt.args.opt.BaseURL != "/" {
-				assert.NotSame(t, defaultServer.httpServer.Handler, defaultServer.baseRouter, "BaseURL ignored")
-			}
-		})
-		SetOptions(DefaultOpt)
-	}
-}
-
-func TestShutdown(t *testing.T) {
-	tests := []struct {
-		name    string
-		started bool
-		wantErr bool
-	}{
-		{name: "started", started: true, wantErr: false},
-		{name: "stopped", started: false, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.started {
-				require.NoError(t, Restart())
-			} else {
-				require.NoError(t, Shutdown()) // Call it twice basically
-			}
-			if err := Shutdown(); (err != nil) != tt.wantErr {
-				t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.Nil(t, defaultServer, "default server not deleted")
-		})
-	}
-}
-
-func TestURL(t *testing.T) {
-	tests := []struct {
-		name string
-		want string
-	}{
-		{name: "basic", want: "http://127.0.0.1:8080/"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			require.NoError(t, Restart())
-			if got := URL(); got != tt.want {
-				t.Errorf("URL() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_server_Mount(t *testing.T) {
-	type args struct {
-		pattern string
-		h       http.Handler
-	}
-	tests := []struct {
-		name string
-		args args
-		opt  Options
-	}{
-		{name: "basic", args: args{
-			pattern: "/",
-			h:       http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}),
-		}, opt: defaultServerOptions},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			s.Mount(tt.args.pattern, tt.args.h)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			assert.NotNil(t, srv)
-			assert.True(t, srv.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to Match() route after registering")
-		})
-	}
-}
-
-func Test_server_Route(t *testing.T) {
-	type args struct {
-		pattern string
-		fn      func(r chi.Router)
-	}
-	tests := []struct {
-		name string
-		args args
-		opt  Options
-		test func(t *testing.T, r chi.Router)
-	}{
-		{
-			name: "basic",
-			args: args{
-				pattern: "/basic",
-				fn: func(r chi.Router) {
-
-				},
-			},
-			test: func(t *testing.T, r chi.Router) {
-				require.Len(t, r.Routes(), 1)
-				assert.Equal(t, r.Routes()[0].Pattern, "/basic/*")
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			s.Route(tt.args.pattern, tt.args.fn)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			assert.NotNil(t, srv)
-			tt.test(t, srv.baseRouter)
-		})
-	}
-}
-
-func Test_server_Shutdown(t *testing.T) {
-	tests := []struct {
-		name    string
-		opt     Options
-		wantErr bool
-	}{
-		{
-			name:    "basic",
-			opt:     defaultServerOptions,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			if err := s.Shutdown(); (err != nil) != tt.wantErr {
-				t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.EqualError(t, srv.httpServer.Serve(listener), http.ErrServerClosed.Error())
-		})
-	}
-}
-
-func Test_start(t *testing.T) {
-	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	sslServerOptions := defaultServerOptions
-	sslServerOptions.SslCertBody = []byte(`-----BEGIN CERTIFICATE-----
-MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw
-DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow
-EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d
-7VNhbWvZLWPuj/RtHFjvtJBEwOkhbN/BnnE8rnZR8+sbwnc/KhCk3FhnpHZnQz7B
-5aETbbIgmuvewdjvSBSjYzBhMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggr
-BgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdEQQiMCCCDmxvY2FsaG9zdDo1
-NDUzgg4xMjcuMC4wLjE6NTQ1MzAKBggqhkjOPQQDAgNIADBFAiEA2zpJEPQyz6/l
-Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc
-6MF9+Yw1Yy0t
------END CERTIFICATE-----`)
-	sslServerOptions.SslKeyBody = []byte(`-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49
-AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q
-EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
------END EC PRIVATE KEY-----`)
-	tests := []struct {
-		name    string
-		opt     Options
-		ssl     bool
-		wantErr bool
-	}{
-		{
-			name:    "basic",
-			opt:     defaultServerOptions,
-			wantErr: false,
-		},
-		{
-			name:    "ssl",
-			opt:     sslServerOptions,
-			ssl:     true,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			defer func() {
-				err := Shutdown()
-				if err != nil {
-					t.Fatal("couldn't shutdown server")
-				}
-			}()
-			SetOptions(tt.opt)
-			if err := start(); (err != nil) != tt.wantErr {
-				t.Errorf("start() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			s := defaultServer
-			router := s.Router()
-			router.Head("/", func(writer http.ResponseWriter, request *http.Request) {
-				writer.WriteHeader(201)
-			})
-			testURL := URL()
-			if tt.ssl {
-				assert.True(t, useSSL(tt.opt))
-				assert.Equal(t, tt.opt.ListenAddr, s.tlsAddrs[0].String())
-				assert.True(t, strings.HasPrefix(testURL, "https://"))
-			} else {
-				assert.True(t, strings.HasPrefix(testURL, "http://"))
-				assert.Equal(t, tt.opt.ListenAddr, s.addrs[0].String())
-			}
-
-			// try to connect to the test server
-			pause := time.Millisecond
-			for i := 0; i < 10; i++ {
-				resp, err := http.Head(testURL)
-				if err == nil {
-					_ = resp.Body.Close()
-					return
-				}
-				// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
-				time.Sleep(pause)
-				pause *= 2
-			}
-			t.Fatal("couldn't connect to server")
-
-			/* accessing s.httpServer.* can't be done synchronously and is a race condition
-			assert.Equal(t, tt.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout)
-			assert.Equal(t, tt.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout)
-			assert.Equal(t, tt.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes)
-			if tt.opt.BaseURL != "" && tt.opt.BaseURL != "/" {
-				assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter")
-			} else {
-				assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter")
-			}
-			if useSSL(tt.opt) {
-				require.NotNil(t, s.httpServer.TLSConfig, "missing SSL config")
-				assert.NotEmpty(t, s.httpServer.TLSConfig.Certificates, "missing SSL config")
-			} else if s.httpServer.TLSConfig != nil {
-				assert.Empty(t, s.httpServer.TLSConfig.Certificates, "unexpectedly has SSL config")
-			}
-			*/
-		})
-	}
-}
-
-func Test_useSSL(t *testing.T) {
-	type args struct {
-		opt Options
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:  "",
-				SslKey:   "",
-				ClientCA: "",
-			}},
-			want: false,
-		},
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:  "",
-				SslKey:   "test",
-				ClientCA: "",
-			}},
-			want: true,
-		},
-		{
-			name: "body",
-			args: args{opt: Options{
-				SslCert:    "",
-				SslKey:     "",
-				SslKeyBody: []byte(`test`),
-				ClientCA:   "",
-			}},
-			want: true,
-		},
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:       "",
-				SslKey:        "test",
-				ClientCA:      "",
-				MinTLSVersion: "tls1.2",
-			}},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := useSSL(tt.args.opt); got != tt.want {
-				t.Errorf("useSSL() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}

lib/http/middleware.go

@@ -0,0 +1,171 @@
+package http
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	goauth "github.com/abbot/go-http-auth"
+	"github.com/rclone/rclone/fs"
+)
+
+// parseAuthorization parses the Authorization header into user, pass
+// it returns a boolean as to whether the parse was successful
+func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
+	authHeader := r.Header.Get("Authorization")
+	if authHeader != "" {
+		s := strings.SplitN(authHeader, " ", 2)
+		if len(s) == 2 && s[0] == "Basic" {
+			b, err := base64.StdEncoding.DecodeString(s[1])
+			if err == nil {
+				parts := strings.SplitN(string(b), ":", 2)
+				user = parts[0]
+				if len(parts) > 1 {
+					pass = parts[1]
+					ok = true
+				}
+			}
+		}
+	}
+	return
+}
+
+type LoggedBasicAuth struct {
+	goauth.BasicAuth
+}
+
+// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests
+func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string {
+	username := a.BasicAuth.CheckAuth(r)
+	if username == "" {
+		user, _, _ := parseAuthorization(r)
+		fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user)
+	}
+	return username
+}
+
+// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator
+func NewLoggedBasicAuthenticator(realm string, secrets goauth.SecretProvider) *LoggedBasicAuth {
+	return &LoggedBasicAuth{BasicAuth: goauth.BasicAuth{Realm: realm, Secrets: secrets}}
+}
+
+// Helper to generate required interface for middleware
+func basicAuth(authenticator *LoggedBasicAuth) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip auth for unix socket
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			username := authenticator.CheckAuth(r)
+			if username == "" {
+				authenticator.RequireAuth(w, r)
+				return
+			}
+			ctx := context.WithValue(r.Context(), ctxKeyUser, username)
+			next.ServeHTTP(w, r.WithContext(ctx))
+		})
+	}
+}
+
+// MiddlewareAuthHtpasswd instantiates middleware that authenticates against the passed htpasswd file
+func MiddlewareAuthHtpasswd(path, realm string) Middleware {
+	fs.Infof(nil, "Using %q as htpasswd storage", path)
+	secretProvider := goauth.HtpasswdFileProvider(path)
+	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
+	return basicAuth(authenticator)
+}
+
+// MiddlewareAuthBasic instantiates middleware that authenticates for a single user
+func MiddlewareAuthBasic(user, pass, realm, salt string) Middleware {
+	fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user)
+	pass = string(goauth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$")))
+	secretProvider := func(u, r string) string {
+		if user == u {
+			return pass
+		}
+		return ""
+	}
+	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
+	return basicAuth(authenticator)
+}
+
+// MiddlewareAuthCustom instantiates middleware that authenticates using a custom function
+func MiddlewareAuthCustom(fn CustomAuthFn, realm string) Middleware {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip auth for unix socket
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			user, pass, ok := parseAuthorization(r)
+			if !ok {
+				code := http.StatusUnauthorized
+				w.Header().Set("Content-Type", "text/plain")
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm=%q, charset="UTF-8"`, realm))
+				http.Error(w, http.StatusText(code), code)
+				return
+			}
+
+			value, err := fn(user, pass)
+			if err != nil {
+				fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err)
+				goauth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting
+				return
+			}
+
+			if value != nil {
+				r = r.WithContext(context.WithValue(r.Context(), ctxKeyAuth, value))
+			}
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+var onlyOnceWarningAllowOrigin sync.Once
+
+// MiddlewareCORS instantiates middleware that handles basic CORS protections for rcd
+func MiddlewareCORS(allowOrigin string) Middleware {
+	onlyOnceWarningAllowOrigin.Do(func() {
+		if allowOrigin == "*" {
+			fs.Logf(nil, "Warning: Allow origin set to *. This can cause serious security problems.")
+		}
+	})
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip cors for unix sockets
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			if allowOrigin != "" {
+				w.Header().Add("Access-Control-Allow-Origin", allowOrigin)
+			} else {
+				w.Header().Add("Access-Control-Allow-Origin", PublicURL(r))
+			}
+
+			// echo back access control headers client needs
+			w.Header().Add("Access-Control-Request-Method", "POST, OPTIONS, GET, HEAD")
+			w.Header().Add("Access-Control-Allow-Headers", "authorization, Content-Type")
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// MiddlewareStripPrefix instantiates middleware that removes the BaseURL from the path
+func MiddlewareStripPrefix(prefix string) Middleware {
+	return func(next http.Handler) http.Handler {
+		return http.StripPrefix(prefix, next)
+	}
+}

lib/http/middleware_test.go

@@ -0,0 +1,231 @@
+package http
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMiddlewareAuth(t *testing.T) {
+	servers := []struct {
+		name string
+		http HTTPConfig
+		auth AuthConfig
+		user string
+		pass string
+	}{
+		{
+			name: "Basic",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:     "test",
+				BasicUser: "test",
+				BasicPass: "test",
+			},
+			user: "test",
+			pass: "test",
+		},
+		{
+			name: "Htpasswd/MD5",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "md5",
+			pass: "md5",
+		},
+		{
+			name: "Htpasswd/SHA",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "sha",
+			pass: "sha",
+		},
+		{
+			name: "Htpasswd/Bcrypt",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "bcrypt",
+			pass: "bcrypt",
+		},
+		{
+			name: "Custom",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm: "test",
+				CustomAuthFn: func(user, pass string) (value interface{}, err error) {
+					if user == "custom" && pass == "custom" {
+						return true, nil
+					}
+					return nil, errors.New("invalid credentials")
+				},
+			},
+			user: "custom",
+			pass: "custom",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http), WithAuth(ss.auth))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("secret-page")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+
+			t.Run("NoCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using no creds should return unauthorized")
+
+				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+			})
+
+			t.Run("BadCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				req.SetBasicAuth(ss.user+"BAD", ss.pass+"BAD")
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using bad creds should return unauthorized")
+
+				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+			})
+
+			t.Run("GoodCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				req.SetBasicAuth(ss.user, ss.pass)
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusOK, resp.StatusCode, "using good creds should return ok")
+
+				testExpectRespBody(t, resp, expected)
+			})
+		})
+	}
+}
+
+var _testCORSHeaderKeys = []string{
+	"Access-Control-Allow-Origin",
+	"Access-Control-Request-Method",
+	"Access-Control-Allow-Headers",
+}
+
+func TestMiddlewareCORS(t *testing.T) {
+	servers := []struct {
+		name   string
+		http   HTTPConfig
+		origin string
+	}{
+		{
+			name: "EmptyOrigin",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			origin: "",
+		},
+		{
+			name: "CustomOrigin",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			origin: "http://test.rclone.org",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			s.Router().Use(MiddlewareCORS(ss.origin))
+
+			expected := []byte("data")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+
+			client := &http.Client{}
+			req, err := http.NewRequest("GET", url, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+
+			for _, key := range _testCORSHeaderKeys {
+				require.Contains(t, resp.Header, key, "CORS headers should be sent")
+			}
+
+			expectedOrigin := url
+			if ss.origin != "" {
+				expectedOrigin = ss.origin
+			}
+			require.Equal(t, expectedOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "allow origin should match")
+		})
+	}
+}

lib/http/serve/dir_test.go

@@ -3,7 +3,7 @@ package serve
 import (
 	"errors"
 	"html/template"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -94,7 +94,7 @@ func TestError(t *testing.T) {
 	Error("potato", w, "sausage", err)
 	resp := w.Result()
 	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "sausage.\n", string(body))
 }
 
@@ -108,7 +108,7 @@ func TestServe(t *testing.T) {
 	d.Serve(w, r)
 	resp := w.Result()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, `<!DOCTYPE html>
 <html lang="en">
 <head>

lib/http/serve/serve_test.go

@@ -1,7 +1,7 @@
 package serve
 
 import (
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -17,7 +17,7 @@ func TestObjectBadMethod(t *testing.T) {
 	Object(w, r, o)
 	resp := w.Result()
 	assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "Method Not Allowed\n", string(body))
 }
 
@@ -30,7 +30,7 @@ func TestObjectHEAD(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "", string(body))
 }
 
@@ -43,7 +43,7 @@ func TestObjectGET(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "hello", string(body))
 }
 
@@ -58,7 +58,7 @@ func TestObjectRange(t *testing.T) {
 	assert.Equal(t, "3", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
 	assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "345", string(body))
 }
 
@@ -71,6 +71,6 @@ func TestObjectBadRange(t *testing.T) {
 	resp := w.Result()
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	assert.Equal(t, "10", resp.Header.Get("Content-Length"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }

lib/http/server.go

@@ -0,0 +1,430 @@
+// Package http provides a registration interface for http services
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"html/template"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Help contains text describing the http server to add to the command
+// help.
+var Help = `
+### Server options
+
+Use ` + "`--addr`" + ` to specify which IP address and port the server should
+listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all
+IPs.  By default it only listens on localhost.  You can use port
+:0 to let the OS choose an available port.
+
+If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address
+then using Authentication is advised - see the next section for info.
+
+You can use a unix socket by setting the url to ` + "`unix:///path/to/socket`" + `
+or just by using an absolute path name. Note that unix sockets bypass the
+authentication - this is expected to be done with file system permissions.
+
+` + "`--addr`" + ` may be repeated to listen on multiple IPs/ports/sockets.
+
+` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to
+control the timeouts on the server.  Note that this is the total time
+for a transfer.
+
+` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will
+accept in the HTTP header.
+
+` + "`--baseurl`" + ` controls the URL prefix that rclone serves from.  By default
+rclone will serve from the root.  If you used ` + "`--baseurl \"/rclone\"`" + ` then
+rclone would serve from a URL starting with "/rclone/".  This is
+useful if you wish to proxy rclone serve.  Rclone automatically
+inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `,
+` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated
+identically.
+
+#### TLS (SSL)
+
+By default this will serve over http.  If you want you can serve over
+https.  You will need to supply the ` + "`--cert` and `--key`" + ` flags.
+If you wish to do client side certificate validation then you will need to
+supply ` + "`--client-ca`" + ` also.
+
+` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation
+of that with the CA certificate.  ` + "`--key`" + ` should be the PEM encoded
+private key and ` + "`--client-ca`" + ` should be the PEM encoded client
+certificate authority certificate.
+
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+`
+
+// Middleware function signature required by chi.Router.Use()
+type Middleware func(http.Handler) http.Handler
+
+// HTTPConfig contains options for the http Server
+type HTTPConfig struct {
+	ListenAddr         []string      // Port to listen on
+	BaseURL            string        // prefix to strip from URLs
+	ServerReadTimeout  time.Duration // Timeout for server reading data
+	ServerWriteTimeout time.Duration // Timeout for server writing data
+	MaxHeaderBytes     int           // Maximum size of request header
+	TLSCert            string        // Path to TLS PEM key (concatenation of certificate and CA certificate)
+	TLSKey             string        // Path to TLS PEM Private key
+	TLSCertBody        []byte        // TLS PEM key (concatenation of certificate and CA certificate) body, ignores TLSCert
+	TLSKeyBody         []byte        // TLS PEM Private key body, ignores TLSKey
+	ClientCA           string        // Client certificate authority to verify clients with
+	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable.
+	Template           string
+}
+
+// AddFlagsPrefix adds flags for the httplib
+func (cfg *HTTPConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringArrayVarP(flagSet, &cfg.ListenAddr, prefix+"addr", "", cfg.ListenAddr, "IPaddress:Port or :Port to bind server to")
+	flags.DurationVarP(flagSet, &cfg.ServerReadTimeout, prefix+"server-read-timeout", "", cfg.ServerReadTimeout, "Timeout for server reading data")
+	flags.DurationVarP(flagSet, &cfg.ServerWriteTimeout, prefix+"server-write-timeout", "", cfg.ServerWriteTimeout, "Timeout for server writing data")
+	flags.IntVarP(flagSet, &cfg.MaxHeaderBytes, prefix+"max-header-bytes", "", cfg.MaxHeaderBytes, "Maximum size of request header")
+	flags.StringVarP(flagSet, &cfg.TLSCert, prefix+"cert", "", cfg.TLSCert, "TLS PEM key (concatenation of certificate and CA certificate)")
+	flags.StringVarP(flagSet, &cfg.TLSKey, prefix+"key", "", cfg.TLSKey, "TLS PEM Private key")
+	flags.StringVarP(flagSet, &cfg.ClientCA, prefix+"client-ca", "", cfg.ClientCA, "Client certificate authority to verify clients with")
+	flags.StringVarP(flagSet, &cfg.BaseURL, prefix+"baseurl", "", cfg.BaseURL, "Prefix for URLs - leave blank for root")
+	flags.StringVarP(flagSet, &cfg.MinTLSVersion, prefix+"min-tls-version", "", cfg.MinTLSVersion, "Minimum TLS version that is acceptable")
+}
+
+// AddHTTPFlagsPrefix adds flags for the httplib
+func AddHTTPFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *HTTPConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultHTTPCfg is the default values used for Config
+func DefaultHTTPCfg() HTTPConfig {
+	return HTTPConfig{
+		ListenAddr:         []string{"127.0.0.1:8080"},
+		ServerReadTimeout:  1 * time.Hour,
+		ServerWriteTimeout: 1 * time.Hour,
+		MaxHeaderBytes:     4096,
+		MinTLSVersion:      "tls1.0",
+	}
+}
+
+// Server interface of http server
+type Server interface {
+	Router() chi.Router
+	Serve()
+	Shutdown() error
+	HTMLTemplate() *template.Template
+	URLs() []string
+	Wait()
+}
+
+type instance struct {
+	url        string
+	listener   net.Listener
+	httpServer *http.Server
+}
+
+func (s instance) serve(wg *sync.WaitGroup) {
+	defer wg.Done()
+	err := s.httpServer.Serve(s.listener)
+	if err != http.ErrServerClosed && err != nil {
+		log.Printf("%s: unexpected error: %s", s.listener.Addr(), err.Error())
+	}
+}
+
+type server struct {
+	wg           sync.WaitGroup
+	mux          chi.Router
+	tlsConfig    *tls.Config
+	instances    []instance
+	auth         AuthConfig
+	cfg          HTTPConfig
+	template     *TemplateConfig
+	htmlTemplate *template.Template
+}
+
+// Option allows customizing the server
+type Option func(*server)
+
+// WithAuth option initializes the appropriate auth middleware
+func WithAuth(cfg AuthConfig) Option {
+	return func(s *server) {
+		s.auth = cfg
+	}
+}
+
+// WithConfig option applies the HTTPConfig to the server, overriding defaults
+func WithConfig(cfg HTTPConfig) Option {
+	return func(s *server) {
+		s.cfg = cfg
+	}
+}
+
+// WithTemplate option allows the parsing of a template
+func WithTemplate(cfg TemplateConfig) Option {
+	return func(s *server) {
+		s.template = &cfg
+	}
+}
+
+// NewServer instantiates a new http server using provided listeners and options
+// This function is provided if the default http server does not meet a services requirements and should not generally be used
+// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
+// tlsListeners are ignored if opt.TLSKey is not provided
+func NewServer(ctx context.Context, options ...Option) (*server, error) {
+	s := &server{
+		mux: chi.NewRouter(),
+		cfg: DefaultHTTPCfg(),
+	}
+
+	for _, opt := range options {
+		opt(s)
+	}
+
+	// Build base router
+	s.mux.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+	})
+	s.mux.NotFound(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+	})
+
+	// Ignore passing "/" for BaseURL
+	s.cfg.BaseURL = strings.Trim(s.cfg.BaseURL, "/")
+	if s.cfg.BaseURL != "" {
+		s.cfg.BaseURL = "/" + s.cfg.BaseURL
+		s.mux.Use(MiddlewareStripPrefix(s.cfg.BaseURL))
+	}
+
+	s.initAuth()
+
+	err := s.initTemplate()
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.initTLS()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, addr := range s.cfg.ListenAddr {
+		var url string
+		var network = "tcp"
+		var tlsCfg *tls.Config
+
+		if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) {
+			network = "unix"
+			addr = strings.TrimPrefix(addr, "unix://")
+			url = addr
+
+		} else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) {
+			tlsCfg = s.tlsConfig
+			addr = strings.TrimPrefix(addr, "tls://")
+		}
+
+		var listener net.Listener
+		if tlsCfg == nil {
+			listener, err = net.Listen(network, addr)
+		} else {
+			listener, err = tls.Listen(network, addr, tlsCfg)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		if network == "tcp" {
+			var secure string
+			if tlsCfg != nil {
+				secure = "s"
+			}
+			url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL)
+		}
+
+		ii := instance{
+			url:      url,
+			listener: listener,
+			httpServer: &http.Server{
+				Handler:           s.mux,
+				ReadTimeout:       s.cfg.ServerReadTimeout,
+				WriteTimeout:      s.cfg.ServerWriteTimeout,
+				MaxHeaderBytes:    s.cfg.MaxHeaderBytes,
+				ReadHeaderTimeout: 10 * time.Second, // time to send the headers
+				IdleTimeout:       60 * time.Second, // time to keep idle connections open
+				TLSConfig:         tlsCfg,
+				BaseContext:       NewBaseContext(ctx, url),
+			},
+		}
+
+		s.instances = append(s.instances, ii)
+	}
+
+	return s, nil
+}
+
+func (s *server) initAuth() {
+	if s.auth.CustomAuthFn != nil {
+		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm))
+		return
+	}
+
+	if s.auth.HtPasswd != "" {
+		s.mux.Use(MiddlewareAuthHtpasswd(s.auth.HtPasswd, s.auth.Realm))
+		return
+	}
+
+	if s.auth.BasicUser != "" {
+		s.mux.Use(MiddlewareAuthBasic(s.auth.BasicUser, s.auth.BasicPass, s.auth.Realm, s.auth.Salt))
+		return
+	}
+}
+
+func (s *server) initTemplate() error {
+	if s.template == nil {
+		return nil
+	}
+
+	var err error
+	s.htmlTemplate, err = GetTemplate(s.template.Path)
+	if err != nil {
+		err = fmt.Errorf("failed to get template: %w", err)
+	}
+
+	return err
+}
+
+var (
+	// hard coded errors, allowing for easier testing
+	ErrInvalidMinTLSVersion = errors.New("invalid value for --min-tls-version")
+	ErrTLSBodyMismatch      = errors.New("need both TLSCertBody and TLSKeyBody to use TLS")
+	ErrTLSFileMismatch      = errors.New("need both --cert and --key to use TLS")
+	ErrTLSParseCA           = errors.New("unable to parse client certificate authority")
+)
+
+func (s *server) initTLS() error {
+	if s.cfg.TLSCert == "" && s.cfg.TLSKey == "" && len(s.cfg.TLSCertBody) == 0 && len(s.cfg.TLSKeyBody) == 0 {
+		return nil
+	}
+
+	if (len(s.cfg.TLSCertBody) > 0) != (len(s.cfg.TLSKeyBody) > 0) {
+		return ErrTLSBodyMismatch
+	}
+
+	if (s.cfg.TLSCert != "") != (s.cfg.TLSKey != "") {
+		return ErrTLSFileMismatch
+	}
+
+	var cert tls.Certificate
+	var err error
+	if len(s.cfg.TLSCertBody) > 0 {
+		cert, err = tls.X509KeyPair(s.cfg.TLSCertBody, s.cfg.TLSKeyBody)
+	} else {
+		cert, err = tls.LoadX509KeyPair(s.cfg.TLSCert, s.cfg.TLSKey)
+	}
+	if err != nil {
+		return err
+	}
+
+	var minTLSVersion uint16
+	switch s.cfg.MinTLSVersion {
+	case "tls1.0":
+		minTLSVersion = tls.VersionTLS10
+	case "tls1.1":
+		minTLSVersion = tls.VersionTLS11
+	case "tls1.2":
+		minTLSVersion = tls.VersionTLS12
+	case "tls1.3":
+		minTLSVersion = tls.VersionTLS13
+	default:
+		return fmt.Errorf("%w: %s", ErrInvalidMinTLSVersion, s.cfg.MinTLSVersion)
+	}
+
+	s.tlsConfig = &tls.Config{
+		MinVersion:   minTLSVersion,
+		Certificates: []tls.Certificate{cert},
+	}
+
+	if s.cfg.ClientCA != "" {
+		// if !useTLS {
+		// 	err := errors.New("can't use --client-ca without --cert and --key")
+		// 	log.Fatalf(err.Error())
+		// }
+		certpool := x509.NewCertPool()
+		pem, err := os.ReadFile(s.cfg.ClientCA)
+		if err != nil {
+			return err
+		}
+
+		if !certpool.AppendCertsFromPEM(pem) {
+			return ErrTLSParseCA
+		}
+
+		s.tlsConfig.ClientCAs = certpool
+		s.tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return nil
+}
+
+// Serve starts the HTTP server on each listener
+func (s *server) Serve() {
+	s.wg.Add(len(s.instances))
+	for _, ii := range s.instances {
+		// TODO: decide how/when to log listening url
+		// log.Printf("listening on %s", ii.url)
+		go ii.serve(&s.wg)
+	}
+}
+
+// Wait blocks while the server is serving requests
+func (s *server) Wait() {
+	s.wg.Wait()
+}
+
+// Router returns the server base router
+func (s *server) Router() chi.Router {
+	return s.mux
+}
+
+// Shutdown gracefully shuts down the server
+func (s *server) Shutdown() error {
+	ctx := context.Background()
+	for _, ii := range s.instances {
+		if err := ii.httpServer.Shutdown(ctx); err != nil {
+			log.Printf("error shutting down server: %s", err)
+			continue
+		}
+	}
+	s.wg.Wait()
+	return nil
+}
+
+// HTMLTemplate returns the parsed template, if WithTemplate option was passed.
+func (s *server) HTMLTemplate() *template.Template {
+	return s.htmlTemplate
+}
+
+// URLs returns all configured URLS
+func (s *server) URLs() []string {
+	var out []string
+	for _, ii := range s.instances {
+		if ii.listener.Addr().Network() == "unix" {
+			continue
+		}
+		out = append(out, ii.url)
+	}
+	return out
+}

lib/http/server_test.go

@@ -0,0 +1,444 @@
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func testEchoHandler(data []byte) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(data)
+	})
+}
+
+func testExpectRespBody(t *testing.T, resp *http.Response, expected []byte) {
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, expected, body)
+}
+
+func testGetServerURL(t *testing.T, s Server) string {
+	urls := s.URLs()
+	require.GreaterOrEqual(t, len(urls), 1, "server should return at least one url")
+	return urls[0]
+}
+
+func testNewHTTPClientUnix(path string) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", path)
+			},
+		},
+	}
+}
+
+func testReadTestdataFile(t *testing.T, path string) []byte {
+	data, err := os.ReadFile(filepath.Join("./testdata", path))
+	require.NoError(t, err, "")
+	return data
+}
+
+func TestNewServerUnix(t *testing.T) {
+	ctx := context.Background()
+
+	tempDir := t.TempDir()
+	path := filepath.Join(tempDir, "rclone.sock")
+
+	cfg := DefaultHTTPCfg()
+	cfg.ListenAddr = []string{path}
+
+	auth := AuthConfig{
+		BasicUser: "test",
+		BasicPass: "test",
+	}
+
+	s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth))
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, s.Shutdown())
+		_, err := os.Stat(path)
+		require.ErrorIs(t, err, os.ErrNotExist, "shutdown should remove socket")
+	}()
+
+	require.Empty(t, s.URLs(), "unix socket should not appear in URLs")
+
+	s.Router().Use(MiddlewareCORS(""))
+
+	expected := []byte("hello world")
+	s.Router().Mount("/", testEchoHandler(expected))
+	s.Serve()
+
+	client := testNewHTTPClientUnix(path)
+	req, err := http.NewRequest("GET", "http://unix", nil)
+	require.NoError(t, err)
+
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+
+	testExpectRespBody(t, resp, expected)
+
+	require.Equal(t, http.StatusOK, resp.StatusCode, "unix sockets should ignore auth")
+
+	for _, key := range _testCORSHeaderKeys {
+		require.NotContains(t, resp.Header, key, "unix sockets should not be sent CORS headers")
+	}
+}
+
+func TestNewServerHTTP(t *testing.T) {
+	ctx := context.Background()
+
+	cfg := DefaultHTTPCfg()
+	cfg.ListenAddr = []string{"127.0.0.1:0"}
+
+	auth := AuthConfig{
+		BasicUser: "test",
+		BasicPass: "test",
+	}
+
+	s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth))
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, s.Shutdown())
+	}()
+
+	url := testGetServerURL(t, s)
+	require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme")
+
+	expected := []byte("hello world")
+	s.Router().Mount("/", testEchoHandler(expected))
+	s.Serve()
+
+	t.Run("StatusUnauthorized", func(t *testing.T) {
+		client := &http.Client{}
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "no basic auth creds should return unauthorized")
+	})
+
+	t.Run("StatusOK", func(t *testing.T) {
+		client := &http.Client{}
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(t, err)
+
+		req.SetBasicAuth(auth.BasicUser, auth.BasicPass)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		require.Equal(t, http.StatusOK, resp.StatusCode, "using basic auth creds should return ok")
+
+		testExpectRespBody(t, resp, expected)
+	})
+}
+func TestNewServerBaseURL(t *testing.T) {
+	servers := []struct {
+		name   string
+		http   HTTPConfig
+		suffix string
+	}{
+		{
+			name: "Empty",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "",
+			},
+			suffix: "/",
+		},
+		{
+			name: "Single/NoTrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone",
+			},
+			suffix: "/rclone/",
+		},
+		{
+			name: "Single/TrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/",
+			},
+			suffix: "/rclone/",
+		},
+		{
+			name: "Multi/NoTrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/test/base/url",
+			},
+			suffix: "/rclone/test/base/url/",
+		},
+		{
+			name: "Multi/TrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/test/base/url/",
+			},
+			suffix: "/rclone/test/base/url/",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("data")
+			s.Router().Get("/", testEchoHandler(expected).ServeHTTP)
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+			require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme")
+			require.True(t, strings.HasSuffix(url, ss.suffix), "url should have the expected suffix")
+
+			client := &http.Client{}
+			req, err := http.NewRequest("GET", url, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			t.Log(url, resp.Request.URL)
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+		})
+	}
+}
+
+func TestNewServerTLS(t *testing.T) {
+	certBytes := testReadTestdataFile(t, "local.crt")
+	keyBytes := testReadTestdataFile(t, "local.key")
+
+	// TODO: generate a proper cert with SAN
+	// TODO: generate CA, test mTLS
+	// clientCert, err := tls.X509KeyPair(certBytes, keyBytes)
+	// require.NoError(t, err, "should be testing with a valid self signed certificate")
+
+	servers := []struct {
+		name    string
+		wantErr bool
+		err     error
+		http    HTTPConfig
+	}{
+		{
+			name: "FromFile/Valid",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/NoCert",
+			wantErr: true,
+			err:     ErrTLSFileMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/InvalidCert",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt.invalid",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/NoKey",
+			wantErr: true,
+			err:     ErrTLSFileMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/InvalidKey",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "./testdata/local.key.invalid",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name: "FromBody/Valid",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/NoCert",
+			wantErr: true,
+			err:     ErrTLSBodyMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   nil,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/InvalidCert",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   []byte("JUNK DATA"),
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/NoKey",
+			wantErr: true,
+			err:     ErrTLSBodyMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    nil,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/InvalidKey",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    []byte("JUNK DATA"),
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.1",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.1",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.2",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.2",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.3",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.3",
+			},
+		},
+		{
+			name:    "MinTLSVersion/Invalid",
+			wantErr: true,
+			err:     ErrInvalidMinTLSVersion,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls0.9",
+			},
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			if ss.wantErr == true {
+				if ss.err != nil {
+					require.ErrorIs(t, err, ss.err, "new server should return the expected error")
+				} else {
+					require.Error(t, err, "new server should return error for invalid TLS config")
+				}
+				return
+			}
+
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("secret-page")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+			require.True(t, strings.HasPrefix(url, "https://"), "url should have https scheme")
+
+			client := &http.Client{
+				Transport: &http.Transport{
+					DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+						dest := strings.TrimPrefix(url, "https://")
+						dest = strings.TrimSuffix(dest, "/")
+						return net.Dial("tcp", dest)
+					},
+					TLSClientConfig: &tls.Config{
+						// Certificates: []tls.Certificate{clientCert},
+						InsecureSkipVerify: true,
+					},
+				},
+			}
+			req, err := http.NewRequest("GET", "https://dev.rclone.org", nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+		})
+	}
+}

lib/http/template.go

@@ -0,0 +1,95 @@
+package http
+
+import (
+	"embed"
+	"html/template"
+	"os"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"github.com/rclone/rclone/fs/config/flags"
+)
+
+// TemplateHelp describes how to use a custom template
+var TemplateHelp = `
+#### Template
+
+` + "`--template`" + ` allows a user to specify a custom markup template for HTTP
+and WebDAV serve functions.  The server exports the following markup
+to be used within the template to server pages:
+
+| Parameter   | Description |
+| :---------- | :---------- |
+| .Name       | The full path of a file/directory. |
+| .Title      | Directory listing of .Name |
+| .Sort       | The current sort used.  This is changeable via ?sort= parameter |
+|             | Sort Options: namedirfirst,name,size,time (default namedirfirst) |
+| .Order      | The current ordering used.  This is changeable via ?order= parameter |
+|             | Order Options: asc,desc (default asc) |
+| .Query      | Currently unused. |
+| .Breadcrumb | Allows for creating a relative navigation |
+|-- .Link     | The relative to the root link of the Text. |
+|-- .Text     | The Name of the directory. |
+| .Entries    | Information about a specific file/directory. |
+|-- .URL      | The 'url' of an entry.  |
+|-- .Leaf     | Currently same as 'URL' but intended to be 'just' the name. |
+|-- .IsDir    | Boolean for if an entry is a directory or not. |
+|-- .Size     | Size in Bytes of the entry. |
+|-- .ModTime  | The UTC timestamp of an entry. |
+`
+
+// TemplateConfig for the templating functionality
+type TemplateConfig struct {
+	Path string
+}
+
+// AddFlagsPrefix for the templating functionality
+func (cfg *TemplateConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringVarP(flagSet, &cfg.Path, prefix+"template", "", cfg.Path, "User-specified template")
+}
+
+// AddTemplateFlagsPrefix for the templating functionality
+func AddTemplateFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *TemplateConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultTemplateCfg returns a new config which can be customized by command line flags
+func DefaultTemplateCfg() TemplateConfig {
+	return TemplateConfig{}
+}
+
+// AfterEpoch returns the time since the epoch for the given time
+func AfterEpoch(t time.Time) bool {
+	return t.After(time.Time{})
+}
+
+// Assets holds the embedded filesystem for the default template
+//
+//go:embed templates
+var Assets embed.FS
+
+// GetTemplate returns the HTML template for serving directories via HTTP/WebDAV
+func GetTemplate(tmpl string) (*template.Template, error) {
+	var readFile = os.ReadFile
+	if tmpl == "" {
+		tmpl = "templates/index.html"
+		readFile = Assets.ReadFile
+	}
+
+	data, err := readFile(tmpl)
+	if err != nil {
+		return nil, err
+	}
+
+	funcMap := template.FuncMap{
+		"afterEpoch": AfterEpoch,
+	}
+
+	tpl, err := template.New("index").Funcs(funcMap).Parse(string(data))
+	if err != nil {
+		return nil, err
+	}
+
+	return tpl, nil
+}

lib/http/templates/index.html

@@ -0,0 +1,389 @@
+<!-- 
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License. 
+Modifications:  Adapted to rclone markup -->
+
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>{{html .Name}}</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<meta name="google" content="notranslate">
+<style>
+* { padding: 0; margin: 0; }
+body {
+	font-family: sans-serif;
+	text-rendering: optimizespeed;
+	background-color: #ffffff;
+}
+a {
+	color: #006ed3;
+	text-decoration: none;
+}
+a:hover,
+h1 a:hover {
+	color: #319cff;
+}
+header,
+#summary {
+	padding-left: 5%;
+	padding-right: 5%;
+}
+th:first-child,
+td:first-child {
+	width: 1%; /* tighter for mobile */
+}
+th:last-child,
+td:last-child {
+	width: 1%; /* tighter for mobile */
+}
+header {
+	padding-top: 25px;
+	padding-bottom: 15px;
+	background-color: #f2f2f2;
+}
+h1 {
+	font-size: 20px;
+	font-weight: normal;
+	white-space: nowrap;
+	overflow-x: hidden;
+	text-overflow: ellipsis;
+	color: #999;
+}
+h1 a {
+	color: #000;
+	margin: 0 4px;
+}
+h1 a:hover {
+	text-decoration: underline;
+}
+h1 a:first-child {
+	margin: 0;
+}
+main {
+	display: block;
+}
+.meta {
+	font-size: 12px;
+	font-family: Verdana, sans-serif;
+	border-bottom: 1px solid #9C9C9C;
+	padding-top: 10px;
+	padding-bottom: 10px;
+}
+.meta-item {
+	margin-right: 1em;
+}
+#filter {
+	padding: 4px;
+	border: 1px solid #CCC;
+}
+table {
+	width: 100%;
+	border-collapse: collapse;
+}
+tr {
+	border-bottom: 1px dashed #dadada;
+}
+tbody tr:hover {
+	background-color: #ffffec;
+}
+th,
+td {
+	text-align: left;
+	padding: 10px 2px; /* Changed from 0 to 2 */
+}
+th {
+	padding-top: 10px;
+	padding-bottom: 10px;
+	font-size: 16px;
+	white-space: nowrap;
+}
+th a {
+	color: black;
+}
+th svg {
+	vertical-align: middle;
+}
+td {
+	white-space: nowrap;
+	font-size: 14px;
+}
+td:nth-child(2) {
+	width: 80%;
+}
+td:nth-child(3) {
+	padding: 0 20px 0 20px;
+}
+th:nth-child(4),
+td:nth-child(4) {
+	text-align: right;
+}
+td:nth-child(2) svg {
+	position: absolute;
+}
+td .name,
+td .goup {
+	margin-left: 1.75em;
+	word-break: break-all;
+	overflow-wrap: break-word;
+	white-space: pre-wrap;
+}
+.icon {
+	margin-right: 5px;
+}
+.icon.sort {
+	display: inline-block;
+	width: 1em;
+	height: 1em;
+	position: relative;
+	top: .2em;
+}
+.icon.sort .top {
+	position: absolute;
+	left: 0;
+	top: -1px;
+}
+.icon.sort .bottom {
+	position: absolute;
+	bottom: -1px;
+	left: 0;
+}
+footer {
+	padding: 40px 20px;
+	font-size: 12px;
+	text-align: center;
+}
+@media (max-width: 600px) {
+/*	.hideable {
+		display: none;
+	} removed to keep dates on mobile */
+	td:nth-child(2) {
+		width: auto;
+	}
+	th:nth-child(3),
+	td:nth-child(3) {
+		padding-right: 5%;
+		text-align: right;
+	}
+	h1 {
+		color: #000;
+	}
+	h1 a {
+		margin: 0;
+	}
+	#filter {
+		max-width: 100px;
+	}
+}
+</style>
+	</head>
+	<body onload='filter();toggle("order");changeSize()'>
+		<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" height="0" width="0" style="position: absolute;">
+			<defs>
+				<!-- Folder -->
+				<g id="folder" fill-rule="nonzero" fill="none">
+					<path d="M285.22 37.55h-142.6L110.9 0H31.7C14.25 0 0 16.9 0 37.55v75.1h316.92V75.1c0-20.65-14.26-37.55-31.7-37.55z" fill="#FFA000"/>
+					<path d="M285.22 36H31.7C14.25 36 0 50.28 0 67.74v158.7c0 17.47 14.26 31.75 31.7 31.75H285.2c17.44 0 31.7-14.3 31.7-31.75V67.75c0-17.47-14.26-31.75-31.7-31.75z" fill="#FFCA28"/>
+				</g>
+				<g id="folder-shortcut" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+					<g id="folder-shortcut-group" fill-rule="nonzero">
+						<g id="folder-shortcut-shape">
+							<path d="M285.224876,37.5486902 L142.612438,37.5486902 L110.920785,0 L31.6916529,0 C14.2612438,0 0,16.8969106 0,37.5486902 L0,112.646071 L316.916529,112.646071 L316.916529,75.0973805 C316.916529,54.4456008 302.655285,37.5486902 285.224876,37.5486902 Z" id="Shape" fill="#FFA000"></path>
+							<path d="M285.224876,36 L31.6916529,36 C14.2612438,36 0,50.2838568 0,67.7419039 L0,226.451424 C0,243.909471 14.2612438,258.193328 31.6916529,258.193328 L285.224876,258.193328 C302.655285,258.193328 316.916529,243.909471 316.916529,226.451424 L316.916529,67.7419039 C316.916529,50.2838568 302.655285,36 285.224876,36 Z" id="Shape" fill="#FFCA28"></path>
+						</g>
+						<path d="M126.154134,250.559184 C126.850974,251.883673 127.300549,253.006122 127.772602,254.106122 C128.469442,255.206122 128.919016,256.104082 129.638335,257.002041 C130.559962,258.326531 131.728855,259 133.100057,259 C134.493737,259 135.415364,258.55102 136.112204,257.67551 C136.809044,257.002041 137.258619,255.902041 137.258619,254.577551 C137.258619,253.904082 137.258619,252.804082 137.033832,251.457143 C136.786566,249.908163 136.561779,249.032653 136.561779,248.583673 C136.089726,242.814286 135.864939,237.920408 135.864939,233.273469 C135.864939,225.057143 136.786566,217.514286 138.180246,210.846939 C139.798713,204.202041 141.889234,198.634694 144.429328,193.763265 C147.216689,188.869388 150.678411,184.873469 154.836973,181.326531 C158.995535,177.779592 163.626149,174.883673 168.481552,172.661224 C173.336954,170.438776 179.113983,168.665306 185.587852,167.340816 C192.061722,166.218367 198.760378,165.342857 205.481514,164.669388 C212.18017,164.220408 219.598146,163.995918 228.162535,163.995918 L246.055591,163.995918 L246.055591,195.514286 C246.055591,197.736735 246.752431,199.510204 248.370899,201.059184 C250.214153,202.608163 252.079886,203.506122 254.372715,203.506122 C256.463236,203.506122 258.531277,202.608163 260.172223,201.059184 L326.102289,137.797959 C327.720757,136.24898 328.642384,134.47551 328.642384,132.253061 C328.642384,130.030612 327.720757,128.257143 326.102289,126.708163 L260.172223,63.4469388 C258.553756,61.8979592 256.463236,61 254.395194,61 C252.079886,61 250.236632,61.8979592 248.393377,63.4469388 C246.77491,64.9959184 246.07807,66.7693878 246.07807,68.9918367 L246.07807,100.510204 L228.162535,100.510204 C166.863084,100.510204 129.166282,117.167347 115.274437,150.459184 C110.666301,161.54898 108.350993,175.310204 108.350993,191.742857 C108.350993,205.279592 113.903236,223.912245 124.760454,247.438776 C125.00772,248.112245 125.457294,249.010204 126.154134,250.559184 Z" id="Shape" fill="#FFFFFF" transform="translate(218.496689, 160.000000) scale(-1, 1) translate(-218.496689, -160.000000) "></path>
+					</g>
+				</g>
+				<!-- File -->
+				<g id="file" stroke="#000" stroke-width="25" fill="#FFF" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round">
+					<path d="M13 24.12v274.76c0 6.16 5.87 11.12 13.17 11.12H239c7.3 0 13.17-4.96 13.17-11.12V136.15S132.6 13 128.37 13H26.17C18.87 13 13 17.96 13 24.12z"/>
+					<path d="M129.37 13L129 113.9c0 10.58 7.26 19.1 16.27 19.1H249L129.37 13z"/>
+				</g>
+				<g id="file-shortcut" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+					<g id="file-shortcut-group" transform="translate(13.000000, 13.000000)">
+						<g id="file-shortcut-shape" stroke="#000000" stroke-width="25" fill="#FFFFFF" stroke-linecap="round" stroke-linejoin="round">
+							<path d="M0,11.1214886 L0,285.878477 C0,292.039924 5.87498876,296.999983 13.1728373,296.999983 L225.997983,296.999983 C233.295974,296.999983 239.17082,292.039942 239.17082,285.878477 L239.17082,123.145388 C239.17082,123.145388 119.58541,2.84217094e-14 115.369423,2.84217094e-14 L13.1728576,2.84217094e-14 C5.87500907,-1.71479982e-05 0,4.96022995 0,11.1214886 Z" id="rect1171"></path>
+							<path d="M116.37005,0 L116,100.904964 C116,111.483663 123.258008,120 132.273377,120 L236,120 L116.37005,0 L116.37005,0 Z" id="rect1794"></path>
+						</g>
+						<path d="M47.803141,294.093878 C48.4999811,295.177551 48.9495553,296.095918 49.4216083,296.995918 C50.1184484,297.895918 50.5680227,298.630612 51.2873415,299.365306 C52.2089688,300.44898 53.3778619,301 54.7490634,301 C56.1427436,301 57.0643709,300.632653 57.761211,299.916327 C58.4580511,299.365306 58.9076254,298.465306 58.9076254,297.381633 C58.9076254,296.830612 58.9076254,295.930612 58.6828382,294.828571 C58.4355724,293.561224 58.2107852,292.844898 58.2107852,292.477551 C57.7387323,287.757143 57.5139451,283.753061 57.5139451,279.95102 C57.5139451,273.228571 58.4355724,267.057143 59.8292526,261.602041 C61.44772,256.165306 63.5382403,251.610204 66.0783349,247.62449 C68.8656954,243.620408 72.3274172,240.35102 76.4859792,237.44898 C80.6445412,234.546939 85.2751561,232.177551 90.1305582,230.359184 C94.9859603,228.540816 100.76299,227.089796 107.236859,226.006122 C113.710728,225.087755 120.409385,224.371429 127.13052,223.820408 C133.829177,223.453061 141.247152,223.269388 149.811542,223.269388 L167.704598,223.269388 L167.704598,249.057143 C167.704598,250.87551 168.401438,252.326531 170.019905,253.593878 C171.86316,254.861224 173.728893,255.595918 176.021722,255.595918 C178.112242,255.595918 180.180284,254.861224 181.82123,253.593878 L247.751296,201.834694 C249.369763,200.567347 250.291391,199.116327 250.291391,197.297959 C250.291391,195.479592 249.369763,194.028571 247.751296,192.761224 L181.82123,141.002041 C180.202763,139.734694 178.112242,139 176.044201,139 C173.728893,139 171.885639,139.734694 170.042384,141.002041 C168.423917,142.269388 167.727077,143.720408 167.727077,145.538776 L167.727077,171.326531 L149.811542,171.326531 C88.5120908,171.326531 50.8152886,184.955102 36.9234437,212.193878 C32.3153075,221.267347 30,232.526531 30,245.971429 C30,257.046939 35.5522422,272.291837 46.4094607,291.540816 C46.6567266,292.091837 47.1063009,292.826531 47.803141,294.093878 Z" id="Shape-Copy" fill="#000000" fill-rule="nonzero" transform="translate(140.145695, 220.000000) scale(-1, 1) translate(-140.145695, -220.000000) "></path>
+					</g>
+				</g>
+				<!-- Up arrow -->
+				<g id="up-arrow" transform="translate(-279.22 -208.12)">
+					<path transform="matrix(.22413 0 0 .12089 335.67 164.35)" stroke-width="0" d="m-194.17 412.01h-28.827-28.827l14.414-24.965 14.414-24.965 14.414 24.965z"/>
+				</g>
+				<!-- Down arrow -->
+				<g id="down-arrow" transform="translate(-279.22 -208.12)">
+					<path transform="matrix(.22413 0 0 -.12089 335.67 257.93)" stroke-width="0" d="m-194.17 412.01h-28.827-28.827l14.414-24.965 14.414-24.965 14.414 24.965z"/>
+				</g>
+			</defs>
+		</svg>
+		<header>
+			<h1>
+				{{range $i, $crumb := .Breadcrumb}}<a href="{{html $crumb.Link}}">{{html $crumb.Text}}</a>{{if ne $i 0}}/{{end}}{{end}}
+			</h1>
+		</header>
+		<main>
+			<div class="meta">
+				<div id="summary">
+					<span class="meta-item"><input type="text" placeholder="filter" id="filter" onkeyup='filter()'></span>
+				</div>
+			</div>
+			<div class="listing">
+				<table aria-describedby="summary">
+					<thead>
+					<tr>
+						<th></th>
+						<th>
+							<a href="?sort=namedirfirst&order=asc" class="icon sort order"><svg class="top" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#up-arrow"></use></svg><svg class="bottom" width="1em" height=".5em" version="1.1" viewBox="0 0 12.922194 6.0358899"><use xlink:href="#down-arrow"></use></svg></a>
+							
+							<a href="?sort=name&order=asc" class="order">Name</a>
+						</th>
+						<th>
+							<a href="?sort=size&order=asc" class="order">Size</a>
+						</th>
+						<th class="hideable">
+							<a href="?sort=time&order=asc" class="order">Modified</a>
+						</th>
+						<th class="hideable"></th>
+					</tr>
+					</thead>
+					<tbody>
+					<tr>
+						<td></td>
+						<td>
+							<a href="..">
+								<span class="goup">Go up</span>
+							</a>
+						</td>
+						<td>&mdash;</td>
+						<td class="hideable">&mdash;</td>
+						<td class="hideable"></td>
+					</tr>
+					{{- range .Entries}}
+					<tr class="file">
+						<td>
+						</td>
+						<td>
+							{{- if .IsDir}}
+							<svg width="1.5em" height="1em" version="1.1" viewBox="0 0 317 259"><use xlink:href="#folder"></use></svg>
+							{{- else}}
+							<svg width="1.5em" height="1em" version="1.1" viewBox="0 0 265 323"><use xlink:href="#file"></use></svg>
+							{{- end}}
+							<span class="name"><a href="{{html .URL}}">{{html .Leaf}}</a></span>
+						</td>
+						{{- if .IsDir}}
+						<td data-order="-1">&mdash;</td>
+						{{- else}}
+						<td data-order="{{.Size}}"><size>{{.Size}}</size></td>
+						{{- end}}
+						{{- if .ModTime | afterEpoch }}
+						<td class="hideable"><time datetime="{{.ModTime }}">{{.ModTime }}</time></td>
+						{{- else}}
+						<td class="hideable">—</td>
+						{{- end}}
+						<td class="hideable"></td>
+					</tr>
+					{{- end}}
+					</tbody>
+				</table>
+			</div>
+		</main>
+		<script>
+			var filterEl = document.getElementById('filter');
+			filterEl.focus();
+			function filter() {
+				var q = filterEl.value.trim().toLowerCase();
+				var elems = document.querySelectorAll('tr.file');
+				elems.forEach(function(el) {
+					if (!q) {
+						el.style.display = '';
+						return;
+					}
+					var nameEl = el.querySelector('.name');
+					var nameVal = nameEl.textContent.trim().toLowerCase();
+					if (nameVal.indexOf(q) !== -1) {
+						el.style.display = '';
+					} else {
+						el.style.display = 'none';
+					}
+				});
+			}
+			function localizeDatetime(e, index, ar) {
+				if (e.textContent === undefined) {
+					return;
+				}
+				var d = new Date(e.getAttribute('datetime'));
+				if (isNaN(d)) {
+					d = new Date(e.textContent);
+					if (isNaN(d)) {
+						return;
+					}
+				}
+				e.textContent = d.toLocaleString([], {day: "2-digit", month: "2-digit", year: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit"});
+			}
+			var timeList = Array.prototype.slice.call(document.getElementsByTagName("time"));
+			timeList.forEach(localizeDatetime);
+
+			var getUrlParameter = function getUrlParameter(sParam) {
+				var sPageURL = window.location.search.substring(1),
+				sURLVariables = sPageURL.split('&'),
+				sParameterName,
+				i;
+
+				for (i = 0; i < sURLVariables.length; i++) {
+					sParameterName = sURLVariables[i].split('=');
+
+					if (sParameterName[0] === sParam) {
+						return sParameterName[1] === undefined ? true : decodeURIComponent(sParameterName[1]);
+					}
+				}
+			};
+			function toggle(className){
+				var order = getUrlParameter('order');
+				var elements = document.getElementsByClassName(className);
+				for(var i = 0, length = elements.length; i < length; i++) {
+					var currHref = elements[i].href;
+					if(order=='desc'){ 
+						var chg = currHref.replace('desc', 'asc');
+						elements[i].href = chg;
+					}
+					if(order=='asc'){ 
+						var chg = currHref.replace('asc', 'desc');
+						elements[i].href = chg;
+					}
+				}
+			};
+			function readableFileSize(size) {
+				var units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+				var i = 0;
+				while(size >= 1024) {
+					size /= 1024;
+					++i;
+				}
+				return parseFloat(size).toFixed(2) + ' ' + units[i];
+			}
+
+			function changeSize() {
+				var sizes = document.getElementsByTagName("size");
+
+				for (var i = 0; i < sizes.length; i++) {
+					humanSize = readableFileSize(sizes[i].innerHTML);
+					sizes[i].innerHTML = humanSize
+				}
+			}
+		</script>
+	</body>
+</html>

lib/http/testdata/.htpasswd

@@ -0,0 +1,3 @@
+sha:{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY=
+md5:$apr1$s7fogein$IK9ItbnGM14ct0bY4Uyik1
+bcrypt:$2y$10$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2

lib/http/testdata/local.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeTCCA2GgAwIBAgIUewDoUKLIPBZlSAdOaBHPtfFDiUYwDQYJKoZIhvcNAQEL
+BQAwTDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBnJjbG9uZTETMBEGA1UECwwKcmNs
+b25lLWRldjEXMBUGA1UEAwwOZGV2LnJjbG9uZS5vcmcwHhcNMjIxMDI3MDM1NjE2
+WhcNMzIxMDI0MDM1NjE2WjBMMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGcmNsb25l
+MRMwEQYDVQQLDApyY2xvbmUtZGV2MRcwFQYDVQQDDA5kZXYucmNsb25lLm9yZzCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJVMPbsmBVBi2DG17cPrJQxM
+hdUxkd8pcRtrKkLzqIQS18IKJ20BTzrVlJJETO1uMeXETSVVQ4znWLvhZo5Pgq72
+FlvEMEl6QXNNihEB+Bx9f6iArS0Tgo919Kg3JEYu9g3HfsPxt3C1xNJOlSrpPo4w
+YAX8MU+uy3IqPwgURVhPauR/2E64b9RSIsmNCJJXWnREeJVOpeYSUXx6S0fNM/wo
+BvbPSuB1BErYug9AfuvTavJO7VYnMuEEig/1DXYnPJpTTaZWzyhQIGDU52T5iOZk
+sdxv/Yxbq1kKDHmyhG9ALpCHqQMmR3bOEhLijU5UHfFR03eKnzfcQ7y5BMJBDIVs
+fbaBxlJZFHKq9ixUT/7crc1iUXb+gPg/FxMRuR96/fTPrluWlDYaZV3QMa3rxuvn
+PJM5boqGIH9fNxnsmho0V4ZrXgxg5VsnlwHZIlMuJVJGkUQFTl6XhIzkQRmjQM4C
+WO9KPSusvCM3gCM7j9Tyhi3xl7XRVlN9xb8vbnbyqcZe0lgJU/y3nrbWqORvUKb0
+FvarmtIH3f9yqu+UB2s8DIg5zPjVmTnIMSJYJi5Cjh1YT2YVJ4+JJ0KlxAwOVTU4
+zdup3fUA3Ne2o9ehJEJwuFPbMDCEHeIuTzOKleKCDbHRIvFLleJP4J9/qA1P2C2/
+ZAoS4M8vgd2O84eVm7pzAgMBAAGjUzBRMB0GA1UdDgQWBBTDsVFbaf4C4yDDOGml
+BMyKSgsT5DAfBgNVHSMEGDAWgBTDsVFbaf4C4yDDOGmlBMyKSgsT5DAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAx2wDldW5wZfBGdIv9UCTL1hdB
+a858v+mWCAjXHaFCyMAe2UOBkSF4M8oIqgMXP3ZwysAhB54qKcpKgxKv6s5oyA/k
+diheH8Zl7GYnUS+A1v9ZUx5AwJbVNKCSXYfXQxTsoC1jV0W3HqeJIoxix3XJDZIL
+TCiA8+BMk72vtJBPNlV8wmuN2+aUEktZF1PqjbqU/6ajRvDIa4ogXIVoSunpFXt9
+h5Af2zOE051zqjqshr9Egz6Hl2EItjJgqZtwLD8qSZFzcXYgDvtG16YceGt+ljeC
+yLYE8Qvm9lfQka9Sxucu4+2kzCwjg5ubFaaSkiX6b/ue9KvPHm36JAV9NDwyyA/q
+BNNxK+0PniNPLBdxIkFZvVraLieDNCXV3cqH7781IP2PRvbzNAR1PFfo19U7bmHV
+PsOBj9kIQBhOLcxtxWvK93ptC6vLJYRsPha7kClVI5kht9oB8Mkfi2mAu2Pi7Pka
+ZYHl14XnJPUWEbxX26I4CAU09yGjnQhRwPfGGNPCaMMGsYl+2nn4j/rzbQ8uz7Kg
+l1TQS4WBpX4T6pxWM/mWERbBTLxniE8DNYPgHpgZJSXD16uG/ksyjdet7B86KA0v
+kymH793pOqW8rtrKbziSZyShJy5AYsGy0Xu4ymW03F8S5FUyjWIkRmyUbGTB1q02
+nniyTJh2BffUcH2iCg==
+-----END CERTIFICATE-----

lib/http/testdata/local.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCVTD27JgVQYtgx
+te3D6yUMTIXVMZHfKXEbaypC86iEEtfCCidtAU861ZSSREztbjHlxE0lVUOM51i7
+4WaOT4Ku9hZbxDBJekFzTYoRAfgcfX+ogK0tE4KPdfSoNyRGLvYNx37D8bdwtcTS
+TpUq6T6OMGAF/DFPrstyKj8IFEVYT2rkf9hOuG/UUiLJjQiSV1p0RHiVTqXmElF8
+ektHzTP8KAb2z0rgdQRK2LoPQH7r02ryTu1WJzLhBIoP9Q12JzyaU02mVs8oUCBg
+1Odk+YjmZLHcb/2MW6tZCgx5soRvQC6Qh6kDJkd2zhIS4o1OVB3xUdN3ip833EO8
+uQTCQQyFbH22gcZSWRRyqvYsVE/+3K3NYlF2/oD4PxcTEbkfev30z65blpQ2GmVd
+0DGt68br5zyTOW6KhiB/XzcZ7JoaNFeGa14MYOVbJ5cB2SJTLiVSRpFEBU5el4SM
+5EEZo0DOAljvSj0rrLwjN4AjO4/U8oYt8Ze10VZTfcW/L2528qnGXtJYCVP8t562
+1qjkb1Cm9Bb2q5rSB93/cqrvlAdrPAyIOcz41Zk5yDEiWCYuQo4dWE9mFSePiSdC
+pcQMDlU1OM3bqd31ANzXtqPXoSRCcLhT2zAwhB3iLk8zipXigg2x0SLxS5XiT+Cf
+f6gNT9gtv2QKEuDPL4HdjvOHlZu6cwIDAQABAoICAD7EzqFb01kgLZf8zqmTt8BL
+fesLy7IA6OpnrF14tq1MhMSyYzALoGVyfWPfbl5WeYkJ9otPJTbc3ywikG0dlap8
+kRrkyY5i5ZiWDYmoA8nqo5zS+LweW0J4i7Obd1dAkDdr2+qCuiabbVQkMMfZR3Ed
+eomZpZvEOAnYJCb/6sW9ognOjEFQfsfL/o8xidyI+GEwlmfjqJEpu3OzsOnPpt8J
+byAeN/NVj0fuhY87BQGeIfUc5ODXWydKsscRtqapyWtywY9BKRhgU7SSXnTQCtQe
+mr68oON9ePVW6bbSrKZfXBRszMyjr+ENs4CYGmPHrs0SI2+7asRgCWSTfyIymhHs
+DjF6BHAlQaOLnVcffuGhc6c0bnzIFvg1JenWVj9uQSkADW8TMkNgMLdcx8JoMd30
+9HiHZyQy7hMuzWjZSgS1wRp3KMGBo+WuMd4NF662nQPfylG2iJvedLtbvY7mBGkX
+1e6L4b1fkFAZWyfBj3JX1SqgcNDnqzDiqqUDwpY8b1G9nAegvWBeFbEgnORa6pVF
+8xWNtWlMsXYjTvPLbvDSZc6l0wOpFw7roGZgQpbaTjmbeqrZMsr/XDtH7ksLCBOh
+V/s2oMU0NUBu/mokvpLupBzVMbA7/TKGr36OrHw57P8dhsdD3bqx+UbUMgjsq6xs
+UAMcjWJFoc3ulgsVHF2BAoIBAQDEo5i63JkQL3+8rXQ25StbEgqVGXcGS2oZLu+G
+GYklJ0NyGrjoYWmZ/QQf9NbAhQKDSe/X2cWtTLMF9Y2yVsxQ2x1df3lonk9dHsoX
+O+Yp8C9zKt24dRWUsyklgkxAmb/nq3dRPtm/80ztNVQvYy5SQ7G5s/suXt9Wyd2y
+nl08ZWclaDkHNS33/8hYLHP3RkjLgcOu45UDb/3vXk6ljnlrnqe7nlRX5UrnVXs3
+K41xJpCSHmUe7SUoW1ExjzWhtQBkWWVWWV8gE8CNpU0nheTkA5KyXQ8QfQ13jxJj
+zPy/3OPGMIElbvEBcXxvcRncpuUIVO4tRy+oZuz2bmLwbBiBAoIBAQDCXhS5wMgJ
+fWNzsd9prFrENVz7CR2NK3LSdVpm4ZtR5f4mLyzYj/RcSIYC4U7uO0WXaoLTnhKs
+KbKAj5KC2HZUcqhLscajXPl6OjBOcEAlujh9VeTjEnx6XU81IwmAjrYr98xAz3HJ
+IOaD8MvfZ13JWDB9q5bq3A8EKTEAPh66mv/w4Ic8rnueFZgxEFpD4Z2bSB7xmEHj
+tE7VuUbdkLQ4jDrxwZuj7vC0+50pqDrSe0VY/KaGOnNtF1fVs3N+xlI9yUBhr9Yy
+ANqUlx8Ql0i2aIuTSR9h/Wk9uu3NmsON9FkLoYK4bg9jiMcA6eKyRK/9Txn/CygG
+zNy3+s2RhHjzAoIBAQCCZ5Pz6DPB3h4yPD2j4hr8jFxkQL0EeaLlDJFgNzMSZpV9
+6GbUBTYJHxhLMQ3yIsNl2fSrCwrjQMhAnXXY3WMmBAnXZaBYVxR+xtpyyhB7o4N0
+NutPVqZ3NNGGxIBZHx17P+UjBjFV8L4FWaZ4vqeLesU0SD29pMEsRzc1K3zdfsoG
+rrWTKBtSKljs0J4fUIcaHvZs1xSNcQnQYpR5iqDPVCocbIW2vKMOA0xxa/qjHVYm
+8O1SsyY/Oz//Q9/nW6fk5LwlpaNGHJNH3GXsXglLhWsVyk0hPC1gKouhj+HWQ2Dy
+oFwlPQurT12ccj8aa7vb6KcDdAARCCEB1HbcxnMBAoIBADnY6FA0eRSh9eRsDvMT
+cdwtiaPJHbtzL/RFKweto51nVxGkPrOhfHeuufvHdMdgaqDa+V7kD+ifbFno4REC
+PY16pm4I1fau6C0hflkJ/X19A+0BkGKokNWWScmlyOEzGDLTyD2Nv+69VP31v6eY
+ywfusFfmpr71iZ6SZ9wLoPemw/+7w2QjBfWRtb78f/DuCAs8FsGOsCWF92SShO3S
+cGDYE376QUk0Bv3GWQsZ34/fUk9eumz+nnXcWa7nfrs/aSCscfXg8F3ndSZ+J6e3
+btOjH89RFv8B/b16keX8ZrEsBQh6JD6huwDDp361HVwzJzG7xh/rARmtBQ/YnC/v
+/lMCggEAZ+EA8vRD7KSDLaxUl/DG0VQYtyJWsf83/NcLAizvYirI9batLjUF8ogy
+pNHmA1STVJko+P+M2V5v56lJWnrj6HPj91NPIqdgJ/hIIZNI1zXm/DlLOegrFH0W
+7fIY3+ZxzCIMuN4TvZ2H7n6NDBvTr6Vno17nFRGzy1suceIk7tx66KiFGmVt0sT/
+yyCQdyk+uM0KG3j19+QDrbtOTTmK3cOHTEOs7D2RmHSbn51jOckzL2mrhXvcGfTS
+cqYW21Gm9U2P7VjgQ9vHuGzUFIrskltAGuX38XdPiBu9vZp1hwPbYY9MLxrbW8UM
+ySc0NWyPv8/wyfen4dBRJ9OthwwcXw==
+-----END PRIVATE KEY-----