From 6d622672271fa638612aa82fec81e4889f9c09fb Mon Sep 17 00:00:00 2001 From: Tom Mombourquette Date: Tue, 8 Nov 2022 07:49:19 -0400 Subject: [PATCH] serve http: support unix sockets and multiple listners - add support for unix sockets (which skip the auth). - add support for multiple listeners - collapse unnecessary internal structure of lib/http so it can all be imported together - moves files in sub directories of lib/http into the main lib/http directory and reworks the code that uses them. See: https://forum.rclone.org/t/wip-rc-rcd-over-unix-socket/33619 Fixes: #6605 --- cmd/serve/http/http.go | 105 ++++--- cmd/serve/http/http_test.go | 38 +-- lib/http/auth.go | 70 +++++ lib/http/auth/auth.go | 84 ------ lib/http/auth/basic.go | 120 -------- lib/http/context.go | 68 +++++ lib/http/http.go | 441 ----------------------------- lib/http/http_test.go | 515 ---------------------------------- lib/http/middleware.go | 171 +++++++++++ lib/http/middleware_test.go | 231 +++++++++++++++ lib/http/serve/dir_test.go | 6 +- lib/http/serve/serve_test.go | 12 +- lib/http/server.go | 430 ++++++++++++++++++++++++++++ lib/http/server_test.go | 444 +++++++++++++++++++++++++++++ lib/http/template.go | 95 +++++++ lib/http/templates/index.html | 389 +++++++++++++++++++++++++ lib/http/testdata/.htpasswd | 3 + lib/http/testdata/local.crt | 32 +++ lib/http/testdata/local.key | 52 ++++ 19 files changed, 2080 insertions(+), 1226 deletions(-) create mode 100644 lib/http/auth.go delete mode 100644 lib/http/auth/auth.go delete mode 100644 lib/http/auth/basic.go create mode 100644 lib/http/context.go delete mode 100644 lib/http/http.go delete mode 100644 lib/http/http_test.go create mode 100644 lib/http/middleware.go create mode 100644 lib/http/middleware_test.go create mode 100644 lib/http/server.go create mode 100644 lib/http/server_test.go create mode 100644 lib/http/template.go create mode 100644 lib/http/templates/index.html create mode 100644 lib/http/testdata/.htpasswd create mode 100644 lib/http/testdata/local.crt create mode 100644 lib/http/testdata/local.key diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index 5db094dce..594ac551c 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -2,7 +2,8 @@ package http import ( - "html/template" + "context" + "fmt" "io" "log" "net/http" @@ -10,16 +11,15 @@ import ( "path" "strconv" "strings" + "sync" "time" - "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/rclone/rclone/cmd" - "github.com/rclone/rclone/cmd/serve/http/data" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" - httplib "github.com/rclone/rclone/lib/http" - "github.com/rclone/rclone/lib/http/auth" + "github.com/rclone/rclone/lib/atexit" + libhttp "github.com/rclone/rclone/lib/http" "github.com/rclone/rclone/lib/http/serve" "github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs/vfsflags" @@ -28,20 +28,27 @@ import ( // Options required for http server type Options struct { - data.Options + Auth libhttp.AuthConfig + HTTP libhttp.HTTPConfig + Template libhttp.TemplateConfig } // DefaultOpt is the default values used for Options -var DefaultOpt = Options{} +var DefaultOpt = Options{ + Auth: libhttp.DefaultAuthCfg(), + HTTP: libhttp.DefaultHTTPCfg(), + Template: libhttp.DefaultTemplateCfg(), +} // Opt is options set by command line flags var Opt = DefaultOpt func init() { - data.AddFlags(Command.Flags(), "", &Opt.Options) - httplib.AddFlags(Command.Flags()) - auth.AddFlags(Command.Flags()) - vfsflags.AddFlags(Command.Flags()) + flagSet := Command.Flags() + libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth) + libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP) + libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template) + vfsflags.AddFlags(flagSet) } // Command definition for cobra @@ -59,60 +66,78 @@ The server will log errors. Use ` + "`-v`" + ` to see access logs. ` + "`--bwlimit`" + ` will be respected for file transfers. Use ` + "`--stats`" + ` to control the stats printing. -` + httplib.Help + data.Help + auth.Help + vfs.Help, +` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help, Annotations: map[string]string{ "versionIntroduced": "v1.39", }, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) f := cmd.NewFsSrc(args) + cmd.Run(false, true, command, func() error { - s := newServer(f, Opt.Template) - router, err := httplib.Router() + ctx := context.Background() + + s, err := run(ctx, f, Opt) if err != nil { - return err + log.Fatal(err) } - s.Bind(router) - httplib.Wait() + + var finaliseOnce sync.Once + finalise := func() { + finaliseOnce.Do(func() { + if err := s.server.Shutdown(); err != nil { + log.Printf("error shutting down server: %v", err) + } + }) + } + fnHandle := atexit.Register(finalise) + defer atexit.Unregister(fnHandle) + + s.server.Wait() return nil }) }, } // server contains everything to run the server -type server struct { - f fs.Fs - vfs *vfs.VFS - HTMLTemplate *template.Template // HTML template for web interface +type serveCmd struct { + f fs.Fs + vfs *vfs.VFS + server libhttp.Server } -func newServer(f fs.Fs, templatePath string) *server { - htmlTemplate, templateErr := data.GetTemplate(templatePath) - if templateErr != nil { - log.Fatalf(templateErr.Error()) - } - s := &server{ - f: f, - vfs: vfs.New(f, &vfsflags.Opt), - HTMLTemplate: htmlTemplate, - } - return s -} +func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) { + var err error -func (s *server) Bind(router chi.Router) { - if m := auth.Auth(auth.Opt); m != nil { - router.Use(m) + s := &serveCmd{ + f: f, + vfs: vfs.New(f, &vfsflags.Opt), } + + s.server, err = libhttp.NewServer(ctx, + libhttp.WithConfig(opt.HTTP), + libhttp.WithAuth(opt.Auth), + libhttp.WithTemplate(opt.Template), + ) + if err != nil { + return nil, fmt.Errorf("failed to init server: %w", err) + } + + router := s.server.Router() router.Use( middleware.SetHeader("Accept-Ranges", "bytes"), middleware.SetHeader("Server", "rclone/"+fs.Version), ) router.Get("/*", s.handler) router.Head("/*", s.handler) + + s.server.Serve() + + return s, nil } // handler reads incoming requests and dispatches them -func (s *server) handler(w http.ResponseWriter, r *http.Request) { +func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) { isDir := strings.HasSuffix(r.URL.Path, "/") remote := strings.Trim(r.URL.Path, "/") if isDir { @@ -123,7 +148,7 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) { } // serveDir serves a directory index at dirRemote -func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) { +func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) { // List the directory node, err := s.vfs.Stat(dirRemote) if err == vfs.ENOENT { @@ -145,7 +170,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri } // Make the entries for display - directory := serve.NewDirectory(dirRemote, s.HTMLTemplate) + directory := serve.NewDirectory(dirRemote, s.server.HTMLTemplate()) for _, node := range dirEntries { if vfsflags.Opt.NoModTime { directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{}) @@ -165,7 +190,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri } // serveFile serves a file object at remote -func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) { +func (s *serveCmd) serveFile(w http.ResponseWriter, r *http.Request, remote string) { node, err := s.vfs.Stat(remote) if err == vfs.ENOENT { fs.Infof(remote, "%s: File not found", r.RemoteAddr) diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go index 0c77e4b61..9f264adce 100644 --- a/cmd/serve/http/http_test.go +++ b/cmd/serve/http/http_test.go @@ -14,14 +14,14 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/filter" - httplib "github.com/rclone/rclone/lib/http" + libhttp "github.com/rclone/rclone/lib/http" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var ( updateGolden = flag.Bool("updategolden", false, "update golden files for regression test") - httpServer *server + sc *serveCmd testURL string ) @@ -30,16 +30,25 @@ const ( testTemplate = "testdata/golden/testindex.html" ) -func startServer(t *testing.T, f fs.Fs) { - opt := httplib.DefaultOpt - opt.ListenAddr = testBindAddress - httpServer = newServer(f, testTemplate) - router, err := httplib.Router() - if err != nil { - t.Fatal(err.Error()) +func start(t *testing.T, f fs.Fs) { + ctx := context.Background() + + opts := Options{ + HTTP: libhttp.DefaultHTTPCfg(), + Template: libhttp.TemplateConfig{ + Path: testTemplate, + }, } - httpServer.Bind(router) - testURL = httplib.URL() + opts.HTTP.ListenAddr = []string{testBindAddress} + + s, err := run(ctx, f, opts) + require.NoError(t, err, "failed to start server") + sc = s + + urls := s.server.URLs() + require.Len(t, urls, 1, "expected one URL") + + testURL = urls[0] // try to connect to the test server pause := time.Millisecond @@ -54,7 +63,6 @@ func startServer(t *testing.T, f fs.Fs) { pause *= 2 } t.Fatal("couldn't connect to server") - } var ( @@ -84,7 +92,7 @@ func TestInit(t *testing.T) { require.NoError(t, err) require.NoError(t, obj.SetModTime(context.Background(), expectedTime)) - startServer(t, f) + start(t, f) } // check body against the file, or re-write body if -updategolden is @@ -229,7 +237,3 @@ func TestGET(t *testing.T) { checkGolden(t, test.Golden, body) } } - -func TestFinalise(t *testing.T) { - _ = httplib.Shutdown() -} diff --git a/lib/http/auth.go b/lib/http/auth.go new file mode 100644 index 000000000..b7784c4b4 --- /dev/null +++ b/lib/http/auth.go @@ -0,0 +1,70 @@ +package http + +import ( + "github.com/rclone/rclone/fs/config/flags" + "github.com/spf13/pflag" +) + +// Help contains text describing the http authentication to add to the command +// help. +var AuthHelp = ` +#### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the ` + "`--user` and `--pass`" + ` flags. + +Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use ` + "`--realm`" + ` to set the authentication realm. + +Use ` + "`--salt`" + ` to change the password hashing salt from the default. +` + +// CustomAuthFn if used will be used to authenticate user, pass. If an error +// is returned then the user is not authenticated. +// +// If a non nil value is returned then it is added to the context under the key +type CustomAuthFn func(user, pass string) (value interface{}, err error) + +// AuthConfig contains options for the http authentication +type AuthConfig struct { + HtPasswd string // htpasswd file - if not provided no authentication is done + Realm string // realm for authentication + BasicUser string // single username for basic auth if not using Htpasswd + BasicPass string // password for BasicUser + Salt string // password hashing salt + CustomAuthFn CustomAuthFn `json:"-"` // custom Auth (not set by command line flags) +} + +// AddFlagsPrefix adds flags to the flag set for AuthConfig +func (cfg *AuthConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) { + flags.StringVarP(flagSet, &cfg.HtPasswd, prefix+"htpasswd", "", cfg.HtPasswd, "A htpasswd file - if not provided no authentication is done") + flags.StringVarP(flagSet, &cfg.Realm, prefix+"realm", "", cfg.Realm, "Realm for authentication") + flags.StringVarP(flagSet, &cfg.BasicUser, prefix+"user", "", cfg.BasicUser, "User name for authentication") + flags.StringVarP(flagSet, &cfg.BasicPass, prefix+"pass", "", cfg.BasicPass, "Password for authentication") + flags.StringVarP(flagSet, &cfg.Salt, prefix+"salt", "", cfg.Salt, "Password hashing salt") +} + +// AddAuthFlagsPrefix adds flags to the flag set for AuthConfig +func AddAuthFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *AuthConfig) { + cfg.AddFlagsPrefix(flagSet, prefix) +} + +// DefaultAuthCfg returns a new config which can be customized by command line flags +func DefaultAuthCfg() AuthConfig { + return AuthConfig{ + Salt: "dlPL2MqE", + } +} diff --git a/lib/http/auth/auth.go b/lib/http/auth/auth.go deleted file mode 100644 index db50c8a30..000000000 --- a/lib/http/auth/auth.go +++ /dev/null @@ -1,84 +0,0 @@ -// Package auth provides authentication for http. -package auth - -import ( - "github.com/rclone/rclone/fs/config/flags" - "github.com/rclone/rclone/lib/http" - "github.com/spf13/pflag" -) - -// Help contains text describing the http authentication to add to the command -// help. -var Help = ` -#### Authentication - -By default this will serve files without needing a login. - -You can either use an htpasswd file which can take lots of users, or -set a single username and password with the ` + "`--user` and `--pass`" + ` flags. - -Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file. This is -in standard apache format and supports MD5, SHA1 and BCrypt for basic -authentication. Bcrypt is recommended. - -To create an htpasswd file: - - touch htpasswd - htpasswd -B htpasswd user - htpasswd -B htpasswd anotherUser - -The password file can be updated while rclone is running. - -Use ` + "`--realm`" + ` to set the authentication realm. - -Use ` + "`--salt`" + ` to change the password hashing salt from the default. -` - -// CustomAuthFn if used will be used to authenticate user, pass. If an error -// is returned then the user is not authenticated. -// -// If a non nil value is returned then it is added to the context under the key -type CustomAuthFn func(user, pass string) (value interface{}, err error) - -// Options contains options for the http authentication -type Options struct { - HtPasswd string // htpasswd file - if not provided no authentication is done - Realm string // realm for authentication - BasicUser string // single username for basic auth if not using Htpasswd - BasicPass string // password for BasicUser - Salt string // password hashing salt - Auth CustomAuthFn `json:"-"` // custom Auth (not set by command line flags) -} - -// Auth instantiates middleware that authenticates users based on the configuration -func Auth(opt Options) http.Middleware { - if opt.Auth != nil { - return CustomAuth(opt.Auth, opt.Realm) - } else if opt.HtPasswd != "" { - return HtPasswdAuth(opt.HtPasswd, opt.Realm) - } else if opt.BasicUser != "" { - return SingleAuth(opt.BasicUser, opt.BasicPass, opt.Realm, opt.Salt) - } - return nil -} - -// Options set by command line flags -var ( - Opt = Options{ - Salt: "dlPL2MqE", - } -) - -// AddFlagsPrefix adds flags for http/auth -func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) { - flags.StringVarP(flagSet, &Opt.HtPasswd, prefix+"htpasswd", "", Opt.HtPasswd, "A htpasswd file - if not provided no authentication is done") - flags.StringVarP(flagSet, &Opt.Realm, prefix+"realm", "", Opt.Realm, "Realm for authentication") - flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication") - flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication") - flags.StringVarP(flagSet, &Opt.Salt, prefix+"salt", "", Opt.Salt, "Password hashing salt") -} - -// AddFlags adds flags for the http/auth -func AddFlags(flagSet *pflag.FlagSet) { - AddFlagsPrefix(flagSet, "", &Opt) -} diff --git a/lib/http/auth/basic.go b/lib/http/auth/basic.go deleted file mode 100644 index 69907c009..000000000 --- a/lib/http/auth/basic.go +++ /dev/null @@ -1,120 +0,0 @@ -package auth - -import ( - "context" - "encoding/base64" - "net/http" - "strings" - - auth "github.com/abbot/go-http-auth" - "github.com/rclone/rclone/fs" - httplib "github.com/rclone/rclone/lib/http" -) - -// parseAuthorization parses the Authorization header into user, pass -// it returns a boolean as to whether the parse was successful -func parseAuthorization(r *http.Request) (user, pass string, ok bool) { - authHeader := r.Header.Get("Authorization") - if authHeader != "" { - s := strings.SplitN(authHeader, " ", 2) - if len(s) == 2 && s[0] == "Basic" { - b, err := base64.StdEncoding.DecodeString(s[1]) - if err == nil { - parts := strings.SplitN(string(b), ":", 2) - user = parts[0] - if len(parts) > 1 { - pass = parts[1] - ok = true - } - } - } - } - return -} - -type contextUserType struct{} - -// ContextUserKey is a simple context key for storing the username of the request -var ContextUserKey = &contextUserType{} - -type contextAuthType struct{} - -// ContextAuthKey is a simple context key for storing info returned by CustomAuthFn -var ContextAuthKey = &contextAuthType{} - -// LoggedBasicAuth extends BasicAuth to include access logging -type LoggedBasicAuth struct { - auth.BasicAuth -} - -// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests -func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string { - username := a.BasicAuth.CheckAuth(r) - if username == "" { - user, _, _ := parseAuthorization(r) - fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user) - } - return username -} - -// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator -func NewLoggedBasicAuthenticator(realm string, secrets auth.SecretProvider) *LoggedBasicAuth { - return &LoggedBasicAuth{BasicAuth: auth.BasicAuth{Realm: realm, Secrets: secrets}} -} - -// Helper to generate required interface for middleware -func basicAuth(authenticator *LoggedBasicAuth) httplib.Middleware { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if username := authenticator.CheckAuth(r); username == "" { - authenticator.RequireAuth(w, r) - } else { - r = r.WithContext(context.WithValue(r.Context(), ContextUserKey, username)) - next.ServeHTTP(w, r) - } - }) - } -} - -// HtPasswdAuth instantiates middleware that authenticates against the passed htpasswd file -func HtPasswdAuth(path, realm string) httplib.Middleware { - fs.Infof(nil, "Using %q as htpasswd storage", path) - secretProvider := auth.HtpasswdFileProvider(path) - authenticator := NewLoggedBasicAuthenticator(realm, secretProvider) - return basicAuth(authenticator) -} - -// SingleAuth instantiates middleware that authenticates for a single user -func SingleAuth(user, pass, realm, salt string) httplib.Middleware { - fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user) - pass = string(auth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$"))) - secretProvider := func(u, r string) string { - if user == u { - return pass - } - return "" - } - authenticator := NewLoggedBasicAuthenticator(realm, secretProvider) - return basicAuth(authenticator) -} - -// CustomAuth instantiates middleware that authenticates using a custom function -func CustomAuth(fn CustomAuthFn, realm string) httplib.Middleware { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - user, pass, ok := parseAuthorization(r) - if ok { - value, err := fn(user, pass) - if err != nil { - fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err) - auth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting - return - } - if value != nil { - r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value)) - } - next.ServeHTTP(w, r) - } - }) - } -} diff --git a/lib/http/context.go b/lib/http/context.go new file mode 100644 index 000000000..34bff8fa6 --- /dev/null +++ b/lib/http/context.go @@ -0,0 +1,68 @@ +package http + +import ( + "context" + "net" + "net/http" +) + +type ctxKey int + +const ( + ctxKeyAuth ctxKey = iota + ctxKeyPublicURL + ctxKeyUnixSock + ctxKeyUser +) + +// NewBaseContext initializes the context for all requests, adding info for use in middleware and handlers +func NewBaseContext(ctx context.Context, url string) func(l net.Listener) context.Context { + return func(l net.Listener) context.Context { + if l.Addr().Network() == "unix" { + ctx = context.WithValue(ctx, ctxKeyUnixSock, true) + return ctx + } + + ctx = context.WithValue(ctx, ctxKeyPublicURL, url) + return ctx + } +} + +// IsAuthenticated checks if this request was authenticated via a middleware +func IsAuthenticated(r *http.Request) bool { + if v := r.Context().Value(ctxKeyAuth); v != nil { + return true + } + if v := r.Context().Value(ctxKeyUser); v != nil { + return true + } + return false +} + +// IsUnixSocket checks if the request was received on a unix socket, used to skip auth & CORS +func IsUnixSocket(r *http.Request) bool { + v, _ := r.Context().Value(ctxKeyUnixSock).(bool) + return v +} + +// PublicURL returns the URL defined in NewBaseContext, used for logging & CORS +func PublicURL(r *http.Request) string { + v, _ := r.Context().Value(ctxKeyPublicURL).(string) + return v +} + +// CtxGetAuth is a wrapper over the private Auth context key +func CtxGetAuth(ctx context.Context) interface{} { + return ctx.Value(ctxKeyAuth) +} + +// CtxGetUser is a wrapper over the private User context key +func CtxGetUser(ctx context.Context) (string, bool) { + v, ok := ctx.Value(ctxKeyUser).(string) + return v, ok +} + +// CtxSetUser is a test helper that injects a User value into context +func CtxSetUser(ctx context.Context, value string) context.Context { + return context.WithValue(ctx, ctxKeyUser, value) +} diff --git a/lib/http/http.go b/lib/http/http.go deleted file mode 100644 index 0138b0fec..000000000 --- a/lib/http/http.go +++ /dev/null @@ -1,441 +0,0 @@ -// Package http provides a registration interface for http services -package http - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "log" - "net" - "net/http" - "os" - "strings" - "sync" - "time" - - "github.com/go-chi/chi/v5" - "github.com/rclone/rclone/fs/config/flags" - "github.com/spf13/pflag" -) - -// Help contains text describing the http server to add to the command -// help. -var Help = ` -### Server options - -Use ` + "`--addr`" + ` to specify which IP address and port the server should -listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all -IPs. By default it only listens on localhost. You can use port -:0 to let the OS choose an available port. - -If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address -then using Authentication is advised - see the next section for info. - -` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to -control the timeouts on the server. Note that this is the total time -for a transfer. - -` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will -accept in the HTTP header. - -` + "`--baseurl`" + ` controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used ` + "`--baseurl \"/rclone\"`" + ` then -rclone would serve from a URL starting with "/rclone/". This is -useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `, -` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated -identically. - -#### SSL/TLS - -By default this will serve over http. If you want you can serve over -https. You will need to supply the ` + "`--cert` and `--key`" + ` flags. -If you wish to do client side certificate validation then you will need to -supply ` + "`--client-ca`" + ` also. - -` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation -of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded -private key and ` + "`--client-ca`" + ` should be the PEM encoded client -certificate authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid - values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default - "tls1.0"). -` - -// Middleware function signature required by chi.Router.Use() -type Middleware func(http.Handler) http.Handler - -// Options contains options for the http Server -type Options struct { - ListenAddr string // Port to listen on - BaseURL string // prefix to strip from URLs - ServerReadTimeout time.Duration // Timeout for server reading data - ServerWriteTimeout time.Duration // Timeout for server writing data - MaxHeaderBytes int // Maximum size of request header - SslCert string // Path to SSL PEM key (concatenation of certificate and CA certificate) - SslKey string // Path to SSL PEM Private key - SslCertBody []byte // SSL PEM key (concatenation of certificate and CA certificate) body, ignores SslCert - SslKeyBody []byte // SSL PEM Private key body, ignores SslKey - ClientCA string // Client certificate authority to verify clients with - MinTLSVersion string // MinTLSVersion contains the minimum TLS version that is acceptable. -} - -// DefaultOpt is the default values used for Options -var DefaultOpt = Options{ - ListenAddr: "127.0.0.1:8080", - ServerReadTimeout: 1 * time.Hour, - ServerWriteTimeout: 1 * time.Hour, - MaxHeaderBytes: 4096, - MinTLSVersion: "tls1.0", -} - -// Server interface of http server -type Server interface { - Router() chi.Router - Route(pattern string, fn func(r chi.Router)) chi.Router - Mount(pattern string, h http.Handler) - Shutdown() error -} - -type server struct { - addrs []net.Addr - tlsAddrs []net.Addr - listeners []net.Listener - tlsListeners []net.Listener - httpServer *http.Server - baseRouter chi.Router - closing *sync.WaitGroup - useSSL bool -} - -var ( - defaultServer *server - defaultServerOptions = DefaultOpt - defaultServerMutex sync.Mutex -) - -func useSSL(opt Options) bool { - return opt.SslKey != "" || len(opt.SslKeyBody) > 0 -} - -// NewServer instantiates a new http server using provided listeners and options -// This function is provided if the default http server does not meet a services requirements and should not generally be used -// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443. -// tlsListeners are ignored if opt.SslKey is not provided -func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, error) { - // Validate input - if len(listeners) == 0 && len(tlsListeners) == 0 { - return nil, errors.New("can't create server without listeners") - } - - // Prepare TLS config - var tlsConfig *tls.Config - - useSSL := useSSL(opt) - if (len(opt.SslCertBody) > 0) != (len(opt.SslKeyBody) > 0) { - err := errors.New("need both SslCertBody and SslKeyBody to use SSL") - log.Fatalf(err.Error()) - return nil, err - } - if (opt.SslCert != "") != (opt.SslKey != "") { - err := errors.New("need both -cert and -key to use SSL") - log.Fatalf(err.Error()) - return nil, err - } - - if useSSL { - var cert tls.Certificate - var err error - if len(opt.SslCertBody) > 0 { - cert, err = tls.X509KeyPair(opt.SslCertBody, opt.SslKeyBody) - } else { - cert, err = tls.LoadX509KeyPair(opt.SslCert, opt.SslKey) - } - if err != nil { - log.Fatal(err) - } - var minTLSVersion uint16 - switch opt.MinTLSVersion { - case "tls1.0": - minTLSVersion = tls.VersionTLS10 - case "tls1.1": - minTLSVersion = tls.VersionTLS11 - case "tls1.2": - minTLSVersion = tls.VersionTLS12 - case "tls1.3": - minTLSVersion = tls.VersionTLS13 - default: - err = errors.New("Invalid value for --min-tls-version") - log.Fatalf(err.Error()) - return nil, err - } - tlsConfig = &tls.Config{ - MinVersion: minTLSVersion, - Certificates: []tls.Certificate{cert}, - } - } else if len(listeners) == 0 && len(tlsListeners) != 0 { - return nil, errors.New("no SslKey or non-tlsListeners") - } - - if opt.ClientCA != "" { - if !useSSL { - err := errors.New("can't use --client-ca without --cert and --key") - log.Fatalf(err.Error()) - return nil, err - } - certpool := x509.NewCertPool() - pem, err := os.ReadFile(opt.ClientCA) - if err != nil { - log.Fatalf("Failed to read client certificate authority: %v", err) - return nil, err - } - if !certpool.AppendCertsFromPEM(pem) { - err := errors.New("can't parse client certificate authority") - log.Fatalf(err.Error()) - return nil, err - } - tlsConfig.ClientCAs = certpool - tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert - } - - // Ignore passing "/" for BaseURL - opt.BaseURL = strings.Trim(opt.BaseURL, "/") - if opt.BaseURL != "" { - opt.BaseURL = "/" + opt.BaseURL - } - - // Build base router - var router chi.Router = chi.NewRouter() - router.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) - }) - router.NotFound(func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - }) - - handler := router.(http.Handler) - if opt.BaseURL != "" { - handler = http.StripPrefix(opt.BaseURL, handler) - } - - // Serve on listeners - httpServer := &http.Server{ - Handler: handler, - ReadTimeout: opt.ServerReadTimeout, - WriteTimeout: opt.ServerWriteTimeout, - MaxHeaderBytes: opt.MaxHeaderBytes, - ReadHeaderTimeout: 10 * time.Second, // time to send the headers - IdleTimeout: 60 * time.Second, // time to keep idle connections open - TLSConfig: tlsConfig, - } - - addrs, tlsAddrs := make([]net.Addr, len(listeners)), make([]net.Addr, len(tlsListeners)) - - wg := &sync.WaitGroup{} - - for i, l := range listeners { - addrs[i] = l.Addr() - } - - if useSSL { - for i, l := range tlsListeners { - tlsAddrs[i] = l.Addr() - } - } - - return &server{addrs, tlsAddrs, listeners, tlsListeners, httpServer, router, wg, useSSL}, nil -} - -func (s *server) Serve() { - serve := func(l net.Listener, tls bool) { - defer s.closing.Done() - var err error - if tls { - err = s.httpServer.ServeTLS(l, "", "") - } else { - err = s.httpServer.Serve(l) - } - if err != http.ErrServerClosed && err != nil { - log.Fatalf(err.Error()) - } - } - - s.closing.Add(len(s.listeners)) - for _, l := range s.listeners { - go serve(l, false) - } - - if s.useSSL { - s.closing.Add(len(s.tlsListeners)) - for _, l := range s.tlsListeners { - go serve(l, true) - } - } -} - -// Wait blocks while the server is serving requests -func (s *server) Wait() { - s.closing.Wait() -} - -// Router returns the server base router -func (s *server) Router() chi.Router { - return s.baseRouter -} - -// Route mounts a sub-Router along a `pattern` string. -func (s *server) Route(pattern string, fn func(r chi.Router)) chi.Router { - return s.baseRouter.Route(pattern, fn) -} - -// Mount attaches another http.Handler along ./pattern/* -func (s *server) Mount(pattern string, h http.Handler) { - s.baseRouter.Mount(pattern, h) -} - -// Shutdown gracefully shuts down the server -func (s *server) Shutdown() error { - if err := s.httpServer.Shutdown(context.Background()); err != nil { - return err - } - s.closing.Wait() - return nil -} - -//---- Default HTTP server convenience functions ---- - -// Router returns the server base router -func Router() (chi.Router, error) { - if err := start(); err != nil { - return nil, err - } - return defaultServer.baseRouter, nil -} - -// Route mounts a sub-Router along a `pattern` string. -func Route(pattern string, fn func(r chi.Router)) (chi.Router, error) { - if err := start(); err != nil { - return nil, err - } - return defaultServer.Route(pattern, fn), nil -} - -// Mount attaches another http.Handler along ./pattern/* -func Mount(pattern string, h http.Handler) error { - if err := start(); err != nil { - return err - } - defaultServer.Mount(pattern, h) - return nil -} - -// Restart or start the default http server using the default options and no handlers -func Restart() error { - if e := Shutdown(); e != nil { - return e - } - - return start() -} - -// Wait blocks while the default http server is serving requests -func Wait() { - defaultServer.Wait() -} - -// Start the default server -func start() error { - defaultServerMutex.Lock() - defer defaultServerMutex.Unlock() - - if defaultServer != nil { - // Server already started, do nothing - return nil - } - - var err error - var l net.Listener - l, err = net.Listen("tcp", defaultServerOptions.ListenAddr) - if err != nil { - return err - } - - var s Server - if useSSL(defaultServerOptions) { - s, err = NewServer([]net.Listener{}, []net.Listener{l}, defaultServerOptions) - } else { - s, err = NewServer([]net.Listener{l}, []net.Listener{}, defaultServerOptions) - } - if err != nil { - return err - } - defaultServer = s.(*server) - defaultServer.Serve() - return nil -} - -// Shutdown gracefully shuts down the default http server -func Shutdown() error { - defaultServerMutex.Lock() - defer defaultServerMutex.Unlock() - if defaultServer != nil { - s := defaultServer - defaultServer = nil - return s.Shutdown() - } - return nil -} - -// GetOptions thread safe getter for the default server options -func GetOptions() Options { - defaultServerMutex.Lock() - defer defaultServerMutex.Unlock() - return defaultServerOptions -} - -// SetOptions thread safe setter for the default server options -func SetOptions(opt Options) { - defaultServerMutex.Lock() - defer defaultServerMutex.Unlock() - defaultServerOptions = opt -} - -//---- Utility functions ---- - -// URL of default http server -func URL() string { - if defaultServer == nil { - panic("Server not running") - } - for _, a := range defaultServer.addrs { - return fmt.Sprintf("http://%s%s/", a.String(), defaultServerOptions.BaseURL) - } - for _, a := range defaultServer.tlsAddrs { - return fmt.Sprintf("https://%s%s/", a.String(), defaultServerOptions.BaseURL) - } - panic("Server is running with no listener") -} - -//---- Command line flags ---- - -// AddFlagsPrefix adds flags for the httplib -func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) { - flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to") - flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, prefix+"server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data") - flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, prefix+"server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data") - flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, prefix+"max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header") - flags.StringVarP(flagSet, &Opt.SslCert, prefix+"cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)") - flags.StringVarP(flagSet, &Opt.SslKey, prefix+"key", "", Opt.SslKey, "SSL PEM Private key") - flags.StringVarP(flagSet, &Opt.ClientCA, prefix+"client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with") - flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root") - flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable") - -} - -// AddFlags adds flags for the httplib -func AddFlags(flagSet *pflag.FlagSet) { - AddFlagsPrefix(flagSet, "", &defaultServerOptions) -} diff --git a/lib/http/http_test.go b/lib/http/http_test.go deleted file mode 100644 index fd9850059..000000000 --- a/lib/http/http_test.go +++ /dev/null @@ -1,515 +0,0 @@ -package http - -import ( - "crypto/tls" - "net" - "net/http" - "reflect" - "strings" - "testing" - "time" - - "golang.org/x/net/nettest" - - "github.com/go-chi/chi/v5" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGetOptions(t *testing.T) { - tests := []struct { - name string - want Options - }{ - {name: "basic", want: defaultServerOptions}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetOptions(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetOptions() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestMount(t *testing.T) { - type args struct { - pattern string - h http.Handler - } - tests := []struct { - name string - args args - wantErr bool - }{ - {name: "basic", args: args{ - pattern: "/", - h: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}), - }, wantErr: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.wantErr { - require.Error(t, Mount(tt.args.pattern, tt.args.h)) - } else { - require.NoError(t, Mount(tt.args.pattern, tt.args.h)) - } - assert.NotNil(t, defaultServer) - assert.True(t, defaultServer.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to match route after registering") - }) - if err := Shutdown(); err != nil { - t.Fatal(err) - } - } -} - -func TestNewServer(t *testing.T) { - type args struct { - listeners []net.Listener - tlsListeners []net.Listener - opt Options - } - listener, err := nettest.NewLocalListener("tcp") - if err != nil { - t.Fatal(err) - } - tests := []struct { - name string - args args - wantErr bool - }{ - {name: "default http", args: args{ - listeners: []net.Listener{listener}, - tlsListeners: []net.Listener{}, - opt: defaultServerOptions, - }, wantErr: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := NewServer(tt.args.listeners, tt.args.tlsListeners, tt.args.opt) - if (err != nil) != tt.wantErr { - t.Errorf("NewServer() error = %v, wantErr %v", err, tt.wantErr) - return - } - s, ok := got.(*server) - require.True(t, ok, "NewServer returned unexpected type") - if len(tt.args.listeners) > 0 { - assert.Equal(t, listener.Addr(), s.addrs[0]) - } else { - assert.Empty(t, s.addrs) - } - if len(tt.args.tlsListeners) > 0 { - assert.Equal(t, listener.Addr(), s.tlsAddrs[0]) - } else { - assert.Empty(t, s.tlsAddrs) - } - if tt.args.opt.BaseURL != "" { - assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter") - } else { - assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter") - } - if useSSL(tt.args.opt) { - assert.NotNil(t, s.httpServer.TLSConfig, "missing SSL config") - } else { - assert.Nil(t, s.httpServer.TLSConfig, "unexpectedly has SSL config") - } - }) - } -} - -func TestRestart(t *testing.T) { - tests := []struct { - name string - started bool - wantErr bool - }{ - {name: "started", started: true, wantErr: false}, - {name: "stopped", started: false, wantErr: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.started { - require.NoError(t, Restart()) // Call it twice basically - } else { - require.NoError(t, Shutdown()) - } - current := defaultServer - if err := Restart(); (err != nil) != tt.wantErr { - t.Errorf("Restart() error = %v, wantErr %v", err, tt.wantErr) - } - assert.NotNil(t, defaultServer, "failed to start default server") - assert.NotSame(t, current, defaultServer, "same server instance as before restart") - }) - } -} - -func TestRoute(t *testing.T) { - type args struct { - pattern string - fn func(r chi.Router) - } - tests := []struct { - name string - args args - test func(t *testing.T, r chi.Router) - }{ - { - name: "basic", - args: args{ - pattern: "/basic", - fn: func(r chi.Router) {}, - }, - test: func(t *testing.T, r chi.Router) { - require.Len(t, r.Routes(), 1) - assert.Equal(t, r.Routes()[0].Pattern, "/basic/*") - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.NoError(t, Restart()) - _, err := Route(tt.args.pattern, tt.args.fn) - require.NoError(t, err) - tt.test(t, defaultServer.baseRouter) - }) - - if err := Shutdown(); err != nil { - t.Fatal(err) - } - } -} - -func TestSetOptions(t *testing.T) { - type args struct { - opt Options - } - tests := []struct { - name string - args args - }{ - { - name: "basic", - args: args{opt: Options{ - ListenAddr: "127.0.0.1:9999", - BaseURL: "/basic", - ServerReadTimeout: 1, - ServerWriteTimeout: 1, - MaxHeaderBytes: 1, - }}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - SetOptions(tt.args.opt) - require.Equal(t, tt.args.opt, defaultServerOptions) - require.NoError(t, Restart()) - if useSSL(tt.args.opt) { - assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.tlsAddrs[0].String()) - } else { - assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.addrs[0].String()) - } - assert.Equal(t, tt.args.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout) - assert.Equal(t, tt.args.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout) - assert.Equal(t, tt.args.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes) - if tt.args.opt.BaseURL != "" && tt.args.opt.BaseURL != "/" { - assert.NotSame(t, defaultServer.httpServer.Handler, defaultServer.baseRouter, "BaseURL ignored") - } - }) - SetOptions(DefaultOpt) - } -} - -func TestShutdown(t *testing.T) { - tests := []struct { - name string - started bool - wantErr bool - }{ - {name: "started", started: true, wantErr: false}, - {name: "stopped", started: false, wantErr: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.started { - require.NoError(t, Restart()) - } else { - require.NoError(t, Shutdown()) // Call it twice basically - } - if err := Shutdown(); (err != nil) != tt.wantErr { - t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr) - } - assert.Nil(t, defaultServer, "default server not deleted") - }) - } -} - -func TestURL(t *testing.T) { - tests := []struct { - name string - want string - }{ - {name: "basic", want: "http://127.0.0.1:8080/"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.NoError(t, Restart()) - if got := URL(); got != tt.want { - t.Errorf("URL() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_server_Mount(t *testing.T) { - type args struct { - pattern string - h http.Handler - } - tests := []struct { - name string - args args - opt Options - }{ - {name: "basic", args: args{ - pattern: "/", - h: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}), - }, opt: defaultServerOptions}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - listener, err := nettest.NewLocalListener("tcp") - require.NoError(t, err) - s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt) - require.NoError(t, err2) - s.Mount(tt.args.pattern, tt.args.h) - srv, ok := s.(*server) - require.True(t, ok) - assert.NotNil(t, srv) - assert.True(t, srv.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to Match() route after registering") - }) - } -} - -func Test_server_Route(t *testing.T) { - type args struct { - pattern string - fn func(r chi.Router) - } - tests := []struct { - name string - args args - opt Options - test func(t *testing.T, r chi.Router) - }{ - { - name: "basic", - args: args{ - pattern: "/basic", - fn: func(r chi.Router) { - - }, - }, - test: func(t *testing.T, r chi.Router) { - require.Len(t, r.Routes(), 1) - assert.Equal(t, r.Routes()[0].Pattern, "/basic/*") - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - listener, err := nettest.NewLocalListener("tcp") - require.NoError(t, err) - s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt) - require.NoError(t, err2) - s.Route(tt.args.pattern, tt.args.fn) - srv, ok := s.(*server) - require.True(t, ok) - assert.NotNil(t, srv) - tt.test(t, srv.baseRouter) - }) - } -} - -func Test_server_Shutdown(t *testing.T) { - tests := []struct { - name string - opt Options - wantErr bool - }{ - { - name: "basic", - opt: defaultServerOptions, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - listener, err := nettest.NewLocalListener("tcp") - require.NoError(t, err) - s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt) - require.NoError(t, err2) - srv, ok := s.(*server) - require.True(t, ok) - if err := s.Shutdown(); (err != nil) != tt.wantErr { - t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr) - } - assert.EqualError(t, srv.httpServer.Serve(listener), http.ErrServerClosed.Error()) - }) - } -} - -func Test_start(t *testing.T) { - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - sslServerOptions := defaultServerOptions - sslServerOptions.SslCertBody = []byte(`-----BEGIN CERTIFICATE----- -MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw -DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow -EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d -7VNhbWvZLWPuj/RtHFjvtJBEwOkhbN/BnnE8rnZR8+sbwnc/KhCk3FhnpHZnQz7B -5aETbbIgmuvewdjvSBSjYzBhMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggr -BgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdEQQiMCCCDmxvY2FsaG9zdDo1 -NDUzgg4xMjcuMC4wLjE6NTQ1MzAKBggqhkjOPQQDAgNIADBFAiEA2zpJEPQyz6/l -Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc -6MF9+Yw1Yy0t ------END CERTIFICATE-----`) - sslServerOptions.SslKeyBody = []byte(`-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49 -AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q -EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA== ------END EC PRIVATE KEY-----`) - tests := []struct { - name string - opt Options - ssl bool - wantErr bool - }{ - { - name: "basic", - opt: defaultServerOptions, - wantErr: false, - }, - { - name: "ssl", - opt: sslServerOptions, - ssl: true, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - defer func() { - err := Shutdown() - if err != nil { - t.Fatal("couldn't shutdown server") - } - }() - SetOptions(tt.opt) - if err := start(); (err != nil) != tt.wantErr { - t.Errorf("start() error = %v, wantErr %v", err, tt.wantErr) - return - } - s := defaultServer - router := s.Router() - router.Head("/", func(writer http.ResponseWriter, request *http.Request) { - writer.WriteHeader(201) - }) - testURL := URL() - if tt.ssl { - assert.True(t, useSSL(tt.opt)) - assert.Equal(t, tt.opt.ListenAddr, s.tlsAddrs[0].String()) - assert.True(t, strings.HasPrefix(testURL, "https://")) - } else { - assert.True(t, strings.HasPrefix(testURL, "http://")) - assert.Equal(t, tt.opt.ListenAddr, s.addrs[0].String()) - } - - // try to connect to the test server - pause := time.Millisecond - for i := 0; i < 10; i++ { - resp, err := http.Head(testURL) - if err == nil { - _ = resp.Body.Close() - return - } - // t.Logf("couldn't connect, sleeping for %v: %v", pause, err) - time.Sleep(pause) - pause *= 2 - } - t.Fatal("couldn't connect to server") - - /* accessing s.httpServer.* can't be done synchronously and is a race condition - assert.Equal(t, tt.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout) - assert.Equal(t, tt.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout) - assert.Equal(t, tt.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes) - if tt.opt.BaseURL != "" && tt.opt.BaseURL != "/" { - assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter") - } else { - assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter") - } - if useSSL(tt.opt) { - require.NotNil(t, s.httpServer.TLSConfig, "missing SSL config") - assert.NotEmpty(t, s.httpServer.TLSConfig.Certificates, "missing SSL config") - } else if s.httpServer.TLSConfig != nil { - assert.Empty(t, s.httpServer.TLSConfig.Certificates, "unexpectedly has SSL config") - } - */ - }) - } -} - -func Test_useSSL(t *testing.T) { - type args struct { - opt Options - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "basic", - args: args{opt: Options{ - SslCert: "", - SslKey: "", - ClientCA: "", - }}, - want: false, - }, - { - name: "basic", - args: args{opt: Options{ - SslCert: "", - SslKey: "test", - ClientCA: "", - }}, - want: true, - }, - { - name: "body", - args: args{opt: Options{ - SslCert: "", - SslKey: "", - SslKeyBody: []byte(`test`), - ClientCA: "", - }}, - want: true, - }, - { - name: "basic", - args: args{opt: Options{ - SslCert: "", - SslKey: "test", - ClientCA: "", - MinTLSVersion: "tls1.2", - }}, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := useSSL(tt.args.opt); got != tt.want { - t.Errorf("useSSL() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/lib/http/middleware.go b/lib/http/middleware.go new file mode 100644 index 000000000..d19d75259 --- /dev/null +++ b/lib/http/middleware.go @@ -0,0 +1,171 @@ +package http + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "strings" + "sync" + + goauth "github.com/abbot/go-http-auth" + "github.com/rclone/rclone/fs" +) + +// parseAuthorization parses the Authorization header into user, pass +// it returns a boolean as to whether the parse was successful +func parseAuthorization(r *http.Request) (user, pass string, ok bool) { + authHeader := r.Header.Get("Authorization") + if authHeader != "" { + s := strings.SplitN(authHeader, " ", 2) + if len(s) == 2 && s[0] == "Basic" { + b, err := base64.StdEncoding.DecodeString(s[1]) + if err == nil { + parts := strings.SplitN(string(b), ":", 2) + user = parts[0] + if len(parts) > 1 { + pass = parts[1] + ok = true + } + } + } + } + return +} + +type LoggedBasicAuth struct { + goauth.BasicAuth +} + +// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests +func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string { + username := a.BasicAuth.CheckAuth(r) + if username == "" { + user, _, _ := parseAuthorization(r) + fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user) + } + return username +} + +// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator +func NewLoggedBasicAuthenticator(realm string, secrets goauth.SecretProvider) *LoggedBasicAuth { + return &LoggedBasicAuth{BasicAuth: goauth.BasicAuth{Realm: realm, Secrets: secrets}} +} + +// Helper to generate required interface for middleware +func basicAuth(authenticator *LoggedBasicAuth) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // skip auth for unix socket + if IsUnixSocket(r) { + next.ServeHTTP(w, r) + return + } + + username := authenticator.CheckAuth(r) + if username == "" { + authenticator.RequireAuth(w, r) + return + } + ctx := context.WithValue(r.Context(), ctxKeyUser, username) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} + +// MiddlewareAuthHtpasswd instantiates middleware that authenticates against the passed htpasswd file +func MiddlewareAuthHtpasswd(path, realm string) Middleware { + fs.Infof(nil, "Using %q as htpasswd storage", path) + secretProvider := goauth.HtpasswdFileProvider(path) + authenticator := NewLoggedBasicAuthenticator(realm, secretProvider) + return basicAuth(authenticator) +} + +// MiddlewareAuthBasic instantiates middleware that authenticates for a single user +func MiddlewareAuthBasic(user, pass, realm, salt string) Middleware { + fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user) + pass = string(goauth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$"))) + secretProvider := func(u, r string) string { + if user == u { + return pass + } + return "" + } + authenticator := NewLoggedBasicAuthenticator(realm, secretProvider) + return basicAuth(authenticator) +} + +// MiddlewareAuthCustom instantiates middleware that authenticates using a custom function +func MiddlewareAuthCustom(fn CustomAuthFn, realm string) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // skip auth for unix socket + if IsUnixSocket(r) { + next.ServeHTTP(w, r) + return + } + + user, pass, ok := parseAuthorization(r) + if !ok { + code := http.StatusUnauthorized + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm=%q, charset="UTF-8"`, realm)) + http.Error(w, http.StatusText(code), code) + return + } + + value, err := fn(user, pass) + if err != nil { + fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err) + goauth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting + return + } + + if value != nil { + r = r.WithContext(context.WithValue(r.Context(), ctxKeyAuth, value)) + } + + next.ServeHTTP(w, r) + }) + } +} + +var onlyOnceWarningAllowOrigin sync.Once + +// MiddlewareCORS instantiates middleware that handles basic CORS protections for rcd +func MiddlewareCORS(allowOrigin string) Middleware { + onlyOnceWarningAllowOrigin.Do(func() { + if allowOrigin == "*" { + fs.Logf(nil, "Warning: Allow origin set to *. This can cause serious security problems.") + } + }) + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // skip cors for unix sockets + if IsUnixSocket(r) { + next.ServeHTTP(w, r) + return + } + + if allowOrigin != "" { + w.Header().Add("Access-Control-Allow-Origin", allowOrigin) + } else { + w.Header().Add("Access-Control-Allow-Origin", PublicURL(r)) + } + + // echo back access control headers client needs + w.Header().Add("Access-Control-Request-Method", "POST, OPTIONS, GET, HEAD") + w.Header().Add("Access-Control-Allow-Headers", "authorization, Content-Type") + + next.ServeHTTP(w, r) + }) + } +} + +// MiddlewareStripPrefix instantiates middleware that removes the BaseURL from the path +func MiddlewareStripPrefix(prefix string) Middleware { + return func(next http.Handler) http.Handler { + return http.StripPrefix(prefix, next) + } +} diff --git a/lib/http/middleware_test.go b/lib/http/middleware_test.go new file mode 100644 index 000000000..4839b87e3 --- /dev/null +++ b/lib/http/middleware_test.go @@ -0,0 +1,231 @@ +package http + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMiddlewareAuth(t *testing.T) { + servers := []struct { + name string + http HTTPConfig + auth AuthConfig + user string + pass string + }{ + { + name: "Basic", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + auth: AuthConfig{ + Realm: "test", + BasicUser: "test", + BasicPass: "test", + }, + user: "test", + pass: "test", + }, + { + name: "Htpasswd/MD5", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + auth: AuthConfig{ + Realm: "test", + HtPasswd: "./testdata/.htpasswd", + }, + user: "md5", + pass: "md5", + }, + { + name: "Htpasswd/SHA", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + auth: AuthConfig{ + Realm: "test", + HtPasswd: "./testdata/.htpasswd", + }, + user: "sha", + pass: "sha", + }, + { + name: "Htpasswd/Bcrypt", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + auth: AuthConfig{ + Realm: "test", + HtPasswd: "./testdata/.htpasswd", + }, + user: "bcrypt", + pass: "bcrypt", + }, + { + name: "Custom", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + auth: AuthConfig{ + Realm: "test", + CustomAuthFn: func(user, pass string) (value interface{}, err error) { + if user == "custom" && pass == "custom" { + return true, nil + } + return nil, errors.New("invalid credentials") + }, + }, + user: "custom", + pass: "custom", + }, + } + + for _, ss := range servers { + t.Run(ss.name, func(t *testing.T) { + s, err := NewServer(context.Background(), WithConfig(ss.http), WithAuth(ss.auth)) + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + }() + + expected := []byte("secret-page") + s.Router().Mount("/", testEchoHandler(expected)) + s.Serve() + + url := testGetServerURL(t, s) + + t.Run("NoCreds", func(t *testing.T) { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using no creds should return unauthorized") + + wwwAuthHeader := resp.Header.Get("WWW-Authenticate") + require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header") + require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam") + }) + + t.Run("BadCreds", func(t *testing.T) { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + req.SetBasicAuth(ss.user+"BAD", ss.pass+"BAD") + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using bad creds should return unauthorized") + + wwwAuthHeader := resp.Header.Get("WWW-Authenticate") + require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header") + require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam") + }) + + t.Run("GoodCreds", func(t *testing.T) { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + req.SetBasicAuth(ss.user, ss.pass) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusOK, resp.StatusCode, "using good creds should return ok") + + testExpectRespBody(t, resp, expected) + }) + }) + } +} + +var _testCORSHeaderKeys = []string{ + "Access-Control-Allow-Origin", + "Access-Control-Request-Method", + "Access-Control-Allow-Headers", +} + +func TestMiddlewareCORS(t *testing.T) { + servers := []struct { + name string + http HTTPConfig + origin string + }{ + { + name: "EmptyOrigin", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + origin: "", + }, + { + name: "CustomOrigin", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + }, + origin: "http://test.rclone.org", + }, + } + + for _, ss := range servers { + t.Run(ss.name, func(t *testing.T) { + s, err := NewServer(context.Background(), WithConfig(ss.http)) + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + }() + + s.Router().Use(MiddlewareCORS(ss.origin)) + + expected := []byte("data") + s.Router().Mount("/", testEchoHandler(expected)) + s.Serve() + + url := testGetServerURL(t, s) + + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok") + + testExpectRespBody(t, resp, expected) + + for _, key := range _testCORSHeaderKeys { + require.Contains(t, resp.Header, key, "CORS headers should be sent") + } + + expectedOrigin := url + if ss.origin != "" { + expectedOrigin = ss.origin + } + require.Equal(t, expectedOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "allow origin should match") + }) + } +} diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go index be128ae33..bdcf3f948 100644 --- a/lib/http/serve/dir_test.go +++ b/lib/http/serve/dir_test.go @@ -3,7 +3,7 @@ package serve import ( "errors" "html/template" - "io" + "io/ioutil" "net/http" "net/http/httptest" "net/url" @@ -94,7 +94,7 @@ func TestError(t *testing.T) { Error("potato", w, "sausage", err) resp := w.Result() assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "sausage.\n", string(body)) } @@ -108,7 +108,7 @@ func TestServe(t *testing.T) { d.Serve(w, r) resp := w.Result() assert.Equal(t, http.StatusOK, resp.StatusCode) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, ` diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go index b6cc70976..934fabe65 100644 --- a/lib/http/serve/serve_test.go +++ b/lib/http/serve/serve_test.go @@ -1,7 +1,7 @@ package serve import ( - "io" + "io/ioutil" "net/http" "net/http/httptest" "testing" @@ -17,7 +17,7 @@ func TestObjectBadMethod(t *testing.T) { Object(w, r, o) resp := w.Result() assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "Method Not Allowed\n", string(body)) } @@ -30,7 +30,7 @@ func TestObjectHEAD(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "5", resp.Header.Get("Content-Length")) assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges")) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "", string(body)) } @@ -43,7 +43,7 @@ func TestObjectGET(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "5", resp.Header.Get("Content-Length")) assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges")) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "hello", string(body)) } @@ -58,7 +58,7 @@ func TestObjectRange(t *testing.T) { assert.Equal(t, "3", resp.Header.Get("Content-Length")) assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges")) assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range")) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "345", string(body)) } @@ -71,6 +71,6 @@ func TestObjectBadRange(t *testing.T) { resp := w.Result() assert.Equal(t, http.StatusBadRequest, resp.StatusCode) assert.Equal(t, "10", resp.Header.Get("Content-Length")) - body, _ := io.ReadAll(resp.Body) + body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, "Bad Request\n", string(body)) } diff --git a/lib/http/server.go b/lib/http/server.go new file mode 100644 index 000000000..d4e5e827b --- /dev/null +++ b/lib/http/server.go @@ -0,0 +1,430 @@ +// Package http provides a registration interface for http services +package http + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "html/template" + "log" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/go-chi/chi/v5" + "github.com/rclone/rclone/fs/config/flags" + "github.com/spf13/pflag" +) + +// Help contains text describing the http server to add to the command +// help. +var Help = ` +### Server options + +Use ` + "`--addr`" + ` to specify which IP address and port the server should +listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +You can use a unix socket by setting the url to ` + "`unix:///path/to/socket`" + ` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +` + "`--addr`" + ` may be repeated to listen on multiple IPs/ports/sockets. + +` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will +accept in the HTTP header. + +` + "`--baseurl`" + ` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used ` + "`--baseurl \"/rclone\"`" + ` then +rclone would serve from a URL starting with "/rclone/". This is +useful if you wish to proxy rclone serve. Rclone automatically +inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `, +` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated +identically. + +#### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the ` + "`--cert` and `--key`" + ` flags. +If you wish to do client side certificate validation then you will need to +supply ` + "`--client-ca`" + ` also. + +` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded +private key and ` + "`--client-ca`" + ` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). +` + +// Middleware function signature required by chi.Router.Use() +type Middleware func(http.Handler) http.Handler + +// HTTPConfig contains options for the http Server +type HTTPConfig struct { + ListenAddr []string // Port to listen on + BaseURL string // prefix to strip from URLs + ServerReadTimeout time.Duration // Timeout for server reading data + ServerWriteTimeout time.Duration // Timeout for server writing data + MaxHeaderBytes int // Maximum size of request header + TLSCert string // Path to TLS PEM key (concatenation of certificate and CA certificate) + TLSKey string // Path to TLS PEM Private key + TLSCertBody []byte // TLS PEM key (concatenation of certificate and CA certificate) body, ignores TLSCert + TLSKeyBody []byte // TLS PEM Private key body, ignores TLSKey + ClientCA string // Client certificate authority to verify clients with + MinTLSVersion string // MinTLSVersion contains the minimum TLS version that is acceptable. + Template string +} + +// AddFlagsPrefix adds flags for the httplib +func (cfg *HTTPConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) { + flags.StringArrayVarP(flagSet, &cfg.ListenAddr, prefix+"addr", "", cfg.ListenAddr, "IPaddress:Port or :Port to bind server to") + flags.DurationVarP(flagSet, &cfg.ServerReadTimeout, prefix+"server-read-timeout", "", cfg.ServerReadTimeout, "Timeout for server reading data") + flags.DurationVarP(flagSet, &cfg.ServerWriteTimeout, prefix+"server-write-timeout", "", cfg.ServerWriteTimeout, "Timeout for server writing data") + flags.IntVarP(flagSet, &cfg.MaxHeaderBytes, prefix+"max-header-bytes", "", cfg.MaxHeaderBytes, "Maximum size of request header") + flags.StringVarP(flagSet, &cfg.TLSCert, prefix+"cert", "", cfg.TLSCert, "TLS PEM key (concatenation of certificate and CA certificate)") + flags.StringVarP(flagSet, &cfg.TLSKey, prefix+"key", "", cfg.TLSKey, "TLS PEM Private key") + flags.StringVarP(flagSet, &cfg.ClientCA, prefix+"client-ca", "", cfg.ClientCA, "Client certificate authority to verify clients with") + flags.StringVarP(flagSet, &cfg.BaseURL, prefix+"baseurl", "", cfg.BaseURL, "Prefix for URLs - leave blank for root") + flags.StringVarP(flagSet, &cfg.MinTLSVersion, prefix+"min-tls-version", "", cfg.MinTLSVersion, "Minimum TLS version that is acceptable") +} + +// AddHTTPFlagsPrefix adds flags for the httplib +func AddHTTPFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *HTTPConfig) { + cfg.AddFlagsPrefix(flagSet, prefix) +} + +// DefaultHTTPCfg is the default values used for Config +func DefaultHTTPCfg() HTTPConfig { + return HTTPConfig{ + ListenAddr: []string{"127.0.0.1:8080"}, + ServerReadTimeout: 1 * time.Hour, + ServerWriteTimeout: 1 * time.Hour, + MaxHeaderBytes: 4096, + MinTLSVersion: "tls1.0", + } +} + +// Server interface of http server +type Server interface { + Router() chi.Router + Serve() + Shutdown() error + HTMLTemplate() *template.Template + URLs() []string + Wait() +} + +type instance struct { + url string + listener net.Listener + httpServer *http.Server +} + +func (s instance) serve(wg *sync.WaitGroup) { + defer wg.Done() + err := s.httpServer.Serve(s.listener) + if err != http.ErrServerClosed && err != nil { + log.Printf("%s: unexpected error: %s", s.listener.Addr(), err.Error()) + } +} + +type server struct { + wg sync.WaitGroup + mux chi.Router + tlsConfig *tls.Config + instances []instance + auth AuthConfig + cfg HTTPConfig + template *TemplateConfig + htmlTemplate *template.Template +} + +// Option allows customizing the server +type Option func(*server) + +// WithAuth option initializes the appropriate auth middleware +func WithAuth(cfg AuthConfig) Option { + return func(s *server) { + s.auth = cfg + } +} + +// WithConfig option applies the HTTPConfig to the server, overriding defaults +func WithConfig(cfg HTTPConfig) Option { + return func(s *server) { + s.cfg = cfg + } +} + +// WithTemplate option allows the parsing of a template +func WithTemplate(cfg TemplateConfig) Option { + return func(s *server) { + s.template = &cfg + } +} + +// NewServer instantiates a new http server using provided listeners and options +// This function is provided if the default http server does not meet a services requirements and should not generally be used +// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443. +// tlsListeners are ignored if opt.TLSKey is not provided +func NewServer(ctx context.Context, options ...Option) (*server, error) { + s := &server{ + mux: chi.NewRouter(), + cfg: DefaultHTTPCfg(), + } + + for _, opt := range options { + opt(s) + } + + // Build base router + s.mux.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + }) + s.mux.NotFound(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + }) + + // Ignore passing "/" for BaseURL + s.cfg.BaseURL = strings.Trim(s.cfg.BaseURL, "/") + if s.cfg.BaseURL != "" { + s.cfg.BaseURL = "/" + s.cfg.BaseURL + s.mux.Use(MiddlewareStripPrefix(s.cfg.BaseURL)) + } + + s.initAuth() + + err := s.initTemplate() + if err != nil { + return nil, err + } + + err = s.initTLS() + if err != nil { + return nil, err + } + + for _, addr := range s.cfg.ListenAddr { + var url string + var network = "tcp" + var tlsCfg *tls.Config + + if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) { + network = "unix" + addr = strings.TrimPrefix(addr, "unix://") + url = addr + + } else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) { + tlsCfg = s.tlsConfig + addr = strings.TrimPrefix(addr, "tls://") + } + + var listener net.Listener + if tlsCfg == nil { + listener, err = net.Listen(network, addr) + } else { + listener, err = tls.Listen(network, addr, tlsCfg) + } + if err != nil { + return nil, err + } + + if network == "tcp" { + var secure string + if tlsCfg != nil { + secure = "s" + } + url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL) + } + + ii := instance{ + url: url, + listener: listener, + httpServer: &http.Server{ + Handler: s.mux, + ReadTimeout: s.cfg.ServerReadTimeout, + WriteTimeout: s.cfg.ServerWriteTimeout, + MaxHeaderBytes: s.cfg.MaxHeaderBytes, + ReadHeaderTimeout: 10 * time.Second, // time to send the headers + IdleTimeout: 60 * time.Second, // time to keep idle connections open + TLSConfig: tlsCfg, + BaseContext: NewBaseContext(ctx, url), + }, + } + + s.instances = append(s.instances, ii) + } + + return s, nil +} + +func (s *server) initAuth() { + if s.auth.CustomAuthFn != nil { + s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm)) + return + } + + if s.auth.HtPasswd != "" { + s.mux.Use(MiddlewareAuthHtpasswd(s.auth.HtPasswd, s.auth.Realm)) + return + } + + if s.auth.BasicUser != "" { + s.mux.Use(MiddlewareAuthBasic(s.auth.BasicUser, s.auth.BasicPass, s.auth.Realm, s.auth.Salt)) + return + } +} + +func (s *server) initTemplate() error { + if s.template == nil { + return nil + } + + var err error + s.htmlTemplate, err = GetTemplate(s.template.Path) + if err != nil { + err = fmt.Errorf("failed to get template: %w", err) + } + + return err +} + +var ( + // hard coded errors, allowing for easier testing + ErrInvalidMinTLSVersion = errors.New("invalid value for --min-tls-version") + ErrTLSBodyMismatch = errors.New("need both TLSCertBody and TLSKeyBody to use TLS") + ErrTLSFileMismatch = errors.New("need both --cert and --key to use TLS") + ErrTLSParseCA = errors.New("unable to parse client certificate authority") +) + +func (s *server) initTLS() error { + if s.cfg.TLSCert == "" && s.cfg.TLSKey == "" && len(s.cfg.TLSCertBody) == 0 && len(s.cfg.TLSKeyBody) == 0 { + return nil + } + + if (len(s.cfg.TLSCertBody) > 0) != (len(s.cfg.TLSKeyBody) > 0) { + return ErrTLSBodyMismatch + } + + if (s.cfg.TLSCert != "") != (s.cfg.TLSKey != "") { + return ErrTLSFileMismatch + } + + var cert tls.Certificate + var err error + if len(s.cfg.TLSCertBody) > 0 { + cert, err = tls.X509KeyPair(s.cfg.TLSCertBody, s.cfg.TLSKeyBody) + } else { + cert, err = tls.LoadX509KeyPair(s.cfg.TLSCert, s.cfg.TLSKey) + } + if err != nil { + return err + } + + var minTLSVersion uint16 + switch s.cfg.MinTLSVersion { + case "tls1.0": + minTLSVersion = tls.VersionTLS10 + case "tls1.1": + minTLSVersion = tls.VersionTLS11 + case "tls1.2": + minTLSVersion = tls.VersionTLS12 + case "tls1.3": + minTLSVersion = tls.VersionTLS13 + default: + return fmt.Errorf("%w: %s", ErrInvalidMinTLSVersion, s.cfg.MinTLSVersion) + } + + s.tlsConfig = &tls.Config{ + MinVersion: minTLSVersion, + Certificates: []tls.Certificate{cert}, + } + + if s.cfg.ClientCA != "" { + // if !useTLS { + // err := errors.New("can't use --client-ca without --cert and --key") + // log.Fatalf(err.Error()) + // } + certpool := x509.NewCertPool() + pem, err := os.ReadFile(s.cfg.ClientCA) + if err != nil { + return err + } + + if !certpool.AppendCertsFromPEM(pem) { + return ErrTLSParseCA + } + + s.tlsConfig.ClientCAs = certpool + s.tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + } + + return nil +} + +// Serve starts the HTTP server on each listener +func (s *server) Serve() { + s.wg.Add(len(s.instances)) + for _, ii := range s.instances { + // TODO: decide how/when to log listening url + // log.Printf("listening on %s", ii.url) + go ii.serve(&s.wg) + } +} + +// Wait blocks while the server is serving requests +func (s *server) Wait() { + s.wg.Wait() +} + +// Router returns the server base router +func (s *server) Router() chi.Router { + return s.mux +} + +// Shutdown gracefully shuts down the server +func (s *server) Shutdown() error { + ctx := context.Background() + for _, ii := range s.instances { + if err := ii.httpServer.Shutdown(ctx); err != nil { + log.Printf("error shutting down server: %s", err) + continue + } + } + s.wg.Wait() + return nil +} + +// HTMLTemplate returns the parsed template, if WithTemplate option was passed. +func (s *server) HTMLTemplate() *template.Template { + return s.htmlTemplate +} + +// URLs returns all configured URLS +func (s *server) URLs() []string { + var out []string + for _, ii := range s.instances { + if ii.listener.Addr().Network() == "unix" { + continue + } + out = append(out, ii.url) + } + return out +} diff --git a/lib/http/server_test.go b/lib/http/server_test.go new file mode 100644 index 000000000..884214f43 --- /dev/null +++ b/lib/http/server_test.go @@ -0,0 +1,444 @@ +package http + +import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func testEchoHandler(data []byte) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(data) + }) +} + +func testExpectRespBody(t *testing.T, resp *http.Response, expected []byte) { + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, expected, body) +} + +func testGetServerURL(t *testing.T, s Server) string { + urls := s.URLs() + require.GreaterOrEqual(t, len(urls), 1, "server should return at least one url") + return urls[0] +} + +func testNewHTTPClientUnix(path string) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", path) + }, + }, + } +} + +func testReadTestdataFile(t *testing.T, path string) []byte { + data, err := os.ReadFile(filepath.Join("./testdata", path)) + require.NoError(t, err, "") + return data +} + +func TestNewServerUnix(t *testing.T) { + ctx := context.Background() + + tempDir := t.TempDir() + path := filepath.Join(tempDir, "rclone.sock") + + cfg := DefaultHTTPCfg() + cfg.ListenAddr = []string{path} + + auth := AuthConfig{ + BasicUser: "test", + BasicPass: "test", + } + + s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth)) + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "shutdown should remove socket") + }() + + require.Empty(t, s.URLs(), "unix socket should not appear in URLs") + + s.Router().Use(MiddlewareCORS("")) + + expected := []byte("hello world") + s.Router().Mount("/", testEchoHandler(expected)) + s.Serve() + + client := testNewHTTPClientUnix(path) + req, err := http.NewRequest("GET", "http://unix", nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + + testExpectRespBody(t, resp, expected) + + require.Equal(t, http.StatusOK, resp.StatusCode, "unix sockets should ignore auth") + + for _, key := range _testCORSHeaderKeys { + require.NotContains(t, resp.Header, key, "unix sockets should not be sent CORS headers") + } +} + +func TestNewServerHTTP(t *testing.T) { + ctx := context.Background() + + cfg := DefaultHTTPCfg() + cfg.ListenAddr = []string{"127.0.0.1:0"} + + auth := AuthConfig{ + BasicUser: "test", + BasicPass: "test", + } + + s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth)) + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + }() + + url := testGetServerURL(t, s) + require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme") + + expected := []byte("hello world") + s.Router().Mount("/", testEchoHandler(expected)) + s.Serve() + + t.Run("StatusUnauthorized", func(t *testing.T) { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "no basic auth creds should return unauthorized") + }) + + t.Run("StatusOK", func(t *testing.T) { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + req.SetBasicAuth(auth.BasicUser, auth.BasicPass) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusOK, resp.StatusCode, "using basic auth creds should return ok") + + testExpectRespBody(t, resp, expected) + }) +} +func TestNewServerBaseURL(t *testing.T) { + servers := []struct { + name string + http HTTPConfig + suffix string + }{ + { + name: "Empty", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + BaseURL: "", + }, + suffix: "/", + }, + { + name: "Single/NoTrailingSlash", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + BaseURL: "/rclone", + }, + suffix: "/rclone/", + }, + { + name: "Single/TrailingSlash", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + BaseURL: "/rclone/", + }, + suffix: "/rclone/", + }, + { + name: "Multi/NoTrailingSlash", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + BaseURL: "/rclone/test/base/url", + }, + suffix: "/rclone/test/base/url/", + }, + { + name: "Multi/TrailingSlash", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + BaseURL: "/rclone/test/base/url/", + }, + suffix: "/rclone/test/base/url/", + }, + } + + for _, ss := range servers { + t.Run(ss.name, func(t *testing.T) { + s, err := NewServer(context.Background(), WithConfig(ss.http)) + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + }() + + expected := []byte("data") + s.Router().Get("/", testEchoHandler(expected).ServeHTTP) + s.Serve() + + url := testGetServerURL(t, s) + require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme") + require.True(t, strings.HasSuffix(url, ss.suffix), "url should have the expected suffix") + + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + t.Log(url, resp.Request.URL) + + require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok") + + testExpectRespBody(t, resp, expected) + }) + } +} + +func TestNewServerTLS(t *testing.T) { + certBytes := testReadTestdataFile(t, "local.crt") + keyBytes := testReadTestdataFile(t, "local.key") + + // TODO: generate a proper cert with SAN + // TODO: generate CA, test mTLS + // clientCert, err := tls.X509KeyPair(certBytes, keyBytes) + // require.NoError(t, err, "should be testing with a valid self signed certificate") + + servers := []struct { + name string + wantErr bool + err error + http HTTPConfig + }{ + { + name: "FromFile/Valid", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCert: "./testdata/local.crt", + TLSKey: "./testdata/local.key", + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromFile/NoCert", + wantErr: true, + err: ErrTLSFileMismatch, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCert: "", + TLSKey: "./testdata/local.key", + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromFile/InvalidCert", + wantErr: true, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCert: "./testdata/local.crt.invalid", + TLSKey: "./testdata/local.key", + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromFile/NoKey", + wantErr: true, + err: ErrTLSFileMismatch, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCert: "./testdata/local.crt", + TLSKey: "", + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromFile/InvalidKey", + wantErr: true, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCert: "./testdata/local.crt", + TLSKey: "./testdata/local.key.invalid", + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromBody/Valid", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromBody/NoCert", + wantErr: true, + err: ErrTLSBodyMismatch, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: nil, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromBody/InvalidCert", + wantErr: true, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: []byte("JUNK DATA"), + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromBody/NoKey", + wantErr: true, + err: ErrTLSBodyMismatch, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: nil, + MinTLSVersion: "tls1.0", + }, + }, + { + name: "FromBody/InvalidKey", + wantErr: true, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: []byte("JUNK DATA"), + MinTLSVersion: "tls1.0", + }, + }, + { + name: "MinTLSVersion/Valid/1.1", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.1", + }, + }, + { + name: "MinTLSVersion/Valid/1.2", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.2", + }, + }, + { + name: "MinTLSVersion/Valid/1.3", + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls1.3", + }, + }, + { + name: "MinTLSVersion/Invalid", + wantErr: true, + err: ErrInvalidMinTLSVersion, + http: HTTPConfig{ + ListenAddr: []string{"127.0.0.1:0"}, + TLSCertBody: certBytes, + TLSKeyBody: keyBytes, + MinTLSVersion: "tls0.9", + }, + }, + } + + for _, ss := range servers { + t.Run(ss.name, func(t *testing.T) { + s, err := NewServer(context.Background(), WithConfig(ss.http)) + if ss.wantErr == true { + if ss.err != nil { + require.ErrorIs(t, err, ss.err, "new server should return the expected error") + } else { + require.Error(t, err, "new server should return error for invalid TLS config") + } + return + } + + require.NoError(t, err) + defer func() { + require.NoError(t, s.Shutdown()) + }() + + expected := []byte("secret-page") + s.Router().Mount("/", testEchoHandler(expected)) + s.Serve() + + url := testGetServerURL(t, s) + require.True(t, strings.HasPrefix(url, "https://"), "url should have https scheme") + + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + dest := strings.TrimPrefix(url, "https://") + dest = strings.TrimSuffix(dest, "/") + return net.Dial("tcp", dest) + }, + TLSClientConfig: &tls.Config{ + // Certificates: []tls.Certificate{clientCert}, + InsecureSkipVerify: true, + }, + }, + } + req, err := http.NewRequest("GET", "https://dev.rclone.org", nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer func() { + _ = resp.Body.Close() + }() + + require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok") + + testExpectRespBody(t, resp, expected) + }) + } +} diff --git a/lib/http/template.go b/lib/http/template.go new file mode 100644 index 000000000..bed80dc30 --- /dev/null +++ b/lib/http/template.go @@ -0,0 +1,95 @@ +package http + +import ( + "embed" + "html/template" + "os" + "time" + + "github.com/spf13/pflag" + + "github.com/rclone/rclone/fs/config/flags" +) + +// TemplateHelp describes how to use a custom template +var TemplateHelp = ` +#### Template + +` + "`--template`" + ` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup +to be used within the template to server pages: + +| Parameter | Description | +| :---------- | :---------- | +| .Name | The full path of a file/directory. | +| .Title | Directory listing of .Name | +| .Sort | The current sort used. This is changeable via ?sort= parameter | +| | Sort Options: namedirfirst,name,size,time (default namedirfirst) | +| .Order | The current ordering used. This is changeable via ?order= parameter | +| | Order Options: asc,desc (default asc) | +| .Query | Currently unused. | +| .Breadcrumb | Allows for creating a relative navigation | +|-- .Link | The relative to the root link of the Text. | +|-- .Text | The Name of the directory. | +| .Entries | Information about a specific file/directory. | +|-- .URL | The 'url' of an entry. | +|-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | +|-- .IsDir | Boolean for if an entry is a directory or not. | +|-- .Size | Size in Bytes of the entry. | +|-- .ModTime | The UTC timestamp of an entry. | +` + +// TemplateConfig for the templating functionality +type TemplateConfig struct { + Path string +} + +// AddFlagsPrefix for the templating functionality +func (cfg *TemplateConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) { + flags.StringVarP(flagSet, &cfg.Path, prefix+"template", "", cfg.Path, "User-specified template") +} + +// AddTemplateFlagsPrefix for the templating functionality +func AddTemplateFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *TemplateConfig) { + cfg.AddFlagsPrefix(flagSet, prefix) +} + +// DefaultTemplateCfg returns a new config which can be customized by command line flags +func DefaultTemplateCfg() TemplateConfig { + return TemplateConfig{} +} + +// AfterEpoch returns the time since the epoch for the given time +func AfterEpoch(t time.Time) bool { + return t.After(time.Time{}) +} + +// Assets holds the embedded filesystem for the default template +// +//go:embed templates +var Assets embed.FS + +// GetTemplate returns the HTML template for serving directories via HTTP/WebDAV +func GetTemplate(tmpl string) (*template.Template, error) { + var readFile = os.ReadFile + if tmpl == "" { + tmpl = "templates/index.html" + readFile = Assets.ReadFile + } + + data, err := readFile(tmpl) + if err != nil { + return nil, err + } + + funcMap := template.FuncMap{ + "afterEpoch": AfterEpoch, + } + + tpl, err := template.New("index").Funcs(funcMap).Parse(string(data)) + if err != nil { + return nil, err + } + + return tpl, nil +} diff --git a/lib/http/templates/index.html b/lib/http/templates/index.html new file mode 100644 index 000000000..348050c02 --- /dev/null +++ b/lib/http/templates/index.html @@ -0,0 +1,389 @@ + + + + + + {{html .Name}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

+ {{range $i, $crumb := .Breadcrumb}}{{html $crumb.Text}}{{if ne $i 0}}/{{end}}{{end}} +

+
+
+
+
+ +
+
+
+ + + + + + + + + + + + + + + + + + + {{- range .Entries}} + + + + {{- if .IsDir}} + + {{- else}} + + {{- end}} + {{- if .ModTime | afterEpoch }} + + {{- else}} + + {{- end}} + + + {{- end}} + +
+ + + Name + + Size + + Modified +
+ + Go up + + ——
+ + {{- if .IsDir}} + + {{- else}} + + {{- end}} + {{html .Leaf}} + —{{.Size}}—
+
+
+ + + diff --git a/lib/http/testdata/.htpasswd b/lib/http/testdata/.htpasswd new file mode 100644 index 000000000..5d0c17779 --- /dev/null +++ b/lib/http/testdata/.htpasswd @@ -0,0 +1,3 @@ +sha:{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY= +md5:$apr1$s7fogein$IK9ItbnGM14ct0bY4Uyik1 +bcrypt:$2y$10$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2 \ No newline at end of file diff --git a/lib/http/testdata/local.crt b/lib/http/testdata/local.crt new file mode 100644 index 000000000..c2bcf28fe --- /dev/null +++ b/lib/http/testdata/local.crt @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFeTCCA2GgAwIBAgIUewDoUKLIPBZlSAdOaBHPtfFDiUYwDQYJKoZIhvcNAQEL +BQAwTDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBnJjbG9uZTETMBEGA1UECwwKcmNs +b25lLWRldjEXMBUGA1UEAwwOZGV2LnJjbG9uZS5vcmcwHhcNMjIxMDI3MDM1NjE2 +WhcNMzIxMDI0MDM1NjE2WjBMMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGcmNsb25l +MRMwEQYDVQQLDApyY2xvbmUtZGV2MRcwFQYDVQQDDA5kZXYucmNsb25lLm9yZzCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJVMPbsmBVBi2DG17cPrJQxM +hdUxkd8pcRtrKkLzqIQS18IKJ20BTzrVlJJETO1uMeXETSVVQ4znWLvhZo5Pgq72 +FlvEMEl6QXNNihEB+Bx9f6iArS0Tgo919Kg3JEYu9g3HfsPxt3C1xNJOlSrpPo4w +YAX8MU+uy3IqPwgURVhPauR/2E64b9RSIsmNCJJXWnREeJVOpeYSUXx6S0fNM/wo +BvbPSuB1BErYug9AfuvTavJO7VYnMuEEig/1DXYnPJpTTaZWzyhQIGDU52T5iOZk +sdxv/Yxbq1kKDHmyhG9ALpCHqQMmR3bOEhLijU5UHfFR03eKnzfcQ7y5BMJBDIVs +fbaBxlJZFHKq9ixUT/7crc1iUXb+gPg/FxMRuR96/fTPrluWlDYaZV3QMa3rxuvn +PJM5boqGIH9fNxnsmho0V4ZrXgxg5VsnlwHZIlMuJVJGkUQFTl6XhIzkQRmjQM4C +WO9KPSusvCM3gCM7j9Tyhi3xl7XRVlN9xb8vbnbyqcZe0lgJU/y3nrbWqORvUKb0 +FvarmtIH3f9yqu+UB2s8DIg5zPjVmTnIMSJYJi5Cjh1YT2YVJ4+JJ0KlxAwOVTU4 +zdup3fUA3Ne2o9ehJEJwuFPbMDCEHeIuTzOKleKCDbHRIvFLleJP4J9/qA1P2C2/ +ZAoS4M8vgd2O84eVm7pzAgMBAAGjUzBRMB0GA1UdDgQWBBTDsVFbaf4C4yDDOGml +BMyKSgsT5DAfBgNVHSMEGDAWgBTDsVFbaf4C4yDDOGmlBMyKSgsT5DAPBgNVHRMB +Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAx2wDldW5wZfBGdIv9UCTL1hdB +a858v+mWCAjXHaFCyMAe2UOBkSF4M8oIqgMXP3ZwysAhB54qKcpKgxKv6s5oyA/k +diheH8Zl7GYnUS+A1v9ZUx5AwJbVNKCSXYfXQxTsoC1jV0W3HqeJIoxix3XJDZIL +TCiA8+BMk72vtJBPNlV8wmuN2+aUEktZF1PqjbqU/6ajRvDIa4ogXIVoSunpFXt9 +h5Af2zOE051zqjqshr9Egz6Hl2EItjJgqZtwLD8qSZFzcXYgDvtG16YceGt+ljeC +yLYE8Qvm9lfQka9Sxucu4+2kzCwjg5ubFaaSkiX6b/ue9KvPHm36JAV9NDwyyA/q +BNNxK+0PniNPLBdxIkFZvVraLieDNCXV3cqH7781IP2PRvbzNAR1PFfo19U7bmHV +PsOBj9kIQBhOLcxtxWvK93ptC6vLJYRsPha7kClVI5kht9oB8Mkfi2mAu2Pi7Pka +ZYHl14XnJPUWEbxX26I4CAU09yGjnQhRwPfGGNPCaMMGsYl+2nn4j/rzbQ8uz7Kg +l1TQS4WBpX4T6pxWM/mWERbBTLxniE8DNYPgHpgZJSXD16uG/ksyjdet7B86KA0v +kymH793pOqW8rtrKbziSZyShJy5AYsGy0Xu4ymW03F8S5FUyjWIkRmyUbGTB1q02 +nniyTJh2BffUcH2iCg== +-----END CERTIFICATE----- diff --git a/lib/http/testdata/local.key b/lib/http/testdata/local.key new file mode 100644 index 000000000..e855ac9fe --- /dev/null +++ b/lib/http/testdata/local.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCVTD27JgVQYtgx +te3D6yUMTIXVMZHfKXEbaypC86iEEtfCCidtAU861ZSSREztbjHlxE0lVUOM51i7 +4WaOT4Ku9hZbxDBJekFzTYoRAfgcfX+ogK0tE4KPdfSoNyRGLvYNx37D8bdwtcTS +TpUq6T6OMGAF/DFPrstyKj8IFEVYT2rkf9hOuG/UUiLJjQiSV1p0RHiVTqXmElF8 +ektHzTP8KAb2z0rgdQRK2LoPQH7r02ryTu1WJzLhBIoP9Q12JzyaU02mVs8oUCBg +1Odk+YjmZLHcb/2MW6tZCgx5soRvQC6Qh6kDJkd2zhIS4o1OVB3xUdN3ip833EO8 +uQTCQQyFbH22gcZSWRRyqvYsVE/+3K3NYlF2/oD4PxcTEbkfev30z65blpQ2GmVd +0DGt68br5zyTOW6KhiB/XzcZ7JoaNFeGa14MYOVbJ5cB2SJTLiVSRpFEBU5el4SM +5EEZo0DOAljvSj0rrLwjN4AjO4/U8oYt8Ze10VZTfcW/L2528qnGXtJYCVP8t562 +1qjkb1Cm9Bb2q5rSB93/cqrvlAdrPAyIOcz41Zk5yDEiWCYuQo4dWE9mFSePiSdC +pcQMDlU1OM3bqd31ANzXtqPXoSRCcLhT2zAwhB3iLk8zipXigg2x0SLxS5XiT+Cf +f6gNT9gtv2QKEuDPL4HdjvOHlZu6cwIDAQABAoICAD7EzqFb01kgLZf8zqmTt8BL +fesLy7IA6OpnrF14tq1MhMSyYzALoGVyfWPfbl5WeYkJ9otPJTbc3ywikG0dlap8 +kRrkyY5i5ZiWDYmoA8nqo5zS+LweW0J4i7Obd1dAkDdr2+qCuiabbVQkMMfZR3Ed +eomZpZvEOAnYJCb/6sW9ognOjEFQfsfL/o8xidyI+GEwlmfjqJEpu3OzsOnPpt8J +byAeN/NVj0fuhY87BQGeIfUc5ODXWydKsscRtqapyWtywY9BKRhgU7SSXnTQCtQe +mr68oON9ePVW6bbSrKZfXBRszMyjr+ENs4CYGmPHrs0SI2+7asRgCWSTfyIymhHs +DjF6BHAlQaOLnVcffuGhc6c0bnzIFvg1JenWVj9uQSkADW8TMkNgMLdcx8JoMd30 +9HiHZyQy7hMuzWjZSgS1wRp3KMGBo+WuMd4NF662nQPfylG2iJvedLtbvY7mBGkX +1e6L4b1fkFAZWyfBj3JX1SqgcNDnqzDiqqUDwpY8b1G9nAegvWBeFbEgnORa6pVF +8xWNtWlMsXYjTvPLbvDSZc6l0wOpFw7roGZgQpbaTjmbeqrZMsr/XDtH7ksLCBOh +V/s2oMU0NUBu/mokvpLupBzVMbA7/TKGr36OrHw57P8dhsdD3bqx+UbUMgjsq6xs +UAMcjWJFoc3ulgsVHF2BAoIBAQDEo5i63JkQL3+8rXQ25StbEgqVGXcGS2oZLu+G +GYklJ0NyGrjoYWmZ/QQf9NbAhQKDSe/X2cWtTLMF9Y2yVsxQ2x1df3lonk9dHsoX +O+Yp8C9zKt24dRWUsyklgkxAmb/nq3dRPtm/80ztNVQvYy5SQ7G5s/suXt9Wyd2y +nl08ZWclaDkHNS33/8hYLHP3RkjLgcOu45UDb/3vXk6ljnlrnqe7nlRX5UrnVXs3 +K41xJpCSHmUe7SUoW1ExjzWhtQBkWWVWWV8gE8CNpU0nheTkA5KyXQ8QfQ13jxJj +zPy/3OPGMIElbvEBcXxvcRncpuUIVO4tRy+oZuz2bmLwbBiBAoIBAQDCXhS5wMgJ +fWNzsd9prFrENVz7CR2NK3LSdVpm4ZtR5f4mLyzYj/RcSIYC4U7uO0WXaoLTnhKs +KbKAj5KC2HZUcqhLscajXPl6OjBOcEAlujh9VeTjEnx6XU81IwmAjrYr98xAz3HJ +IOaD8MvfZ13JWDB9q5bq3A8EKTEAPh66mv/w4Ic8rnueFZgxEFpD4Z2bSB7xmEHj +tE7VuUbdkLQ4jDrxwZuj7vC0+50pqDrSe0VY/KaGOnNtF1fVs3N+xlI9yUBhr9Yy +ANqUlx8Ql0i2aIuTSR9h/Wk9uu3NmsON9FkLoYK4bg9jiMcA6eKyRK/9Txn/CygG +zNy3+s2RhHjzAoIBAQCCZ5Pz6DPB3h4yPD2j4hr8jFxkQL0EeaLlDJFgNzMSZpV9 +6GbUBTYJHxhLMQ3yIsNl2fSrCwrjQMhAnXXY3WMmBAnXZaBYVxR+xtpyyhB7o4N0 +NutPVqZ3NNGGxIBZHx17P+UjBjFV8L4FWaZ4vqeLesU0SD29pMEsRzc1K3zdfsoG +rrWTKBtSKljs0J4fUIcaHvZs1xSNcQnQYpR5iqDPVCocbIW2vKMOA0xxa/qjHVYm +8O1SsyY/Oz//Q9/nW6fk5LwlpaNGHJNH3GXsXglLhWsVyk0hPC1gKouhj+HWQ2Dy +oFwlPQurT12ccj8aa7vb6KcDdAARCCEB1HbcxnMBAoIBADnY6FA0eRSh9eRsDvMT +cdwtiaPJHbtzL/RFKweto51nVxGkPrOhfHeuufvHdMdgaqDa+V7kD+ifbFno4REC +PY16pm4I1fau6C0hflkJ/X19A+0BkGKokNWWScmlyOEzGDLTyD2Nv+69VP31v6eY +ywfusFfmpr71iZ6SZ9wLoPemw/+7w2QjBfWRtb78f/DuCAs8FsGOsCWF92SShO3S +cGDYE376QUk0Bv3GWQsZ34/fUk9eumz+nnXcWa7nfrs/aSCscfXg8F3ndSZ+J6e3 +btOjH89RFv8B/b16keX8ZrEsBQh6JD6huwDDp361HVwzJzG7xh/rARmtBQ/YnC/v +/lMCggEAZ+EA8vRD7KSDLaxUl/DG0VQYtyJWsf83/NcLAizvYirI9batLjUF8ogy +pNHmA1STVJko+P+M2V5v56lJWnrj6HPj91NPIqdgJ/hIIZNI1zXm/DlLOegrFH0W +7fIY3+ZxzCIMuN4TvZ2H7n6NDBvTr6Vno17nFRGzy1suceIk7tx66KiFGmVt0sT/ +yyCQdyk+uM0KG3j19+QDrbtOTTmK3cOHTEOs7D2RmHSbn51jOckzL2mrhXvcGfTS +cqYW21Gm9U2P7VjgQ9vHuGzUFIrskltAGuX38XdPiBu9vZp1hwPbYY9MLxrbW8UM +ySc0NWyPv8/wyfen4dBRJ9OthwwcXw== +-----END PRIVATE KEY-----