From e4fdf171c773657f08ed79ab7dd17a3e9a8dcdb3 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 4 May 2015 11:49:49 -0600 Subject: [PATCH] More refactoring - nearly complete --- config/directives.go | 43 +- .../template.go => config/setup/browse.go | 81 ++- config/setup/git.go | 171 +++++++ config/setup/markdown.go | 74 +++ config/setup/templates.go | 54 ++ config/setup/websocket.go | 82 +++ middleware/browse/browse.go | 79 +-- middleware/extensions/ext.go | 49 +- middleware/git/git.go | 477 ++++++++++++------ middleware/git/gitclient.go | 328 ------------ middleware/markdown/markdown.go | 75 +-- middleware/templates/templates.go | 48 -- middleware/websockets/websocket.go | 2 +- middleware/websockets/websockets.go | 84 +-- 14 files changed, 833 insertions(+), 814 deletions(-) rename middleware/browse/template.go => config/setup/browse.go (55%) create mode 100644 config/setup/git.go create mode 100644 config/setup/markdown.go create mode 100644 config/setup/templates.go create mode 100644 config/setup/websocket.go delete mode 100644 middleware/git/gitclient.go diff --git a/config/directives.go b/config/directives.go index f3d297fd5..679ea5ea2 100644 --- a/config/directives.go +++ b/config/directives.go @@ -9,18 +9,47 @@ import ( func init() { // The parse package must know which directives // are valid, but it must not import the setup - // or config package. + // or config package. To solve this problem, we + // fill up this map in our init function here. + // The parse package does not need to know the + // ordering of the directives. for _, dir := range directiveOrder { parse.ValidDirectives[dir.name] = struct{}{} } } +// Directives are registered in the order they should be +// executed. Middleware (directives that inject a handler) +// are executed in the order A-B-C-*-C-B-A, assuming +// they all call the Next handler in the chain. +// +// Ordering is VERY important. Every middleware will +// feel the effects of all other middleware below +// (after) them during a request, but they must not +// care what middleware above them are doing. +// +// For example, log needs to know the status code and +// exactly how many bytes were written to the client, +// which every other middleware can affect, so it gets +// registered first. The errors middleware does not +// care if gzip or log modifies its response, so it +// gets registered below them. Gzip, on the other hand, +// DOES care what errors does to the response since it +// must compress every output to the client, even error +// pages, so it must be registered before the errors +// middleware and any others that would write to the +// response. var directiveOrder = []directive{ + // Essential directives that initialize vital configuration settings {"root", setup.Root}, {"tls", setup.TLS}, + + // Other directives that don't create HTTP handlers {"startup", setup.Startup}, {"shutdown", setup.Shutdown}, + {"git", setup.Git}, + // Directives that inject handlers (middleware) {"log", setup.Log}, {"gzip", setup.Gzip}, {"errors", setup.Errors}, @@ -31,15 +60,19 @@ var directiveOrder = []directive{ {"basicauth", setup.BasicAuth}, {"proxy", setup.Proxy}, {"fastcgi", setup.FastCGI}, - // {"websocket", setup.WebSocket}, - // {"markdown", setup.Markdown}, - // {"templates", setup.Templates}, - // {"browse", setup.Browse}, + {"websocket", setup.WebSocket}, + {"markdown", setup.Markdown}, + {"templates", setup.Templates}, + {"browse", setup.Browse}, } +// directive ties together a directive name with its setup function. type directive struct { name string setup setupFunc } +// A setup function takes a setup controller. Its return values may +// both be nil. If middleware is not nil, it will be chained into +// the HTTP handlers in the order specified in this package. type setupFunc func(c *setup.Controller) (middleware.Middleware, error) diff --git a/middleware/browse/template.go b/config/setup/browse.go similarity index 55% rename from middleware/browse/template.go rename to config/setup/browse.go index 8e97af4dc..23fdaea7e 100644 --- a/middleware/browse/template.go +++ b/config/setup/browse.go @@ -1,4 +1,83 @@ -package browse +package setup + +import ( + "fmt" + "html/template" + "io/ioutil" + + "github.com/mholt/caddy/middleware" + "github.com/mholt/caddy/middleware/browse" +) + +// Browse configures a new Browse middleware instance. +func Browse(c *Controller) (middleware.Middleware, error) { + configs, err := browseParse(c) + if err != nil { + return nil, err + } + + browse := browse.Browse{ + Root: c.Root, + Configs: configs, + } + + return func(next middleware.Handler) middleware.Handler { + browse.Next = next + return browse + }, nil +} + +func browseParse(c *Controller) ([]browse.Config, error) { + var configs []browse.Config + + appendCfg := func(bc browse.Config) error { + for _, c := range configs { + if c.PathScope == bc.PathScope { + return fmt.Errorf("Duplicate browsing config for %s", c.PathScope) + } + } + configs = append(configs, bc) + return nil + } + + for c.Next() { + var bc browse.Config + + // First argument is directory to allow browsing; default is site root + if c.NextArg() { + bc.PathScope = c.Val() + } else { + bc.PathScope = "/" + } + + // Second argument would be the template file to use + var tplText string + if c.NextArg() { + tplBytes, err := ioutil.ReadFile(c.Val()) + if err != nil { + return configs, err + } + tplText = string(tplBytes) + } else { + tplText = defaultTemplate + } + + // Build the template + tpl, err := template.New("listing").Parse(tplText) + if err != nil { + return configs, err + } + bc.Template = tpl + + // Save configuration + err = appendCfg(bc) + if err != nil { + return configs, err + } + } + + return configs, nil +} // The default template to use when serving up directory listings const defaultTemplate = ` diff --git a/config/setup/git.go b/config/setup/git.go new file mode 100644 index 000000000..499ca3fac --- /dev/null +++ b/config/setup/git.go @@ -0,0 +1,171 @@ +package setup + +import ( + "fmt" + "log" + "net/url" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/mholt/caddy/middleware" + "github.com/mholt/caddy/middleware/git" +) + +// Git configures a new Git service routine. +func Git(c *Controller) (middleware.Middleware, error) { + repo, err := gitParse(c) + if err != nil { + return nil, err + } + + c.Startup = append(c.Startup, func() error { + // Startup functions are blocking; start + // service routine in background + go func() { + for { + time.Sleep(repo.Interval) + + err := repo.Pull() + if err != nil { + if git.Logger == nil { + log.Println(err) + } else { + git.Logger.Println(err) + } + } + } + }() + + // Do a pull right away to return error + return repo.Pull() + }) + + return nil, err +} + +func gitParse(c *Controller) (*git.Repo, error) { + repo := &git.Repo{Branch: "master", Interval: git.DefaultInterval, Path: c.Root} + + for c.Next() { + args := c.RemainingArgs() + + switch len(args) { + case 2: + repo.Path = filepath.Clean(c.Root + string(filepath.Separator) + args[1]) + fallthrough + case 1: + repo.Url = args[0] + } + + for c.NextBlock() { + switch c.Val() { + case "repo": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Url = c.Val() + case "path": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Path = filepath.Clean(c.Root + string(filepath.Separator) + c.Val()) + case "branch": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Branch = c.Val() + case "key": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.KeyPath = c.Val() + case "interval": + if !c.NextArg() { + return nil, c.ArgErr() + } + t, _ := strconv.Atoi(c.Val()) + if t > 0 { + repo.Interval = time.Duration(t) * time.Second + } + case "then": + thenArgs := c.RemainingArgs() + if len(thenArgs) == 0 { + return nil, c.ArgErr() + } + repo.Then = strings.Join(thenArgs, " ") + } + } + } + + // if repo is not specified, return error + if repo.Url == "" { + return nil, c.ArgErr() + } + + // if private key is not specified, convert repository url to https + // to avoid ssh authentication + // else validate git url + // Note: private key support not yet available on Windows + var err error + if repo.KeyPath == "" { + repo.Url, repo.Host, err = sanitizeHttp(repo.Url) + } else { + repo.Url, repo.Host, err = sanitizeGit(repo.Url) + // TODO add Windows support for private repos + if runtime.GOOS == "windows" { + return nil, fmt.Errorf("Private repository not yet supported on Windows") + } + } + + if err != nil { + return nil, err + } + + // validate git availability in PATH + if err = git.InitGit(); err != nil { + return nil, err + } + + return repo, repo.Prepare() +} + +// sanitizeHttp cleans up repository url and converts to https format +// if currently in ssh format. +// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) +// and possible error +func sanitizeHttp(repoUrl string) (string, string, error) { + url, err := url.Parse(repoUrl) + if err != nil { + return "", "", err + } + + if url.Host == "" && strings.HasPrefix(url.Path, "git@") { + url.Path = url.Path[len("git@"):] + i := strings.Index(url.Path, ":") + if i < 0 { + return "", "", fmt.Errorf("Invalid git url %s", repoUrl) + } + url.Host = url.Path[:i] + url.Path = "/" + url.Path[i+1:] + } + + repoUrl = "https://" + url.Host + url.Path + return repoUrl, url.Host, nil +} + +// sanitizeGit cleans up repository url and validate ssh format. +// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) +// and possible error +func sanitizeGit(repoUrl string) (string, string, error) { + repoUrl = strings.TrimSpace(repoUrl) + if !strings.HasPrefix(repoUrl, "git@") || strings.Index(repoUrl, ":") < len("git@a:") { + return "", "", fmt.Errorf("Invalid git url %s", repoUrl) + } + hostUrl := repoUrl[len("git@"):] + i := strings.Index(hostUrl, ":") + host := hostUrl[:i] + return repoUrl, host, nil +} diff --git a/config/setup/markdown.go b/config/setup/markdown.go new file mode 100644 index 000000000..525458ac5 --- /dev/null +++ b/config/setup/markdown.go @@ -0,0 +1,74 @@ +package setup + +import ( + "github.com/mholt/caddy/middleware" + "github.com/mholt/caddy/middleware/markdown" + "github.com/russross/blackfriday" +) + +// Markdown configures a new Markdown middleware instance. +func Markdown(c *Controller) (middleware.Middleware, error) { + mdconfigs, err := markdownParse(c) + if err != nil { + return nil, err + } + + md := markdown.Markdown{ + Root: c.Root, + Configs: mdconfigs, + } + + return func(next middleware.Handler) middleware.Handler { + md.Next = next + return md + }, nil +} + +func markdownParse(c *Controller) ([]markdown.Config, error) { + var mdconfigs []markdown.Config + + for c.Next() { + md := markdown.Config{ + Renderer: blackfriday.HtmlRenderer(0, "", ""), + } + + // Get the path scope + if !c.NextArg() || c.Val() == "{" { + return mdconfigs, c.ArgErr() + } + md.PathScope = c.Val() + + // Load any other configuration parameters + for c.NextBlock() { + switch c.Val() { + case "ext": + exts := c.RemainingArgs() + if len(exts) == 0 { + return mdconfigs, c.ArgErr() + } + md.Extensions = append(md.Extensions, exts...) + case "css": + if !c.NextArg() { + return mdconfigs, c.ArgErr() + } + md.Styles = append(md.Styles, c.Val()) + case "js": + if !c.NextArg() { + return mdconfigs, c.ArgErr() + } + md.Scripts = append(md.Scripts, c.Val()) + default: + return mdconfigs, c.Err("Expected valid markdown configuration property") + } + } + + // If no extensions were specified, assume .md + if len(md.Extensions) == 0 { + md.Extensions = []string{".md"} + } + + mdconfigs = append(mdconfigs, md) + } + + return mdconfigs, nil +} diff --git a/config/setup/templates.go b/config/setup/templates.go new file mode 100644 index 000000000..d3af7ca5d --- /dev/null +++ b/config/setup/templates.go @@ -0,0 +1,54 @@ +package setup + +import ( + "github.com/mholt/caddy/middleware" + "github.com/mholt/caddy/middleware/templates" +) + +// Templates configures a new Templates middleware instance. +func Templates(c *Controller) (middleware.Middleware, error) { + rules, err := templatesParse(c) + if err != nil { + return nil, err + } + + tmpls := templates.Templates{ + Root: c.Root, + Rules: rules, + } + + return func(next middleware.Handler) middleware.Handler { + tmpls.Next = next + return tmpls + }, nil +} + +func templatesParse(c *Controller) ([]templates.Rule, error) { + var rules []templates.Rule + + for c.Next() { + var rule templates.Rule + + if c.NextArg() { + // First argument would be the path + rule.Path = c.Val() + + // Any remaining arguments are extensions + rule.Extensions = c.RemainingArgs() + if len(rule.Extensions) == 0 { + rule.Extensions = defaultExtensions + } + } else { + rule.Path = defaultPath + rule.Extensions = defaultExtensions + } + + rules = append(rules, rule) + } + + return rules, nil +} + +const defaultPath = "/" + +var defaultExtensions = []string{".html", ".htm", ".txt"} diff --git a/config/setup/websocket.go b/config/setup/websocket.go new file mode 100644 index 000000000..d3ad2df8a --- /dev/null +++ b/config/setup/websocket.go @@ -0,0 +1,82 @@ +package setup + +import ( + "github.com/mholt/caddy/middleware" + "github.com/mholt/caddy/middleware/websockets" +) + +// WebSocket configures a new WebSockets middleware instance. +func WebSocket(c *Controller) (middleware.Middleware, error) { + var websocks []websockets.Config + var respawn bool + + optionalBlock := func() (hadBlock bool, err error) { + for c.NextBlock() { + hadBlock = true + if c.Val() == "respawn" { + respawn = true + } else { + return true, c.Err("Expected websocket configuration parameter in block") + } + } + return + } + + for c.Next() { + var val, path, command string + + // Path or command; not sure which yet + if !c.NextArg() { + return nil, c.ArgErr() + } + val = c.Val() + + // Extra configuration may be in a block + hadBlock, err := optionalBlock() + if err != nil { + return nil, err + } + + if !hadBlock { + // The next argument on this line will be the command or an open curly brace + if c.NextArg() { + path = val + command = c.Val() + } else { + path = "/" + command = val + } + + // Okay, check again for optional block + hadBlock, err = optionalBlock() + if err != nil { + return nil, err + } + } + + // Split command into the actual command and its arguments + cmd, args, err := middleware.SplitCommandAndArgs(command) + if err != nil { + return nil, err + } + + websocks = append(websocks, websockets.Config{ + Path: path, + Command: cmd, + Arguments: args, + Respawn: respawn, // TODO: This isn't used currently + }) + } + + websockets.GatewayInterface = envGatewayInterface + websockets.ServerSoftware = envServerSoftware + + return func(next middleware.Handler) middleware.Handler { + return websockets.WebSockets{Next: next, Sockets: websocks} + }, nil +} + +const ( + envGatewayInterface = "caddy-CGI/1.1" + envServerSoftware = "caddy/" // TODO: Version +) diff --git a/middleware/browse/browse.go b/middleware/browse/browse.go index a3708b11e..d09bd253f 100644 --- a/middleware/browse/browse.go +++ b/middleware/browse/browse.go @@ -4,9 +4,7 @@ package browse import ( "bytes" - "fmt" "html/template" - "io/ioutil" "net/http" "net/url" "os" @@ -23,11 +21,11 @@ import ( type Browse struct { Next middleware.Handler Root string - Configs []BrowseConfig + Configs []Config } -// BrowseConfig is a configuration for browsing in a particular path. -type BrowseConfig struct { +// Config is a configuration for browsing in a particular path. +type Config struct { PathScope string Template *template.Template } @@ -72,24 +70,6 @@ var IndexPages = []string{ "default.htm", } -// New creates a new instance of browse middleware. -func New(c middleware.Controller) (middleware.Middleware, error) { - configs, err := parse(c) - if err != nil { - return nil, err - } - - browse := Browse{ - Root: c.Root(), - Configs: configs, - } - - return func(next middleware.Handler) middleware.Handler { - browse.Next = next - return browse - }, nil -} - // ServeHTTP implements the middleware.Handler interface. func (b Browse) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { filename := b.Root + r.URL.Path @@ -196,56 +176,3 @@ func (b Browse) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { // Didn't qualify; pass-thru return b.Next.ServeHTTP(w, r) } - -// parse returns a list of browsing configurations -func parse(c middleware.Controller) ([]BrowseConfig, error) { - var configs []BrowseConfig - - appendCfg := func(bc BrowseConfig) error { - for _, c := range configs { - if c.PathScope == bc.PathScope { - return fmt.Errorf("Duplicate browsing config for %s", c.PathScope) - } - } - configs = append(configs, bc) - return nil - } - - for c.Next() { - var bc BrowseConfig - - // First argument is directory to allow browsing; default is site root - if c.NextArg() { - bc.PathScope = c.Val() - } else { - bc.PathScope = "/" - } - - // Second argument would be the template file to use - var tplText string - if c.NextArg() { - tplBytes, err := ioutil.ReadFile(c.Val()) - if err != nil { - return configs, err - } - tplText = string(tplBytes) - } else { - tplText = defaultTemplate - } - - // Build the template - tpl, err := template.New("listing").Parse(tplText) - if err != nil { - return configs, err - } - bc.Template = tpl - - // Save configuration - err = appendCfg(bc) - if err != nil { - return configs, err - } - } - - return configs, nil -} diff --git a/middleware/extensions/ext.go b/middleware/extensions/ext.go index 006ba07a0..6b0293271 100644 --- a/middleware/extensions/ext.go +++ b/middleware/extensions/ext.go @@ -1,8 +1,9 @@ -// Package extension is middleware for clean URLs. The root path -// of the site is passed in as well as possible extensions to try -// internally for paths requested that don't match an existing -// resource. The first path+ext combination that matches a valid -// file will be used. +// Package extension is middleware for clean URLs. +// +// The root path of the site is passed in as well as possible extensions +// to try internally for paths requested that don't match an existing +// resource. The first path+ext combination that matches a valid file +// will be used. package extensions import ( @@ -14,25 +15,6 @@ import ( "github.com/mholt/caddy/middleware" ) -// New creates a new instance of middleware that assumes extensions -// so the site can use cleaner, extensionless URLs -func New(c middleware.Controller) (middleware.Middleware, error) { - root := c.Root() - - extensions, err := parse(c) - if err != nil { - return nil, err - } - - return func(next middleware.Handler) middleware.Handler { - return Ext{ - Next: next, - Extensions: extensions, - Root: root, - } - }, nil -} - // Ext can assume an extension from clean URLs. // It tries extensions in the order listed in Extensions. type Ext struct { @@ -60,25 +42,6 @@ func (e Ext) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { return e.Next.ServeHTTP(w, r) } -// parse sets up an instance of extension middleware -// from a middleware controller and returns a list of extensions. -func parse(c middleware.Controller) ([]string, error) { - var extensions []string - - for c.Next() { - // At least one extension is required - if !c.NextArg() { - return extensions, c.ArgErr() - } - extensions = append(extensions, c.Val()) - - // Tack on any other extensions that may have been listed - extensions = append(extensions, c.RemainingArgs()...) - } - - return extensions, nil -} - // resourceExists returns true if the file specified at // root + path exists; false otherwise. func resourceExists(root, path string) bool { diff --git a/middleware/git/git.go b/middleware/git/git.go index 045002fd5..5a617ce34 100644 --- a/middleware/git/git.go +++ b/middleware/git/git.go @@ -1,178 +1,36 @@ package git import ( + "bytes" "fmt" + "io/ioutil" "log" - "net/url" "os" - "path/filepath" - "runtime" - "strconv" + "os/exec" "strings" + "sync" "time" "github.com/mholt/caddy/middleware" ) +// DefaultInterval is the minimum interval to delay before +// requesting another git pull +const DefaultInterval time.Duration = time.Hour * 1 + +// Number of retries if git pull fails +const numRetries = 3 + +// gitBinary holds the absolute path to git executable +var gitBinary string + +// initMutex prevents parallel attempt to validate +// git availability in PATH +var initMutex sync.Mutex = sync.Mutex{} + // Logger is used to log errors; if nil, the default log.Logger is used. var Logger *log.Logger -// New creates a new instance of git middleware. -func New(c middleware.Controller) (middleware.Middleware, error) { - repo, err := parse(c) - if err != nil { - return nil, err - } - - c.Startup(func() error { - // Startup functions are blocking; start - // service routine in background - go func() { - for { - time.Sleep(repo.Interval) - - err := repo.Pull() - if err != nil { - if Logger == nil { - log.Println(err) - } else { - Logger.Println(err) - } - } - } - }() - - // Do a pull right away to return error - return repo.Pull() - }) - - return nil, err -} - -func parse(c middleware.Controller) (*Repo, error) { - repo := &Repo{Branch: "master", Interval: DefaultInterval, Path: c.Root()} - - for c.Next() { - args := c.RemainingArgs() - - switch len(args) { - case 2: - repo.Path = filepath.Clean(c.Root() + string(filepath.Separator) + args[1]) - fallthrough - case 1: - repo.Url = args[0] - } - - for c.NextBlock() { - switch c.Val() { - case "repo": - if !c.NextArg() { - return nil, c.ArgErr() - } - repo.Url = c.Val() - case "path": - if !c.NextArg() { - return nil, c.ArgErr() - } - repo.Path = filepath.Clean(c.Root() + string(filepath.Separator) + c.Val()) - case "branch": - if !c.NextArg() { - return nil, c.ArgErr() - } - repo.Branch = c.Val() - case "key": - if !c.NextArg() { - return nil, c.ArgErr() - } - repo.KeyPath = c.Val() - case "interval": - if !c.NextArg() { - return nil, c.ArgErr() - } - t, _ := strconv.Atoi(c.Val()) - if t > 0 { - repo.Interval = time.Duration(t) * time.Second - } - case "then": - thenArgs := c.RemainingArgs() - if len(thenArgs) == 0 { - return nil, c.ArgErr() - } - repo.Then = strings.Join(thenArgs, " ") - } - } - } - - // if repo is not specified, return error - if repo.Url == "" { - return nil, c.ArgErr() - } - - // if private key is not specified, convert repository url to https - // to avoid ssh authentication - // else validate git url - // Note: private key support not yet available on Windows - var err error - if repo.KeyPath == "" { - repo.Url, repo.Host, err = sanitizeHttp(repo.Url) - } else { - repo.Url, repo.Host, err = sanitizeGit(repo.Url) - // TODO add Windows support for private repos - if runtime.GOOS == "windows" { - return nil, fmt.Errorf("Private repository not yet supported on Windows") - } - } - - if err != nil { - return nil, err - } - - // validate git availability in PATH - if err = initGit(); err != nil { - return nil, err - } - - return repo, repo.prepare() -} - -// sanitizeHttp cleans up repository url and converts to https format -// if currently in ssh format. -// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) -// and possible error -func sanitizeHttp(repoUrl string) (string, string, error) { - url, err := url.Parse(repoUrl) - if err != nil { - return "", "", err - } - - if url.Host == "" && strings.HasPrefix(url.Path, "git@") { - url.Path = url.Path[len("git@"):] - i := strings.Index(url.Path, ":") - if i < 0 { - return "", "", fmt.Errorf("Invalid git url %s", repoUrl) - } - url.Host = url.Path[:i] - url.Path = "/" + url.Path[i+1:] - } - - repoUrl = "https://" + url.Host + url.Path - return repoUrl, url.Host, nil -} - -// sanitizeGit cleans up repository url and validate ssh format. -// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) -// and possible error -func sanitizeGit(repoUrl string) (string, string, error) { - repoUrl = strings.TrimSpace(repoUrl) - if !strings.HasPrefix(repoUrl, "git@") || strings.Index(repoUrl, ":") < len("git@a:") { - return "", "", fmt.Errorf("Invalid git url %s", repoUrl) - } - hostUrl := repoUrl[len("git@"):] - i := strings.Index(hostUrl, ":") - host := hostUrl[:i] - return repoUrl, host, nil -} - // logger is an helper function to retrieve the available logger func logger() *log.Logger { if Logger == nil { @@ -180,3 +38,302 @@ func logger() *log.Logger { } return Logger } + +// Repo is the structure that holds required information +// of a git repository. +type Repo struct { + Url string // Repository URL + Path string // Directory to pull to + Host string // Git domain host e.g. github.com + Branch string // Git branch + KeyPath string // Path to private ssh key + Interval time.Duration // Interval between pulls + Then string // Command to execute after successful git pull + pulled bool // true if there was a successful pull + lastPull time.Time // time of the last successful pull + lastCommit string // hash for the most recent commit + sync.Mutex +} + +// Pull attempts a git clone. +// It retries at most numRetries times if error occurs +func (r *Repo) Pull() error { + r.Lock() + defer r.Unlock() + // if it is less than interval since last pull, return + if time.Since(r.lastPull) <= r.Interval { + return nil + } + + // keep last commit hash for comparison later + lastCommit := r.lastCommit + + var err error + // Attempt to pull at most numRetries times + for i := 0; i < numRetries; i++ { + if err = r.pull(); err == nil { + break + } + logger().Println(err) + } + + if err != nil { + return err + } + + // check if there are new changes, + // then execute post pull command + if r.lastCommit == lastCommit { + logger().Println("No new changes.") + return nil + } + return r.postPullCommand() +} + +// Pull performs git clone, or git pull if repository exists +func (r *Repo) pull() error { + params := []string{"clone", "-b", r.Branch, r.Url, r.Path} + if r.pulled { + params = []string{"pull", "origin", r.Branch} + } + + // if key is specified, pull using ssh key + if r.KeyPath != "" { + return r.pullWithKey(params) + } + + dir := "" + if r.pulled { + dir = r.Path + } + + var err error + if err = runCmd(gitBinary, params, dir); err == nil { + r.pulled = true + r.lastPull = time.Now() + logger().Printf("%v pulled.\n", r.Url) + r.lastCommit, err = r.getMostRecentCommit() + } + return err +} + +// pullWithKey is used for private repositories and requires an ssh key. +// Note: currently only limited to Linux and OSX. +func (r *Repo) pullWithKey(params []string) error { + var gitSsh, script *os.File + // ensure temporary files deleted after usage + defer func() { + if gitSsh != nil { + os.Remove(gitSsh.Name()) + } + if script != nil { + os.Remove(script.Name()) + } + }() + + var err error + // write git.sh script to temp file + gitSsh, err = writeScriptFile(gitWrapperScript(gitBinary)) + if err != nil { + return err + } + + // write git clone bash script to file + script, err = writeScriptFile(bashScript(gitSsh.Name(), r, params)) + if err != nil { + return err + } + + dir := "" + if r.pulled { + dir = r.Path + } + + if err = runCmd(script.Name(), nil, dir); err == nil { + r.pulled = true + r.lastPull = time.Now() + logger().Printf("%v pulled.\n", r.Url) + r.lastCommit, err = r.getMostRecentCommit() + } + return err +} + +// Prepare prepares for a git pull +// and validates the configured directory +func (r *Repo) Prepare() error { + // check if directory exists or is empty + // if not, create directory + fs, err := ioutil.ReadDir(r.Path) + if err != nil || len(fs) == 0 { + return os.MkdirAll(r.Path, os.FileMode(0755)) + } + + // validate git repo + isGit := false + for _, f := range fs { + if f.IsDir() && f.Name() == ".git" { + isGit = true + break + } + } + + if isGit { + // check if same repository + var repoUrl string + if repoUrl, err = r.getRepoUrl(); err == nil && repoUrl == r.Url { + r.pulled = true + return nil + } + if err != nil { + return fmt.Errorf("Cannot retrieve repo url for %v Error: %v", r.Path, err) + } + return fmt.Errorf("Another git repo '%v' exists at %v", repoUrl, r.Path) + } + return fmt.Errorf("Cannot git clone into %v, directory not empty.", r.Path) +} + +// getMostRecentCommit gets the hash of the most recent commit to the +// repository. Useful for checking if changes occur. +func (r *Repo) getMostRecentCommit() (string, error) { + command := gitBinary + ` --no-pager log -n 1 --pretty=format:"%H"` + c, args, err := middleware.SplitCommandAndArgs(command) + if err != nil { + return "", err + } + return runCmdOutput(c, args, r.Path) +} + +// getRepoUrl retrieves remote origin url for the git repository at path +func (r *Repo) getRepoUrl() (string, error) { + _, err := os.Stat(r.Path) + if err != nil { + return "", err + } + args := []string{"config", "--get", "remote.origin.url"} + return runCmdOutput(gitBinary, args, r.Path) +} + +// postPullCommand executes r.Then. +// It is trigged after successful git pull +func (r *Repo) postPullCommand() error { + if r.Then == "" { + return nil + } + c, args, err := middleware.SplitCommandAndArgs(r.Then) + if err != nil { + return err + } + + if err = runCmd(c, args, r.Path); err == nil { + logger().Printf("Command %v successful.\n", r.Then) + } + return err +} + +// InitGit validates git installation and locates the git executable +// binary in PATH +func InitGit() error { + // prevent concurrent call + initMutex.Lock() + defer initMutex.Unlock() + + // if validation has been done before and binary located in + // PATH, return. + if gitBinary != "" { + return nil + } + + // locate git binary in path + var err error + gitBinary, err = exec.LookPath("git") + return err +} + +// runCmd is a helper function to run commands. +// It runs command with args from directory at dir. +// The executed process outputs to os.Stderr +func runCmd(command string, args []string, dir string) error { + cmd := exec.Command(command, args...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stderr + cmd.Dir = dir + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +// runCmdOutput is a helper function to run commands and return output. +// It runs command with args from directory at dir. +// If successful, returns output and nil error +func runCmdOutput(command string, args []string, dir string) (string, error) { + cmd := exec.Command(command, args...) + cmd.Dir = dir + var err error + if output, err := cmd.Output(); err == nil { + return string(bytes.TrimSpace(output)), nil + } + return "", err +} + +// writeScriptFile writes content to a temporary file. +// It changes the temporary file mode to executable and +// closes it to prepare it for execution. +func writeScriptFile(content []byte) (file *os.File, err error) { + if file, err = ioutil.TempFile("", "caddy"); err != nil { + return nil, err + } + if _, err = file.Write(content); err != nil { + return nil, err + } + if err = file.Chmod(os.FileMode(0755)); err != nil { + return nil, err + } + return file, file.Close() +} + +// gitWrapperScript forms content for git.sh script +var gitWrapperScript = func(gitBinary string) []byte { + return []byte(fmt.Sprintf(`#!/bin/bash + +# The MIT License (MIT) +# Copyright (c) 2013 Alvin Abad + +if [ $# -eq 0 ]; then + echo "Git wrapper script that can specify an ssh-key file +Usage: + git.sh -i ssh-key-file git-command + " + exit 1 +fi + +# remove temporary file on exit +trap 'rm -f /tmp/.git_ssh.$$' 0 + +if [ "$1" = "-i" ]; then + SSH_KEY=$2; shift; shift + echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$ + chmod +x /tmp/.git_ssh.$$ + export GIT_SSH=/tmp/.git_ssh.$$ +fi + +# in case the git command is repeated +[ "$1" = "git" ] && shift + +# Run the git command +%v "$@" + +`, gitBinary)) +} + +// bashScript forms content of bash script to clone or update a repo using ssh +var bashScript = func(gitShPath string, repo *Repo, params []string) []byte { + return []byte(fmt.Sprintf(`#!/bin/bash + +mkdir -p ~/.ssh; +touch ~/.ssh/known_hosts; +ssh-keyscan -t rsa,dsa %v 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts; +cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts; +%v -i %v %v; +`, repo.Host, gitShPath, repo.KeyPath, strings.Join(params, " "))) +} diff --git a/middleware/git/gitclient.go b/middleware/git/gitclient.go deleted file mode 100644 index 3c6642ade..000000000 --- a/middleware/git/gitclient.go +++ /dev/null @@ -1,328 +0,0 @@ -package git - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "os/exec" - "strings" - "sync" - "time" - - "github.com/mholt/caddy/middleware" -) - -// DefaultInterval is the minimum interval to delay before -// requesting another git pull -const DefaultInterval time.Duration = time.Hour * 1 - -// Number of retries if git pull fails -const numRetries = 3 - -// gitBinary holds the absolute path to git executable -var gitBinary string - -// initMutex prevents parallel attempt to validate -// git availability in PATH -var initMutex sync.Mutex = sync.Mutex{} - -// Repo is the structure that holds required information -// of a git repository. -type Repo struct { - Url string // Repository URL - Path string // Directory to pull to - Host string // Git domain host e.g. github.com - Branch string // Git branch - KeyPath string // Path to private ssh key - Interval time.Duration // Interval between pulls - Then string // Command to execute after successful git pull - pulled bool // true if there was a successful pull - lastPull time.Time // time of the last successful pull - lastCommit string // hash for the most recent commit - sync.Mutex -} - -// Pull attempts a git clone. -// It retries at most numRetries times if error occurs -func (r *Repo) Pull() error { - r.Lock() - defer r.Unlock() - // if it is less than interval since last pull, return - if time.Since(r.lastPull) <= r.Interval { - return nil - } - - // keep last commit hash for comparison later - lastCommit := r.lastCommit - - var err error - // Attempt to pull at most numRetries times - for i := 0; i < numRetries; i++ { - if err = r.pull(); err == nil { - break - } - logger().Println(err) - } - - if err != nil { - return err - } - - // check if there are new changes, - // then execute post pull command - if r.lastCommit == lastCommit { - logger().Println("No new changes.") - return nil - } - return r.postPullCommand() -} - -// Pull performs git clone, or git pull if repository exists -func (r *Repo) pull() error { - params := []string{"clone", "-b", r.Branch, r.Url, r.Path} - if r.pulled { - params = []string{"pull", "origin", r.Branch} - } - - // if key is specified, pull using ssh key - if r.KeyPath != "" { - return r.pullWithKey(params) - } - - dir := "" - if r.pulled { - dir = r.Path - } - - var err error - if err = runCmd(gitBinary, params, dir); err == nil { - r.pulled = true - r.lastPull = time.Now() - logger().Printf("%v pulled.\n", r.Url) - r.lastCommit, err = r.getMostRecentCommit() - } - return err -} - -// pullWithKey is used for private repositories and requires an ssh key. -// Note: currently only limited to Linux and OSX. -func (r *Repo) pullWithKey(params []string) error { - var gitSsh, script *os.File - // ensure temporary files deleted after usage - defer func() { - if gitSsh != nil { - os.Remove(gitSsh.Name()) - } - if script != nil { - os.Remove(script.Name()) - } - }() - - var err error - // write git.sh script to temp file - gitSsh, err = writeScriptFile(gitWrapperScript(gitBinary)) - if err != nil { - return err - } - - // write git clone bash script to file - script, err = writeScriptFile(bashScript(gitSsh.Name(), r, params)) - if err != nil { - return err - } - - dir := "" - if r.pulled { - dir = r.Path - } - - if err = runCmd(script.Name(), nil, dir); err == nil { - r.pulled = true - r.lastPull = time.Now() - logger().Printf("%v pulled.\n", r.Url) - r.lastCommit, err = r.getMostRecentCommit() - } - return err -} - -// prepare prepares for a git pull -// and validates the configured directory -func (r *Repo) prepare() error { - // check if directory exists or is empty - // if not, create directory - fs, err := ioutil.ReadDir(r.Path) - if err != nil || len(fs) == 0 { - return os.MkdirAll(r.Path, os.FileMode(0755)) - } - - // validate git repo - isGit := false - for _, f := range fs { - if f.IsDir() && f.Name() == ".git" { - isGit = true - break - } - } - - if isGit { - // check if same repository - var repoUrl string - if repoUrl, err = r.getRepoUrl(); err == nil && repoUrl == r.Url { - r.pulled = true - return nil - } - if err != nil { - return fmt.Errorf("Cannot retrieve repo url for %v Error: %v", r.Path, err) - } - return fmt.Errorf("Another git repo '%v' exists at %v", repoUrl, r.Path) - } - return fmt.Errorf("Cannot git clone into %v, directory not empty.", r.Path) -} - -// getMostRecentCommit gets the hash of the most recent commit to the -// repository. Useful for checking if changes occur. -func (r *Repo) getMostRecentCommit() (string, error) { - command := gitBinary + ` --no-pager log -n 1 --pretty=format:"%H"` - c, args, err := middleware.SplitCommandAndArgs(command) - if err != nil { - return "", err - } - return runCmdOutput(c, args, r.Path) -} - -// getRepoUrl retrieves remote origin url for the git repository at path -func (r *Repo) getRepoUrl() (string, error) { - _, err := os.Stat(r.Path) - if err != nil { - return "", err - } - args := []string{"config", "--get", "remote.origin.url"} - return runCmdOutput(gitBinary, args, r.Path) -} - -// postPullCommand executes r.Then. -// It is trigged after successful git pull -func (r *Repo) postPullCommand() error { - if r.Then == "" { - return nil - } - c, args, err := middleware.SplitCommandAndArgs(r.Then) - if err != nil { - return err - } - - if err = runCmd(c, args, r.Path); err == nil { - logger().Printf("Command %v successful.\n", r.Then) - } - return err -} - -// initGit validates git installation and locates the git executable -// binary in PATH -func initGit() error { - // prevent concurrent call - initMutex.Lock() - defer initMutex.Unlock() - - // if validation has been done before and binary located in - // PATH, return. - if gitBinary != "" { - return nil - } - - // locate git binary in path - var err error - gitBinary, err = exec.LookPath("git") - return err - -} - -// runCmd is a helper function to run commands. -// It runs command with args from directory at dir. -// The executed process outputs to os.Stderr -func runCmd(command string, args []string, dir string) error { - cmd := exec.Command(command, args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stderr - cmd.Dir = dir - if err := cmd.Start(); err != nil { - return err - } - return cmd.Wait() -} - -// runCmdOutput is a helper function to run commands and return output. -// It runs command with args from directory at dir. -// If successful, returns output and nil error -func runCmdOutput(command string, args []string, dir string) (string, error) { - cmd := exec.Command(command, args...) - cmd.Dir = dir - var err error - if output, err := cmd.Output(); err == nil { - return string(bytes.TrimSpace(output)), nil - } - return "", err -} - -// writeScriptFile writes content to a temporary file. -// It changes the temporary file mode to executable and -// closes it to prepare it for execution. -func writeScriptFile(content []byte) (file *os.File, err error) { - if file, err = ioutil.TempFile("", "caddy"); err != nil { - return nil, err - } - if _, err = file.Write(content); err != nil { - return nil, err - } - if err = file.Chmod(os.FileMode(0755)); err != nil { - return nil, err - } - return file, file.Close() -} - -// gitWrapperScript forms content for git.sh script -var gitWrapperScript = func(gitBinary string) []byte { - return []byte(fmt.Sprintf(`#!/bin/bash - -# The MIT License (MIT) -# Copyright (c) 2013 Alvin Abad - -if [ $# -eq 0 ]; then - echo "Git wrapper script that can specify an ssh-key file -Usage: - git.sh -i ssh-key-file git-command - " - exit 1 -fi - -# remove temporary file on exit -trap 'rm -f /tmp/.git_ssh.$$' 0 - -if [ "$1" = "-i" ]; then - SSH_KEY=$2; shift; shift - echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$ - chmod +x /tmp/.git_ssh.$$ - export GIT_SSH=/tmp/.git_ssh.$$ -fi - -# in case the git command is repeated -[ "$1" = "git" ] && shift - -# Run the git command -%v "$@" - -`, gitBinary)) -} - -// bashScript forms content of bash script to clone or update a repo using ssh -var bashScript = func(gitShPath string, repo *Repo, params []string) []byte { - return []byte(fmt.Sprintf(`#!/bin/bash - -mkdir -p ~/.ssh; -touch ~/.ssh/known_hosts; -ssh-keyscan -t rsa,dsa %v 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts; -cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts; -%v -i %v %v; -`, repo.Host, gitShPath, repo.KeyPath, strings.Join(params, " "))) -} diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index 162528e4e..db2ff3b60 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -24,11 +24,11 @@ type Markdown struct { Next middleware.Handler // The list of markdown configurations - Configs []MarkdownConfig + Configs []Config } -// MarkdownConfig stores markdown middleware configurations. -type MarkdownConfig struct { +// Config stores markdown middleware configurations. +type Config struct { // Markdown renderer Renderer blackfriday.Renderer @@ -45,25 +45,6 @@ type MarkdownConfig struct { Scripts []string } -// New creates a new instance of Markdown middleware that -// renders markdown to HTML on-the-fly. -func New(c middleware.Controller) (middleware.Middleware, error) { - mdconfigs, err := parse(c) - if err != nil { - return nil, err - } - - md := Markdown{ - Root: c.Root(), - Configs: mdconfigs, - } - - return func(next middleware.Handler) middleware.Handler { - md.Next = next - return md - }, nil -} - // ServeHTTP implements the http.Handler interface. func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { for _, m := range md.Configs { @@ -125,56 +106,6 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error return md.Next.ServeHTTP(w, r) } -// parse creates new instances of Markdown middleware. -func parse(c middleware.Controller) ([]MarkdownConfig, error) { - var mdconfigs []MarkdownConfig - - for c.Next() { - md := MarkdownConfig{ - Renderer: blackfriday.HtmlRenderer(0, "", ""), - } - - // Get the path scope - if !c.NextArg() || c.Val() == "{" { - return mdconfigs, c.ArgErr() - } - md.PathScope = c.Val() - - // Load any other configuration parameters - for c.NextBlock() { - switch c.Val() { - case "ext": - exts := c.RemainingArgs() - if len(exts) == 0 { - return mdconfigs, c.ArgErr() - } - md.Extensions = append(md.Extensions, exts...) - case "css": - if !c.NextArg() { - return mdconfigs, c.ArgErr() - } - md.Styles = append(md.Styles, c.Val()) - case "js": - if !c.NextArg() { - return mdconfigs, c.ArgErr() - } - md.Scripts = append(md.Scripts, c.Val()) - default: - return mdconfigs, c.Err("Expected valid markdown configuration property") - } - } - - // If no extensions were specified, assume .md - if len(md.Extensions) == 0 { - md.Extensions = []string{".md"} - } - - mdconfigs = append(mdconfigs, md) - } - - return mdconfigs, nil -} - const ( htmlTemplate = ` diff --git a/middleware/templates/templates.go b/middleware/templates/templates.go index ecd92c088..7c2ca34e6 100644 --- a/middleware/templates/templates.go +++ b/middleware/templates/templates.go @@ -10,24 +10,6 @@ import ( "github.com/mholt/caddy/middleware" ) -// New constructs a new Templates middleware instance. -func New(c middleware.Controller) (middleware.Middleware, error) { - rules, err := parse(c) - if err != nil { - return nil, err - } - - tmpls := Templates{ - Root: c.Root(), - Rules: rules, - } - - return func(next middleware.Handler) middleware.Handler { - tmpls.Next = next - return tmpls - }, nil -} - // ServeHTTP implements the middleware.Handler interface. func (t Templates) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { for _, rule := range t.Rules { @@ -64,32 +46,6 @@ func (t Templates) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error return t.Next.ServeHTTP(w, r) } -func parse(c middleware.Controller) ([]Rule, error) { - var rules []Rule - - for c.Next() { - var rule Rule - - if c.NextArg() { - // First argument would be the path - rule.Path = c.Val() - - // Any remaining arguments are extensions - rule.Extensions = c.RemainingArgs() - if len(rule.Extensions) == 0 { - rule.Extensions = defaultExtensions - } - } else { - rule.Path = defaultPath - rule.Extensions = defaultExtensions - } - - rules = append(rules, rule) - } - - return rules, nil -} - // Templates is middleware to render templated files as the HTTP response. type Templates struct { Next middleware.Handler @@ -104,7 +60,3 @@ type Rule struct { Path string Extensions []string } - -const defaultPath = "/" - -var defaultExtensions = []string{".html", ".htm", ".txt"} diff --git a/middleware/websockets/websocket.go b/middleware/websockets/websocket.go index e830ce2b2..fa213c4ec 100644 --- a/middleware/websockets/websocket.go +++ b/middleware/websockets/websocket.go @@ -12,7 +12,7 @@ import ( // WebSocket represents a web socket server instance. A WebSocket // is instantiated for each new websocket request/connection. type WebSocket struct { - WSConfig + Config *http.Request } diff --git a/middleware/websockets/websockets.go b/middleware/websockets/websockets.go index 561d84d6a..35838aadb 100644 --- a/middleware/websockets/websockets.go +++ b/middleware/websockets/websockets.go @@ -19,12 +19,12 @@ type ( Next middleware.Handler // Sockets holds all the web socket endpoint configurations - Sockets []WSConfig + Sockets []Config } // WSConfig holds the configuration for a single websocket // endpoint which may serve multiple websocket connections. - WSConfig struct { + Config struct { Path string Command string Arguments []string @@ -37,8 +37,8 @@ func (ws WebSockets) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, err for _, sockconfig := range ws.Sockets { if middleware.Path(r.URL.Path).Matches(sockconfig.Path) { socket := WebSocket{ - WSConfig: sockconfig, - Request: r, + Config: sockconfig, + Request: r, } websocket.Handler(socket.Handle).ServeHTTP(w, r) return 0, nil @@ -49,77 +49,6 @@ func (ws WebSockets) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, err return ws.Next.ServeHTTP(w, r) } -// New constructs and configures a new websockets middleware instance. -func New(c middleware.Controller) (middleware.Middleware, error) { - var websocks []WSConfig - var respawn bool - - optionalBlock := func() (hadBlock bool, err error) { - for c.NextBlock() { - hadBlock = true - if c.Val() == "respawn" { - respawn = true - } else { - return true, c.Err("Expected websocket configuration parameter in block") - } - } - return - } - - for c.Next() { - var val, path, command string - - // Path or command; not sure which yet - if !c.NextArg() { - return nil, c.ArgErr() - } - val = c.Val() - - // Extra configuration may be in a block - hadBlock, err := optionalBlock() - if err != nil { - return nil, err - } - - if !hadBlock { - // The next argument on this line will be the command or an open curly brace - if c.NextArg() { - path = val - command = c.Val() - } else { - path = "/" - command = val - } - - // Okay, check again for optional block - hadBlock, err = optionalBlock() - if err != nil { - return nil, err - } - } - - // Split command into the actual command and its arguments - cmd, args, err := middleware.SplitCommandAndArgs(command) - if err != nil { - return nil, err - } - - websocks = append(websocks, WSConfig{ - Path: path, - Command: cmd, - Arguments: args, - Respawn: respawn, // TODO: This isn't used currently - }) - } - - GatewayInterface = envGatewayInterface - ServerSoftware = envServerSoftware - - return func(next middleware.Handler) middleware.Handler { - return WebSockets{Next: next, Sockets: websocks} - }, nil -} - var ( // See CGI spec, 4.1.4 GatewayInterface string @@ -127,8 +56,3 @@ var ( // See CGI spec, 4.1.17 ServerSoftware string ) - -const ( - envGatewayInterface = "caddy-CGI/1.1" - envServerSoftware = "caddy/" // TODO: Version -)