From 3b910645e7dd7d96ba1e4069bd5c98d7d9375e39 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 5 Aug 2015 09:55:04 +0100 Subject: [PATCH] Markdown: generate static sites after links. --- config/setup/markdown.go | 58 +------- middleware/markdown/generator.go | 135 ++++++++++++++++++ middleware/markdown/markdown.go | 2 +- middleware/markdown/markdown_test.go | 21 +-- middleware/markdown/page.go | 66 +-------- .../testdata/og_static/og/first.md/index.html | 13 -- middleware/markdown/watcher.go | 13 +- 7 files changed, 167 insertions(+), 141 deletions(-) create mode 100644 middleware/markdown/generator.go delete mode 100644 middleware/markdown/testdata/og_static/og/first.md/index.html diff --git a/config/setup/markdown.go b/config/setup/markdown.go index 562673a0c..f16558ec4 100644 --- a/config/setup/markdown.go +++ b/config/setup/markdown.go @@ -1,9 +1,7 @@ package setup import ( - "io/ioutil" "net/http" - "os" "path" "path/filepath" "strings" @@ -32,63 +30,15 @@ func Markdown(c *Controller) (middleware.Middleware, error) { for i := range mdconfigs { cfg := &mdconfigs[i] - // Links generation. - if err := markdown.GenerateLinks(md, cfg); err != nil { + // Generate static files. + if err := markdown.GenerateStatic(md, cfg); err != nil { return err } - // Watch file changes for links generation if not in development mode. + + // Watch file changes for static site generation if not in development mode. if !cfg.Development { markdown.Watch(md, cfg, markdown.DefaultInterval) } - - if cfg.StaticDir == "" { - continue - } - - // If generated site already exists, clear it out - _, err := os.Stat(cfg.StaticDir) - if err == nil { - err := os.RemoveAll(cfg.StaticDir) - if err != nil { - return err - } - } - - fp := filepath.Join(md.Root, cfg.PathScope) - - err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { - for _, ext := range cfg.Extensions { - if !info.IsDir() && strings.HasSuffix(info.Name(), ext) { - // Load the file - body, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - // Get the relative path as if it were a HTTP request, - // then prepend with "/" (like a real HTTP request) - reqPath, err := filepath.Rel(md.Root, path) - if err != nil { - return err - } - reqPath = "/" + reqPath - - // Generate the static file - ctx := middleware.Context{Root: md.FileSys} - _, err = md.Process(*cfg, reqPath, body, ctx) - if err != nil { - return err - } - - break // don't try other file extensions - } - } - return nil - }) - - if err != nil { - return err - } } return nil diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go new file mode 100644 index 000000000..685a210e1 --- /dev/null +++ b/middleware/markdown/generator.go @@ -0,0 +1,135 @@ +package markdown + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/mholt/caddy/middleware" +) + +// GenerateStatic generate static files from markdowns. +func GenerateStatic(md Markdown, cfg *Config) error { + generated, err := generateLinks(md, cfg) + if err != nil { + return err + } + + // No new file changes, return. + if !generated { + return nil + } + + // If static site generation is enabled. + if cfg.StaticDir != "" { + if err := generateStaticHTML(md, cfg); err != nil { + return err + } + } + return nil +} + +type linkGenerator struct { + gens map[*Config]*linkGen + sync.Mutex +} + +var generator = linkGenerator{gens: make(map[*Config]*linkGen)} + +// generateLinks generates links to all markdown files ordered by newest date. +// This blocks until link generation is done. When called by multiple goroutines, +// the first caller starts the generation and others only wait. +// It returns if generation is done and any error that occurred. +func generateLinks(md Markdown, cfg *Config) (bool, error) { + generator.Lock() + + // if link generator exists for config and running, wait. + if g, ok := generator.gens[cfg]; ok { + if g.started() { + g.addWaiter() + generator.Unlock() + g.Wait() + // another goroutine has done the generation. + return false, g.lastErr + } + } + + g := &linkGen{} + generator.gens[cfg] = g + generator.Unlock() + + generated := g.generateLinks(md, cfg) + g.discardWaiters() + return generated, g.lastErr +} + +// generateStaticFiles generates static html files from markdowns. +func generateStaticHTML(md Markdown, cfg *Config) error { + // If generated site already exists, clear it out + _, err := os.Stat(cfg.StaticDir) + if err == nil { + err := os.RemoveAll(cfg.StaticDir) + if err != nil { + return err + } + } + + fp := filepath.Join(md.Root, cfg.PathScope) + + return filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { + for _, ext := range cfg.Extensions { + if !info.IsDir() && strings.HasSuffix(info.Name(), ext) { + // Load the file + body, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + // Get the relative path as if it were a HTTP request, + // then prepend with "/" (like a real HTTP request) + reqPath, err := filepath.Rel(md.Root, path) + if err != nil { + return err + } + reqPath = "/" + reqPath + + // Generate the static file + ctx := middleware.Context{Root: md.FileSys} + _, err = md.Process(*cfg, reqPath, body, ctx) + if err != nil { + return err + } + + break // don't try other file extensions + } + } + return nil + }) +} + +// computeDirHash computes an hash on static directory of c. +func computeDirHash(md Markdown, c Config) (string, error) { + dir := filepath.Join(md.Root, c.PathScope) + if _, err := os.Stat(dir); err != nil { + return "", err + } + + hashString := "" + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) { + hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path) + } + return nil + }) + if err != nil { + return "", err + } + + sum := sha1.Sum([]byte(hashString)) + return hex.EncodeToString(sum[:]), nil +} diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index eba56001c..80ca45b12 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -122,7 +122,7 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error // if development is set, scan directory for file changes for links. if m.Development { - if err := GenerateLinks(md, m); err != nil { + if err := GenerateStatic(md, m); err != nil { log.Println(err) } } diff --git a/middleware/markdown/markdown_test.go b/middleware/markdown/markdown_test.go index ac91f677e..03c6093f6 100644 --- a/middleware/markdown/markdown_test.go +++ b/middleware/markdown/markdown_test.go @@ -68,6 +68,14 @@ func TestMarkdown(t *testing.T) { }), } + for i := range md.Configs { + c := &md.Configs[i] + if err := GenerateStatic(md, c); err != nil { + t.Fatalf("Error: %v", err) + } + Watch(md, c, time.Millisecond*100) + } + req, err := http.NewRequest("GET", "/blog/test.md", nil) if err != nil { t.Fatalf("Could not create HTTP request: %v", err) @@ -157,6 +165,7 @@ func getTrue() bool { err = os.Chtimes("testdata/og/first.md", currenttime, currenttime) currenttime = time.Now().Local() err = os.Chtimes("testdata/og_static/og/first.md/index.html", currenttime, currenttime) + time.Sleep(time.Millisecond * 200) md.ServeHTTP(rec, req) if rec.Code != http.StatusOK { @@ -169,8 +178,9 @@ func getTrue() bool { first_post -

Header title

+

Header

+Welcome to title!

Test h1

@@ -185,13 +195,6 @@ func getTrue() bool { "/log/test.md", } - for i := range md.Configs { - c := &md.Configs[i] - if err := GenerateLinks(md, c); err != nil { - t.Fatalf("Error: %v", err) - } - } - for i, c := range md.Configs[:2] { log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c) if c.Links[0].URL != expectedLinks[i] { @@ -218,7 +221,7 @@ func getTrue() bool { w.Wait() f = func() { - GenerateLinks(md, &md.Configs[0]) + GenerateStatic(md, &md.Configs[0]) w.Done() } for i := 0; i < 5; i++ { diff --git a/middleware/markdown/page.go b/middleware/markdown/page.go index 444b39ad3..4b179ebc7 100644 --- a/middleware/markdown/page.go +++ b/middleware/markdown/page.go @@ -2,9 +2,6 @@ package markdown import ( "bytes" - "crypto/sha1" - "encoding/hex" - "fmt" "io/ioutil" "log" "os" @@ -67,7 +64,9 @@ func (l *linkGen) started() bool { return l.generating } -func (l *linkGen) generateLinks(md Markdown, cfg *Config) { +// generateLinks generate links to markdown files if there are file changes. +// It returns true when generation is done and false otherwise. +func (l *linkGen) generateLinks(md Markdown, cfg *Config) bool { l.Lock() l.generating = true l.Unlock() @@ -81,7 +80,7 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) { l.lastErr = err l.generating = false l.Unlock() - return + return false } hash, err := computeDirHash(md, *cfg) @@ -91,7 +90,7 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) { l.Lock() l.generating = false l.Unlock() - return + return false } else if err != nil { log.Println("Error:", err) } @@ -162,58 +161,5 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) { l.Lock() l.generating = false l.Unlock() -} - -type linkGenerator struct { - gens map[*Config]*linkGen - sync.Mutex -} - -var generator = linkGenerator{gens: make(map[*Config]*linkGen)} - -// GenerateLinks generates links to all markdown files ordered by newest date. -// This blocks until link generation is done. When called by multiple goroutines, -// the first caller starts the generation and others only wait. -func GenerateLinks(md Markdown, cfg *Config) error { - generator.Lock() - - // if link generator exists for config and running, wait. - if g, ok := generator.gens[cfg]; ok { - if g.started() { - g.addWaiter() - generator.Unlock() - g.Wait() - return g.lastErr - } - } - - g := &linkGen{} - generator.gens[cfg] = g - generator.Unlock() - - g.generateLinks(md, cfg) - g.discardWaiters() - return g.lastErr -} - -// computeDirHash computes an hash on static directory of c. -func computeDirHash(md Markdown, c Config) (string, error) { - dir := filepath.Join(md.Root, c.PathScope) - if _, err := os.Stat(dir); err != nil { - return "", err - } - - hashString := "" - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) { - hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path) - } - return nil - }) - if err != nil { - return "", err - } - - sum := sha1.Sum([]byte(hashString)) - return hex.EncodeToString(sum[:]), nil + return true } diff --git a/middleware/markdown/testdata/og_static/og/first.md/index.html b/middleware/markdown/testdata/og_static/og/first.md/index.html deleted file mode 100644 index a58e17c1d..000000000 --- a/middleware/markdown/testdata/og_static/og/first.md/index.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - first_post - - -

Header title

- -

Test h1

- - - \ No newline at end of file diff --git a/middleware/markdown/watcher.go b/middleware/markdown/watcher.go index fcf1d36ea..76ef2ed5f 100644 --- a/middleware/markdown/watcher.go +++ b/middleware/markdown/watcher.go @@ -1,6 +1,9 @@ package markdown -import "time" +import ( + "log" + "time" +) const DefaultInterval = time.Second * 60 @@ -8,12 +11,14 @@ const DefaultInterval = time.Second * 60 // when there are changes. func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) { return TickerFunc(interval, func() { - GenerateLinks(md, c) + if err := GenerateStatic(md, c); err != nil { + log.Println(err) + } }) } -// TickerFunc runs f at interval. If interval is <= 0, it loops f. A message to the -// returned channel will stop the executing goroutine. +// TickerFunc runs f at interval. A message to the returned channel will stop the +// executing goroutine. func TickerFunc(interval time.Duration, f func()) chan struct{} { stopChan := make(chan struct{})