From 851026d3faee3890084f972877d21f803ff53611 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 4 Aug 2015 23:35:09 +0100 Subject: [PATCH] Markdown: Watch for file changes. Removed sitegen dependency for links. --- config/setup/markdown.go | 31 ++++++++-- middleware/markdown/markdown.go | 25 +++++--- middleware/markdown/markdown_test.go | 37 ++++++++---- middleware/markdown/page.go | 37 ++++++++++++ middleware/markdown/testdata/og/first.md | 4 ++ .../testdata/og_static/og/first.md/index.html | 5 +- middleware/markdown/watcher.go | 58 +++++++++++++++++++ middleware/markdown/watcher_test.go | 34 +++++++++++ 8 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 middleware/markdown/watcher.go create mode 100644 middleware/markdown/watcher_test.go diff --git a/config/setup/markdown.go b/config/setup/markdown.go index 953e668dd..043ead3ad 100644 --- a/config/setup/markdown.go +++ b/config/setup/markdown.go @@ -29,13 +29,22 @@ func Markdown(c *Controller) (middleware.Middleware, error) { // For any configs that enabled static site gen, sweep the whole path at startup c.Startup = append(c.Startup, func() error { - for _, cfg := range mdconfigs { - if cfg.StaticDir == "" { - continue + for i := range mdconfigs { + cfg := &mdconfigs[i] + + // Links generation. + if err := markdown.GenerateLinks(md, cfg); err != nil { + return err + } + // Watch file changes for links generation. + if cfg.Development { + markdown.Watch(md, cfg, 0) + } else { + markdown.Watch(md, cfg, markdown.DefaultInterval) } - if err := markdown.GenerateLinks(md, &cfg); err != nil { - return err + if cfg.StaticDir == "" { + continue } // If generated site already exists, clear it out @@ -68,7 +77,7 @@ func Markdown(c *Controller) (middleware.Middleware, error) { // Generate the static file ctx := middleware.Context{Root: md.FileSys} - _, err = md.Process(cfg, reqPath, body, ctx) + _, err = md.Process(*cfg, reqPath, body, ctx) if err != nil { return err } @@ -155,6 +164,16 @@ func markdownParse(c *Controller) ([]markdown.Config, error) { // only 1 argument allowed return mdconfigs, c.ArgErr() } + case "development": + if c.NextArg() { + md.Development = strings.ToLower(c.Val()) == "true" + } else { + md.Development = true + } + if c.NextArg() { + // only 1 argument allowed + return mdconfigs, c.ArgErr() + } default: return mdconfigs, c.Err("Expected valid markdown configuration property") } diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index 03f4e077a..5855413fa 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -4,7 +4,6 @@ package markdown import ( "io/ioutil" - "log" "net/http" "os" "strings" @@ -69,12 +68,29 @@ type Config struct { // Links to all markdown pages ordered by date. Links []PageLink + // Stores a directory hash to check for changes. + linksHash string + // Directory to store static files StaticDir string + // If in development mode. i.e. Actively editing markdown files. + Development bool + sync.RWMutex } +// IsValidExt checks to see if an extension is a valid markdown extension +// for config. +func (c Config) IsValidExt(ext string) bool { + for _, e := range c.Extensions { + if e == ext { + return true + } + } + return false +} + // ServeHTTP implements the http.Handler interface. func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { for i := range md.Configs { @@ -122,13 +138,6 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error } } - if m.StaticDir != "" { - // Markdown modified or new. Update links. - if err := GenerateLinks(md, m); err != nil { - log.Println(err) - } - } - body, err := ioutil.ReadAll(f) if err != nil { return http.StatusInternalServerError, err diff --git a/middleware/markdown/markdown_test.go b/middleware/markdown/markdown_test.go index bb906972c..69af55b79 100644 --- a/middleware/markdown/markdown_test.go +++ b/middleware/markdown/markdown_test.go @@ -1,6 +1,7 @@ package markdown import ( + "bufio" "log" "net/http" "net/http/httptest" @@ -102,7 +103,7 @@ func getTrue() bool { ` - if respBody != expectedBody { + if !equalStrings(respBody, expectedBody) { t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } @@ -143,10 +144,7 @@ func getTrue() bool { ` - replacer := strings.NewReplacer("\r", "", "\n", "") - respBody = replacer.Replace(respBody) - expectedBody = replacer.Replace(expectedBody) - if respBody != expectedBody { + if !equalStrings(respBody, expectedBody) { t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } @@ -177,19 +175,24 @@ func getTrue() bool { ` - respBody = replacer.Replace(respBody) - expectedBody = replacer.Replace(expectedBody) - if respBody != expectedBody { + + if !equalStrings(respBody, expectedBody) { t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } expectedLinks := []string{ "/blog/test.md", "/log/test.md", - "/og/first.md", } - for i, c := range md.Configs { + for i := range md.Configs { + c := &md.Configs[i] + if err := GenerateLinks(md, c); err != nil { + t.Fatalf("Error: %v", err) + } + } + + for i, c := range md.Configs[:2] { log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c) if c.Links[0].URL != expectedLinks[i] { t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].URL) @@ -219,3 +222,17 @@ func getTrue() bool { } } + +func equalStrings(s1, s2 string) bool { + s1 = strings.TrimSpace(s1) + s2 = strings.TrimSpace(s2) + in := bufio.NewScanner(strings.NewReader(s1)) + for in.Scan() { + txt := strings.TrimSpace(in.Text()) + if !strings.HasPrefix(strings.TrimSpace(s2), txt) { + return false + } + s2 = strings.Replace(s2, txt, "", 1) + } + return true +} diff --git a/middleware/markdown/page.go b/middleware/markdown/page.go index d70d89425..97f1c5a9a 100644 --- a/middleware/markdown/page.go +++ b/middleware/markdown/page.go @@ -2,7 +2,11 @@ package markdown import ( "bytes" + "crypto/sha1" + "encoding/hex" + "fmt" "io/ioutil" + "log" "os" "path/filepath" "sort" @@ -79,6 +83,15 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) { return } + hash, err := computeDirHash(md, *cfg) + + // same hash, return. + if err == nil && hash == cfg.linksHash { + return + } else if err != nil { + log.Println("Error:", err) + } + cfg.Links = []PageLink{} cfg.Lock() @@ -138,6 +151,8 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) { // sort by newest date sort.Sort(byDate(cfg.Links)) + + cfg.linksHash = hash cfg.Unlock() l.Lock() @@ -176,3 +191,25 @@ func GenerateLinks(md Markdown, cfg *Config) error { g.discardWaiters() return g.lastErr } + +// computeDirHash computes an hash on static directory of c. +func computeDirHash(md Markdown, c Config) (string, error) { + dir := filepath.Join(md.Root, c.PathScope) + if _, err := os.Stat(dir); err != nil { + return "", err + } + + hashString := "" + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) { + hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path) + } + return nil + }) + if err != nil { + return "", err + } + + sum := sha1.Sum([]byte(hashString)) + return hex.EncodeToString(sum[:]), nil +} diff --git a/middleware/markdown/testdata/og/first.md b/middleware/markdown/testdata/og/first.md index f26583b75..4d7a4251f 100644 --- a/middleware/markdown/testdata/og/first.md +++ b/middleware/markdown/testdata/og/first.md @@ -1 +1,5 @@ +--- +title: first_post +sitename: title +--- # Test h1 diff --git a/middleware/markdown/testdata/og_static/og/first.md/index.html b/middleware/markdown/testdata/og_static/og/first.md/index.html index 4dd4a5a24..a58e17c1d 100644 --- a/middleware/markdown/testdata/og_static/og/first.md/index.html +++ b/middleware/markdown/testdata/og_static/og/first.md/index.html @@ -1,7 +1,8 @@ + -first_post + first_post

Header title

@@ -9,4 +10,4 @@

Test h1

- + \ No newline at end of file diff --git a/middleware/markdown/watcher.go b/middleware/markdown/watcher.go new file mode 100644 index 000000000..5f311fe1b --- /dev/null +++ b/middleware/markdown/watcher.go @@ -0,0 +1,58 @@ +package markdown + +import "time" + +const ( + DefaultInterval = time.Second * 60 + DevInterval = time.Second * 1 +) + +// Watch monitors the configured markdown directory for changes. It calls GenerateLinks +// when there are changes. +func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) { + return TickerFunc(interval, func() { + GenerateLinks(md, c) + }) +} + +// TickerFunc runs f at interval. If interval is <= 0, it loops f. A message to the +// returned channel will stop the executing goroutine. +func TickerFunc(interval time.Duration, f func()) chan struct{} { + stopChan := make(chan struct{}) + + if interval > 0 { + ticker := time.NewTicker(interval) + go func() { + loop: + for { + select { + case <-ticker.C: + f() + case <-stopChan: + ticker.Stop() + break loop + } + } + }() + } else { + go func() { + loop: + for { + m := make(chan struct{}) + go func() { + f() + m <- struct{}{} + }() + select { + case <-m: + continue loop + case <-stopChan: + break loop + } + time.Sleep(DevInterval) + + } + }() + } + return stopChan +} diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go new file mode 100644 index 000000000..6aa7364b5 --- /dev/null +++ b/middleware/markdown/watcher_test.go @@ -0,0 +1,34 @@ +package markdown + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestWatcher(t *testing.T) { + expected := "12345678" + interval := time.Millisecond * 100 + i := 0 + out := "" + stopChan := TickerFunc(interval, func() { + i++ + out += fmt.Sprint(i) + }) + time.Sleep(interval * 8) + stopChan <- struct{}{} + if expected != out { + t.Fatalf("Expected %v, found %v", expected, out) + } + out = "" + i = 0 + stopChan = TickerFunc(interval, func() { + i++ + out += fmt.Sprint(i) + }) + time.Sleep(interval * 10) + if !strings.HasPrefix(out, expected) || out == expected { + t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out) + } +}