Merge pull request #214 from abiosoft/master

markdown: generate static sites after links

config/setup/markdown.go

@@ -1,9 +1,7 @@
 package setup
 
 import (
-	"io/ioutil"
 	"net/http"
-	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -32,63 +30,15 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
 		for i := range mdconfigs {
 			cfg := &mdconfigs[i]
 
-			// Links generation.
+			// Generate static files.
-			if err := markdown.GenerateLinks(md, cfg); err != nil {
+			if err := markdown.GenerateStatic(md, cfg); err != nil {
 				return err
 			}
-			// Watch file changes for links generation if not in development mode.
+
+			// Watch file changes for static site generation if not in development mode.
 			if !cfg.Development {
 				markdown.Watch(md, cfg, markdown.DefaultInterval)
 			}
-
-			if cfg.StaticDir == "" {
-				continue
-			}
-
-			// If generated site already exists, clear it out
-			_, err := os.Stat(cfg.StaticDir)
-			if err == nil {
-				err := os.RemoveAll(cfg.StaticDir)
-				if err != nil {
-					return err
-				}
-			}
-
-			fp := filepath.Join(md.Root, cfg.PathScope)
-
-			err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
-				for _, ext := range cfg.Extensions {
-					if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
-						// Load the file
-						body, err := ioutil.ReadFile(path)
-						if err != nil {
-							return err
-						}
-
-						// Get the relative path as if it were a HTTP request,
-						// then prepend with "/" (like a real HTTP request)
-						reqPath, err := filepath.Rel(md.Root, path)
-						if err != nil {
-							return err
-						}
-						reqPath = "/" + reqPath
-
-						// Generate the static file
-						ctx := middleware.Context{Root: md.FileSys}
-						_, err = md.Process(*cfg, reqPath, body, ctx)
-						if err != nil {
-							return err
-						}
-
-						break // don't try other file extensions
-					}
-				}
-				return nil
-			})
-
-			if err != nil {
-				return err
-			}
 		}
 
 		return nil

middleware/markdown/generator.go

@@ -0,0 +1,135 @@
+package markdown
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/mholt/caddy/middleware"
+)
+
+// GenerateStatic generate static files from markdowns.
+func GenerateStatic(md Markdown, cfg *Config) error {
+	generated, err := generateLinks(md, cfg)
+	if err != nil {
+		return err
+	}
+
+	// No new file changes, return.
+	if !generated {
+		return nil
+	}
+
+	// If static site generation is enabled.
+	if cfg.StaticDir != "" {
+		if err := generateStaticHTML(md, cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type linkGenerator struct {
+	gens map[*Config]*linkGen
+	sync.Mutex
+}
+
+var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
+
+// generateLinks generates links to all markdown files ordered by newest date.
+// This blocks until link generation is done. When called by multiple goroutines,
+// the first caller starts the generation and others only wait.
+// It returns if generation is done and any error that occurred.
+func generateLinks(md Markdown, cfg *Config) (bool, error) {
+	generator.Lock()
+
+	// if link generator exists for config and running, wait.
+	if g, ok := generator.gens[cfg]; ok {
+		if g.started() {
+			g.addWaiter()
+			generator.Unlock()
+			g.Wait()
+			// another goroutine has done the generation.
+			return false, g.lastErr
+		}
+	}
+
+	g := &linkGen{}
+	generator.gens[cfg] = g
+	generator.Unlock()
+
+	generated := g.generateLinks(md, cfg)
+	g.discardWaiters()
+	return generated, g.lastErr
+}
+
+// generateStaticFiles generates static html files from markdowns.
+func generateStaticHTML(md Markdown, cfg *Config) error {
+	// If generated site already exists, clear it out
+	_, err := os.Stat(cfg.StaticDir)
+	if err == nil {
+		err := os.RemoveAll(cfg.StaticDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	fp := filepath.Join(md.Root, cfg.PathScope)
+
+	return filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
+		for _, ext := range cfg.Extensions {
+			if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
+				// Load the file
+				body, err := ioutil.ReadFile(path)
+				if err != nil {
+					return err
+				}
+
+				// Get the relative path as if it were a HTTP request,
+				// then prepend with "/" (like a real HTTP request)
+				reqPath, err := filepath.Rel(md.Root, path)
+				if err != nil {
+					return err
+				}
+				reqPath = "/" + reqPath
+
+				// Generate the static file
+				ctx := middleware.Context{Root: md.FileSys}
+				_, err = md.Process(*cfg, reqPath, body, ctx)
+				if err != nil {
+					return err
+				}
+
+				break // don't try other file extensions
+			}
+		}
+		return nil
+	})
+}
+
+// computeDirHash computes an hash on static directory of c.
+func computeDirHash(md Markdown, c Config) (string, error) {
+	dir := filepath.Join(md.Root, c.PathScope)
+	if _, err := os.Stat(dir); err != nil {
+		return "", err
+	}
+
+	hashString := ""
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) {
+			hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path)
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	sum := sha1.Sum([]byte(hashString))
+	return hex.EncodeToString(sum[:]), nil
+}

middleware/markdown/markdown.go

@@ -122,7 +122,7 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 
 				// if development is set, scan directory for file changes for links.
 				if m.Development {
-					if err := GenerateLinks(md, m); err != nil {
+					if err := GenerateStatic(md, m); err != nil {
 						log.Println(err)
 					}
 				}

middleware/markdown/markdown_test.go

@@ -68,6 +68,14 @@ func TestMarkdown(t *testing.T) {
 		}),
 	}
 
+	for i := range md.Configs {
+		c := &md.Configs[i]
+		if err := GenerateStatic(md, c); err != nil {
+			t.Fatalf("Error: %v", err)
+		}
+		Watch(md, c, time.Millisecond*100)
+	}
+
 	req, err := http.NewRequest("GET", "/blog/test.md", nil)
 	if err != nil {
 		t.Fatalf("Could not create HTTP request: %v", err)
@@ -157,6 +165,7 @@ func getTrue() bool {
 	err = os.Chtimes("testdata/og/first.md", currenttime, currenttime)
 	currenttime = time.Now().Local()
 	err = os.Chtimes("testdata/og_static/og/first.md/index.html", currenttime, currenttime)
+	time.Sleep(time.Millisecond * 200)
 
 	md.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -169,8 +178,9 @@ func getTrue() bool {
 <title>first_post</title>
 </head>
 <body>
-<h1>Header title</h1>
+<h1>Header</h1>
 
+Welcome to title!
 <h1>Test h1</h1>
 
 </body>
@@ -185,13 +195,6 @@ func getTrue() bool {
 		"/log/test.md",
 	}
 
-	for i := range md.Configs {
-		c := &md.Configs[i]
-		if err := GenerateLinks(md, c); err != nil {
-			t.Fatalf("Error: %v", err)
-		}
-	}
-
 	for i, c := range md.Configs[:2] {
 		log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c)
 		if c.Links[0].URL != expectedLinks[i] {
@@ -218,7 +221,7 @@ func getTrue() bool {
 	w.Wait()
 
 	f = func() {
-		GenerateLinks(md, &md.Configs[0])
+		GenerateStatic(md, &md.Configs[0])
 		w.Done()
 	}
 	for i := 0; i < 5; i++ {

middleware/markdown/page.go

@@ -2,9 +2,6 @@ package markdown
 
 import (
 	"bytes"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -67,7 +64,9 @@ func (l *linkGen) started() bool {
 	return l.generating
 }
 
-func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
+// generateLinks generate links to markdown files if there are file changes.
+// It returns true when generation is done and false otherwise.
+func (l *linkGen) generateLinks(md Markdown, cfg *Config) bool {
 	l.Lock()
 	l.generating = true
 	l.Unlock()
@@ -81,7 +80,7 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
 		l.lastErr = err
 		l.generating = false
 		l.Unlock()
-		return
+		return false
 	}
 
 	hash, err := computeDirHash(md, *cfg)
@@ -91,7 +90,7 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
 		l.Lock()
 		l.generating = false
 		l.Unlock()
-		return
+		return false
 	} else if err != nil {
 		log.Println("Error:", err)
 	}
@@ -162,58 +161,5 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
 	l.Lock()
 	l.generating = false
 	l.Unlock()
-}
+	return true
-
-type linkGenerator struct {
-	gens map[*Config]*linkGen
-	sync.Mutex
-}
-
-var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
-
-// GenerateLinks generates links to all markdown files ordered by newest date.
-// This blocks until link generation is done. When called by multiple goroutines,
-// the first caller starts the generation and others only wait.
-func GenerateLinks(md Markdown, cfg *Config) error {
-	generator.Lock()
-
-	// if link generator exists for config and running, wait.
-	if g, ok := generator.gens[cfg]; ok {
-		if g.started() {
-			g.addWaiter()
-			generator.Unlock()
-			g.Wait()
-			return g.lastErr
-		}
-	}
-
-	g := &linkGen{}
-	generator.gens[cfg] = g
-	generator.Unlock()
-
-	g.generateLinks(md, cfg)
-	g.discardWaiters()
-	return g.lastErr
-}
-
-// computeDirHash computes an hash on static directory of c.
-func computeDirHash(md Markdown, c Config) (string, error) {
-	dir := filepath.Join(md.Root, c.PathScope)
-	if _, err := os.Stat(dir); err != nil {
-		return "", err
-	}
-
-	hashString := ""
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) {
-			hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path)
-		}
-		return nil
-	})
-	if err != nil {
-		return "", err
-	}
-
-	sum := sha1.Sum([]byte(hashString))
-	return hex.EncodeToString(sum[:]), nil
 }

middleware/markdown/testdata/og_static/og/first.md/index.html

@@ -1,13 +0,0 @@
-
-<!DOCTYPE html>
-<html>
-<head>
-    <title>first_post</title>
-</head>
-<body>
-<h1>Header title</h1>
-
-<h1>Test h1</h1>
-
-</body>
-</html>

middleware/markdown/watcher.go

@@ -1,6 +1,9 @@
 package markdown
 
-import "time"
+import (
+	"log"
+	"time"
+)
 
 const DefaultInterval = time.Second * 60
 
@@ -8,12 +11,14 @@ const DefaultInterval = time.Second * 60
 // when there are changes.
 func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) {
 	return TickerFunc(interval, func() {
-		GenerateLinks(md, c)
+		if err := GenerateStatic(md, c); err != nil {
+			log.Println(err)
+		}
 	})
 }
 
-// TickerFunc runs f at interval. If interval is <= 0, it loops f. A message to the
+// TickerFunc runs f at interval. A message to the returned channel will stop the
-// returned channel will stop the executing goroutine.
+// executing goroutine.
 func TickerFunc(interval time.Duration, f func()) chan struct{} {
 	stopChan := make(chan struct{})