diff --git a/config/setup/fastcgi_test.go b/config/setup/fastcgi_test.go index a0643e30c..f32f84d3d 100644 --- a/config/setup/fastcgi_test.go +++ b/config/setup/fastcgi_test.go @@ -1,6 +1,7 @@ package setup import ( + "fmt" "github.com/mholt/caddy/middleware/fastcgi" "testing" ) @@ -34,3 +35,63 @@ func TestFastCGI(t *testing.T) { } } + +func TestFastcgiParse(t *testing.T) { + tests := []struct { + inputFastcgiConfig string + shouldErr bool + expectedFastcgiConfig []fastcgi.Rule + }{ + + {`fastcgi /blog 127.0.0.1:9000 php`, + false, []fastcgi.Rule{{ + Path: "/blog", + Address: "127.0.0.1:9000", + Ext: ".php", + SplitPath: ".php", + IndexFiles: []string{"index.php"}, + }}}, + } + for i, test := range tests { + c := NewTestController(test.inputFastcgiConfig) + actualFastcgiConfigs, err := fastcgiParse(c) + + if err == nil && test.shouldErr { + t.Errorf("Test %d didn't error, but it should have", i) + } else if err != nil && !test.shouldErr { + t.Errorf("Test %d errored, but it shouldn't have; got '%v'", i, err) + } + if len(actualFastcgiConfigs) != len(test.expectedFastcgiConfig) { + t.Fatalf("Test %d expected %d no of FastCGI configs, but got %d ", + i, len(test.expectedFastcgiConfig), len(actualFastcgiConfigs)) + } + for j, actualFastcgiConfig := range actualFastcgiConfigs { + + if actualFastcgiConfig.Path != test.expectedFastcgiConfig[j].Path { + t.Errorf("Test %d expected %dth FastCGI Path to be %s , but got %s", + i, j, test.expectedFastcgiConfig[j].Path, actualFastcgiConfig.Path) + } + + if actualFastcgiConfig.Address != test.expectedFastcgiConfig[j].Address { + t.Errorf("Test %d expected %dth FastCGI Address to be %s , but got %s", + i, j, test.expectedFastcgiConfig[j].Address, actualFastcgiConfig.Address) + } + + if actualFastcgiConfig.Ext != test.expectedFastcgiConfig[j].Ext { + t.Errorf("Test %d expected %dth FastCGI Ext to be %s , but got %s", + i, j, test.expectedFastcgiConfig[j].Ext, actualFastcgiConfig.Ext) + } + + if actualFastcgiConfig.SplitPath != test.expectedFastcgiConfig[j].SplitPath { + t.Errorf("Test %d expected %dth FastCGI SplitPath to be %s , but got %s", + i, j, test.expectedFastcgiConfig[j].SplitPath, actualFastcgiConfig.SplitPath) + } + + if fmt.Sprint(actualFastcgiConfig.IndexFiles) != fmt.Sprint(test.expectedFastcgiConfig[j].IndexFiles) { + t.Errorf("Test %d expected %dth FastCGI IndexFiles to be %s , but got %s", + i, j, test.expectedFastcgiConfig[j].IndexFiles, actualFastcgiConfig.IndexFiles) + } + } + } + +} diff --git a/config/setup/markdown.go b/config/setup/markdown.go index bbccf9d5f..e5428c59d 100644 --- a/config/setup/markdown.go +++ b/config/setup/markdown.go @@ -34,6 +34,10 @@ func Markdown(c *Controller) (middleware.Middleware, error) { continue } + if err := markdown.GenerateLinks(md, &cfg); err != nil { + return err + } + // If generated site already exists, clear it out _, err := os.Stat(cfg.StaticDir) if err == nil { diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index 71b189068..03f4e077a 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -4,9 +4,11 @@ package markdown import ( "io/ioutil" + "log" "net/http" "os" "strings" + "sync" "github.com/mholt/caddy/middleware" "github.com/russross/blackfriday" @@ -64,13 +66,19 @@ type Config struct { // Map of request URL to static files generated StaticFiles map[string]string + // Links to all markdown pages ordered by date. + Links []PageLink + // Directory to store static files StaticDir string + + sync.RWMutex } // ServeHTTP implements the http.Handler interface. func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { - for _, m := range md.Configs { + for i := range md.Configs { + m := &md.Configs[i] if !middleware.Path(r.URL.Path).Matches(m.PathScope) { continue } @@ -114,6 +122,13 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error } } + if m.StaticDir != "" { + // Markdown modified or new. Update links. + if err := GenerateLinks(md, m); err != nil { + log.Println(err) + } + } + body, err := ioutil.ReadAll(f) if err != nil { return http.StatusInternalServerError, err @@ -124,7 +139,7 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error Req: r, URL: r.URL, } - html, err := md.Process(m, fpath, body, ctx) + html, err := md.Process(*m, fpath, body, ctx) if err != nil { return http.StatusInternalServerError, err } diff --git a/middleware/markdown/markdown_test.go b/middleware/markdown/markdown_test.go index 1cb5301dc..917cff9bf 100644 --- a/middleware/markdown/markdown_test.go +++ b/middleware/markdown/markdown_test.go @@ -1,10 +1,12 @@ package markdown import ( + "log" "net/http" "net/http/httptest" "os" "strings" + "sync" "testing" "time" @@ -20,20 +22,24 @@ func TestMarkdown(t *testing.T) { FileSys: http.Dir("./testdata"), Configs: []Config{ Config{ - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/blog", - Extensions: []string{".md"}, - Styles: []string{}, - Scripts: []string{}, - Templates: templates, + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/blog", + Extensions: []string{".md"}, + Styles: []string{}, + Scripts: []string{}, + Templates: templates, + StaticDir: DefaultStaticDir, + StaticFiles: make(map[string]string), }, Config{ - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/log", - Extensions: []string{".md"}, - Styles: []string{"/resources/css/log.css", "/resources/css/default.css"}, - Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"}, - Templates: make(map[string]string), + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/log", + Extensions: []string{".md"}, + Styles: []string{"/resources/css/log.css", "/resources/css/default.css"}, + Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"}, + Templates: make(map[string]string), + StaticDir: DefaultStaticDir, + StaticFiles: make(map[string]string), }, Config{ Renderer: blackfriday.HtmlRenderer(0, "", ""), @@ -44,6 +50,14 @@ func TestMarkdown(t *testing.T) { Templates: templates, StaticDir: "testdata/og_static", StaticFiles: map[string]string{"/og/first.md": "testdata/og_static/og/first.md/index.html"}, + Links: []PageLink{ + PageLink{ + Title: "first", + Summary: "", + Date: time.Now(), + Url: "/og/first.md", + }, + }, }, }, IndexFiles: []string{"index.html"}, @@ -168,4 +182,40 @@ func getTrue() bool { if respBody != expectedBody { t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } + + expectedLinks := []string{ + "/blog/test.md", + "/log/test.md", + "/og/first.md", + } + + for i, c := range md.Configs { + log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c) + if c.Links[0].Url != expectedLinks[i] { + t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].Url) + } + } + + // attempt to trigger race condition + var w sync.WaitGroup + f := func() { + req, err := http.NewRequest("GET", "/log/test.md", nil) + if err != nil { + t.Fatalf("Could not create HTTP request: %v", err) + } + rec := httptest.NewRecorder() + + md.ServeHTTP(rec, req) + w.Done() + } + for i := 0; i < 5; i++ { + w.Add(1) + go f() + } + w.Wait() + + if err = os.RemoveAll(DefaultStaticDir); err != nil { + t.Errorf("Error while removing the generated static files: %v", err) + } + } diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 9dc2cf1ba..103b97b23 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -9,14 +9,7 @@ import ( "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" -) - -var ( - parsers = []MetadataParser{ - &JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - } + "time" ) // Metadata stores a page's metadata @@ -27,20 +20,31 @@ type Metadata struct { // Page template Template string + // Publish date + Date time.Time + // Variables to be used with Template Variables map[string]string } // load loads parsed values in parsedMap into Metadata func (m *Metadata) load(parsedMap map[string]interface{}) { - if template, ok := parsedMap["title"]; ok { - m.Title, _ = template.(string) + if title, ok := parsedMap["title"]; ok { + m.Title, _ = title.(string) } if template, ok := parsedMap["template"]; ok { m.Template, _ = template.(string) } - if variables, ok := parsedMap["variables"]; ok { - m.Variables, _ = variables.(map[string]string) + if date, ok := parsedMap["date"].(string); ok { + if t, err := time.Parse(timeLayout, date); err == nil { + m.Date = t + } + } + // store everything as a variable + for key, val := range parsedMap { + if v, ok := val.(string); ok { + m.Variables[key] = v + } } } @@ -62,7 +66,7 @@ type MetadataParser interface { Metadata() Metadata } -// JSONMetadataParser is the MetdataParser for JSON +// JSONMetadataParser is the MetadataParser for JSON type JSONMetadataParser struct { metadata Metadata } @@ -76,16 +80,6 @@ func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { if err := decoder.Decode(&m); err != nil { return b, err } - if vars, ok := m["variables"].(map[string]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if val, ok := v.(string); ok { - vars1[k] = val - } - } - m["variables"] = vars1 - } - j.metadata.load(m) // Retrieve remaining bytes after decoding @@ -129,15 +123,6 @@ func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) { if err := toml.Unmarshal(b, &m); err != nil { return markdown, err } - if vars, ok := m["variables"].(map[string]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if val, ok := v.(string); ok { - vars1[k] = val - } - } - m["variables"] = vars1 - } t.metadata.load(m) return markdown, nil } @@ -174,21 +159,6 @@ func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) { if err := yaml.Unmarshal(b, &m); err != nil { return markdown, err } - - // convert variables (if present) to map[string]interface{} - // to match expected type - if vars, ok := m["variables"].(map[interface{}]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if key, ok := k.(string); ok { - if val, ok := v.(string); ok { - vars1[key] = val - } - } - } - m["variables"] = vars1 - } - y.metadata.load(m) return markdown, nil } @@ -260,10 +230,19 @@ func findParser(b []byte) MetadataParser { return nil } line = bytes.TrimSpace(line) - for _, parser := range parsers { + for _, parser := range parsers() { if bytes.Equal(parser.Opening(), line) { return parser } } return nil } + +// parsers returns all available parsers +func parsers() []MetadataParser { + return []MetadataParser{ + &JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + &TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + &YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + } +} diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index c1028f293..b93c9ddcb 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -11,13 +11,11 @@ import ( var TOML = [4]string{` title = "A title" template = "default" -[variables] name = "value" `, `+++ title = "A title" template = "default" -[variables] name = "value" +++ Page content @@ -25,7 +23,6 @@ Page content `+++ title = "A title" template = "default" -[variables] name = "value" `, `title = "A title" template = "default" [variables] name = "value"`, @@ -34,38 +31,31 @@ name = "value" var YAML = [4]string{` title : A title template : default -variables : - name : value +name : value `, `--- title : A title template : default -variables : - name : value +name : value --- Page content `, `--- title : A title template : default -variables : - name : value +name : value `, `title : A title template : default variables : name : value`, } var JSON = [4]string{` "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" `, `{ "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" } Page content `, @@ -73,17 +63,13 @@ Page content { "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" `, ` {{ "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" } `, } @@ -96,9 +82,13 @@ func check(t *testing.T, err error) { func TestParsers(t *testing.T) { expected := Metadata{ - Title: "A title", - Template: "default", - Variables: map[string]string{"name": "value"}, + Title: "A title", + Template: "default", + Variables: map[string]string{ + "name": "value", + "title": "A title", + "template": "default", + }, } compare := func(m Metadata) bool { if m.Title != expected.Title { @@ -112,7 +102,7 @@ func TestParsers(t *testing.T) { return false } } - return len(m.Variables) == 1 + return len(m.Variables) == len(expected.Variables) } data := []struct { @@ -120,9 +110,9 @@ func TestParsers(t *testing.T) { testData [4]string name string }{ - {&JSONMetadataParser{}, JSON, "json"}, - {&YAMLMetadataParser{}, YAML, "yaml"}, - {&TOMLMetadataParser{}, TOML, "toml"}, + {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, + {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, } for _, v := range data { diff --git a/middleware/markdown/page.go b/middleware/markdown/page.go new file mode 100644 index 000000000..91dfb37b9 --- /dev/null +++ b/middleware/markdown/page.go @@ -0,0 +1,161 @@ +package markdown + +import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/russross/blackfriday" +) + +const ( + // Date format YYYY-MM-DD HH:MM:SS + timeLayout = `2006-01-02 15:04:05` + + // Length of page summary. + summaryLen = 150 +) + +// PageLink represents a statically generated markdown page. +type PageLink struct { + Title string + Summary string + Date time.Time + Url string +} + +// byDate sorts PageLink by newest date to oldest. +type byDate []PageLink + +func (p byDate) Len() int { return len(p) } +func (p byDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) } + +type linkGen struct { + generating bool + waiters int + lastErr error + sync.RWMutex + sync.WaitGroup +} + +func (l *linkGen) addWaiter() { + l.WaitGroup.Add(1) + l.waiters++ +} + +func (l *linkGen) discardWaiters() { + l.Lock() + defer l.Unlock() + for i := 0; i < l.waiters; i++ { + l.Done() + } +} + +func (l *linkGen) started() bool { + l.RLock() + defer l.RUnlock() + return l.generating +} + +func (l *linkGen) generateLinks(md Markdown, cfg *Config) { + l.Lock() + l.generating = true + l.Unlock() + + fp := filepath.Join(md.Root, cfg.PathScope) + + cfg.Links = []PageLink{} + + cfg.Lock() + l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { + for _, ext := range cfg.Extensions { + if !info.IsDir() && strings.HasSuffix(info.Name(), ext) { + // Load the file + body, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + // Get the relative path as if it were a HTTP request, + // then prepend with "/" (like a real HTTP request) + reqPath, err := filepath.Rel(md.Root, path) + if err != nil { + return err + } + reqPath = "/" + reqPath + + parser := findParser(body) + if parser == nil { + // no metadata, ignore. + continue + } + summary, err := parser.Parse(body) + if err != nil { + return err + } + + if len(summary) > summaryLen { + summary = summary[:summaryLen] + } + + metadata := parser.Metadata() + + cfg.Links = append(cfg.Links, PageLink{ + Title: metadata.Title, + Url: reqPath, + Date: metadata.Date, + Summary: string(blackfriday.Markdown(summary, PlaintextRenderer{}, 0)), + }) + + break // don't try other file extensions + } + } + + return nil + }) + + // sort by newest date + sort.Sort(byDate(cfg.Links)) + cfg.Unlock() + + l.Lock() + l.generating = false + l.Unlock() +} + +type linkGenerator struct { + gens map[*Config]*linkGen + sync.Mutex +} + +var generator = linkGenerator{gens: make(map[*Config]*linkGen)} + +// GenerateLinks generates links to all markdown files ordered by newest date. +// This blocks until link generation is done. When called by multiple goroutines, +// the first caller starts the generation and others only wait. +func GenerateLinks(md Markdown, cfg *Config) error { + generator.Lock() + + // if link generator exists for config and running, wait. + if g, ok := generator.gens[cfg]; ok { + if g.started() { + g.addWaiter() + generator.Unlock() + g.Wait() + return g.lastErr + } + } + + g := &linkGen{} + generator.gens[cfg] = g + generator.Unlock() + + g.generateLinks(md, cfg) + g.discardWaiters() + return g.lastErr +} diff --git a/middleware/markdown/process.go b/middleware/markdown/process.go index b97c205a5..8afd92b7b 100644 --- a/middleware/markdown/process.go +++ b/middleware/markdown/process.go @@ -20,7 +20,8 @@ const ( type MarkdownData struct { middleware.Context - Doc map[string]string + Doc map[string]string + Links []PageLink } // Process processes the contents of a page in b. It parses the metadata @@ -97,9 +98,14 @@ func (md Markdown) processTemplate(c Config, requestPath string, tmpl []byte, me mdData := MarkdownData{ Context: ctx, Doc: metadata.Variables, + Links: c.Links, } - if err = t.Execute(b, mdData); err != nil { + c.RLock() + err = t.Execute(b, mdData) + c.RUnlock() + + if err != nil { return nil, err } diff --git a/middleware/markdown/renderer.go b/middleware/markdown/renderer.go new file mode 100644 index 000000000..8fab8d2ac --- /dev/null +++ b/middleware/markdown/renderer.go @@ -0,0 +1,93 @@ +package markdown + +import ( + "bytes" +) + +type PlaintextRenderer struct{} + +// Block-level callbacks + +func (r PlaintextRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {} + +func (r PlaintextRenderer) BlockQuote(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) BlockHtml(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {} + +func (r PlaintextRenderer) HRule(out *bytes.Buffer) {} + +func (r PlaintextRenderer) List(out *bytes.Buffer, text func() bool, flags int) {} + +func (r PlaintextRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) Paragraph(out *bytes.Buffer, text func() bool) { + marker := out.Len() + if !text() { + out.Truncate(marker) + } + out.Write([]byte{' '}) +} + +func (r PlaintextRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {} + +func (r PlaintextRenderer) TableRow(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) Footnotes(out *bytes.Buffer, text func() bool) {} + +func (r PlaintextRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {} + +func (r PlaintextRenderer) TitleBlock(out *bytes.Buffer, text []byte) {} + +// Span-level callbacks + +func (r PlaintextRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {} + +func (r PlaintextRenderer) CodeSpan(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +func (r PlaintextRenderer) Emphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +func (r PlaintextRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {} + +func (r PlaintextRenderer) LineBreak(out *bytes.Buffer) {} + +func (r PlaintextRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { + out.Write(content) +} +func (r PlaintextRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {} + +func (r PlaintextRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} +func (r PlaintextRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {} + +// Low-level callbacks + +func (r PlaintextRenderer) Entity(out *bytes.Buffer, entity []byte) { + out.Write(entity) +} + +func (r PlaintextRenderer) NormalText(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +// Header and footer + +func (r PlaintextRenderer) DocumentHeader(out *bytes.Buffer) {} + +func (r PlaintextRenderer) DocumentFooter(out *bytes.Buffer) {} + +func (r PlaintextRenderer) GetFlags() int { return 0 } diff --git a/middleware/markdown/testdata/blog/test.md b/middleware/markdown/testdata/blog/test.md index 7ec766160..3d33ad918 100644 --- a/middleware/markdown/testdata/blog/test.md +++ b/middleware/markdown/testdata/blog/test.md @@ -1,7 +1,6 @@ --- title: Markdown test -variables: - sitename: A Caddy website +sitename: A Caddy website --- ## Welcome on the blog diff --git a/middleware/markdown/testdata/log/test.md b/middleware/markdown/testdata/log/test.md index 7ec766160..3d33ad918 100644 --- a/middleware/markdown/testdata/log/test.md +++ b/middleware/markdown/testdata/log/test.md @@ -1,7 +1,6 @@ --- title: Markdown test -variables: - sitename: A Caddy website +sitename: A Caddy website --- ## Welcome on the blog diff --git a/middleware/proxy/upstream.go b/middleware/proxy/upstream.go index cca5c7728..9a2a01048 100644 --- a/middleware/proxy/upstream.go +++ b/middleware/proxy/upstream.go @@ -109,6 +109,8 @@ func NewStaticUpstreams(c parse.Dispenser) ([]Upstream, error) { return upstreams, c.ArgErr() } upstream.WithoutPathPrefix = c.Val() + default: + return upstreams, c.Errf("unknown property '%s'", c.Val()) } }