Feature: CommonMark support

This adds the markdown.it engine to Discourse. https://github.com/markdown-it/markdown-it As the migration is going to take a while the new engine is default disabled. To enable it you must change the hidden site setting: enable_experimental_markdown_it. This commit is a squash of many other commits, it also includes some improvements to autospec (ability to run plugins), and a dev dependency on the og gem for html normalization.
2024-11-22 12:38:26 +08:00 · 2017-06-08 18:02:30 -04:00 · 2017-06-08 18:02:30 -04:00 · 234694b50f
commit 234694b50f
parent 6048ca2b7d
57 changed files with 11146 additions and 96 deletions
--- a/4
+++ b/4
@ -75,6 +75,10 @@ gem 'discourse_image_optim', require: 'image_optim'
 gem 'multi_json'
 gem 'mustache'
 gem 'nokogiri'
+
+# this may end up deprecating nokogiri
+gem 'oga', require: false
+
 gem 'omniauth'
 gem 'omniauth-openid'
 gem 'openid-redis-store'
--- a/Gemfile.lock
+++ b/Gemfile.lock
@ -42,7 +42,9 @@ GEM
    annotate (2.7.2)
      activerecord (>= 3.2, < 6.0)
      rake (>= 10.4, < 13.0)
+    ansi (1.5.0)
    arel (6.0.4)
+    ast (2.3.0)
    aws-sdk (2.5.3)
      aws-sdk-resources (= 2.5.3)
    aws-sdk-core (2.5.3)
@ -181,6 +183,9 @@ GEM
      multi_json (~> 1.3)
      multi_xml (~> 0.5)
      rack (>= 1.2, < 3)
+    oga (2.10)
+      ast
+      ruby-ll (~> 2.1)
    oj (3.1.0)
    omniauth (1.6.1)
      hashie (>= 3.4.6, < 3.6.0)
@ -311,6 +316,9 @@ GEM
      rspec-support (~> 3.6.0)
    rspec-support (3.6.0)
    rtlit (0.0.5)
+    ruby-ll (2.1.2)
+      ansi
+      ast
    ruby-openid (2.7.0)
    ruby-readability (0.7.0)
      guess_html_encoding (>= 0.0.4)
@ -428,6 +436,7 @@ DEPENDENCIES
  multi_json
  mustache
  nokogiri
+  oga
  oj
  omniauth
  omniauth-facebook
--- a/app/assets/javascripts/discourse/components/d-editor.js.es6
+++ b/app/assets/javascripts/discourse/components/d-editor.js.es6
@ -6,7 +6,7 @@ import { categoryHashtagTriggerRule } from 'discourse/lib/category-hashtags';
 import { TAG_HASHTAG_POSTFIX } from 'discourse/lib/tag-hashtags';
 import { search as searchCategoryTag  } from 'discourse/lib/category-tag-search';
 import { SEPARATOR } from 'discourse/lib/category-hashtags';
-import { cook } from 'discourse/lib/text';
+import { cookAsync } from 'discourse/lib/text';
 import { translations } from 'pretty-text/emoji/data';
 import { emojiSearch, isSkinTonableEmoji } from 'pretty-text/emoji';
 import { emojiUrlFor } from 'discourse/lib/text';
@ -279,14 +279,14 @@ export default Ember.Component.extend({
    const value = this.get('value');
    const markdownOptions = this.get('markdownOptions') || {};

-    markdownOptions.siteSettings = this.siteSettings;
-
-    this.set('preview', cook(value));
-    Ember.run.scheduleOnce('afterRender', () => {
-      if (this._state !== "inDOM") { return; }
-      const $preview = this.$('.d-editor-preview');
-      if ($preview.length === 0) return;
-      this.sendAction('previewUpdated', $preview);
+    cookAsync(value, markdownOptions).then(cooked => {
+      this.set('preview', cooked);
+      Ember.run.scheduleOnce('afterRender', () => {
+        if (this._state !== "inDOM") { return; }
+        const $preview = this.$('.d-editor-preview');
+        if ($preview.length === 0) return;
+        this.sendAction('previewUpdated', $preview);
+      });
    });
  },

--- a/app/assets/javascripts/discourse/lib/text.js.es6
+++ b/app/assets/javascripts/discourse/lib/text.js.es6
@ -2,22 +2,37 @@ import { default as PrettyText, buildOptions } from 'pretty-text/pretty-text';
 import { performEmojiUnescape, buildEmojiUrl } from 'pretty-text/emoji';
 import WhiteLister from 'pretty-text/white-lister';
 import { sanitize as textSanitize } from 'pretty-text/sanitizer';
+import loadScript from 'discourse/lib/load-script';

-function getOpts() {
+function getOpts(opts) {
  const siteSettings = Discourse.__container__.lookup('site-settings:main');

-  return buildOptions({
+  opts = _.merge({
    getURL: Discourse.getURLWithCDN,
    currentUser: Discourse.__container__.lookup('current-user:main'),
    siteSettings
-  });
+  }, opts);
+
+  return buildOptions(opts);
 }

 // Use this to easily create a pretty text instance with proper options
-export function cook(text) {
-  return new Handlebars.SafeString(new PrettyText(getOpts()).cook(text));
+export function cook(text, options) {
+  return new Handlebars.SafeString(new PrettyText(getOpts(options)).cook(text));
 }

+// everything should eventually move to async API and this should be renamed
+// cook
+export function cookAsync(text, options) {
+  if (Discourse.MarkdownItURL) {
+    return loadScript(Discourse.MarkdownItURL)
+      .then(()=>cook(text, options));
+  } else {
+    return Ember.RSVP.Promise.resolve(cook(text));
+  }
+}
+
+
 export function sanitize(text) {
  return textSanitize(text, new WhiteLister(getOpts()));
 }
--- a/app/assets/javascripts/markdown-it-bundle.js
+++ b/app/assets/javascripts/markdown-it-bundle.js
@ -0,0 +1,11 @@
+//= require markdown-it.js
+//= require ./pretty-text/engines/markdown-it/helpers
+//= require ./pretty-text/engines/markdown-it/mentions
+//= require ./pretty-text/engines/markdown-it/quotes
+//= require ./pretty-text/engines/markdown-it/emoji
+//= require ./pretty-text/engines/markdown-it/onebox
+//= require ./pretty-text/engines/markdown-it/bbcode-block
+//= require ./pretty-text/engines/markdown-it/bbcode-inline
+//= require ./pretty-text/engines/markdown-it/code
+//= require ./pretty-text/engines/markdown-it/category-hashtag
+//= require ./pretty-text/engines/markdown-it/censored
--- a/app/assets/javascripts/pretty-text-bundle.js
+++ b/app/assets/javascripts/pretty-text-bundle.js
@ -4,6 +4,7 @@
 //= require ./pretty-text/emoji/data
 //= require ./pretty-text/emoji
 //= require ./pretty-text/engines/discourse-markdown
+//= require ./pretty-text/engines/discourse-markdown-it
 //= require_tree ./pretty-text/engines/discourse-markdown
 //= require xss.min
 //= require better_markdown.js
--- a/app/assets/javascripts/pretty-text/censored-words.js.es6
+++ b/app/assets/javascripts/pretty-text/censored-words.js.es6
@ -2,9 +2,11 @@ function escapeRegexp(text) {
  return text.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
 }

-export function censor(text, censoredWords, censoredPattern) {
-  let patterns = [],
-      originalText = text;
+export function censorFn(censoredWords, censoredPattern, replacementLetter) {
+
+  let patterns = [];
+
+  replacementLetter = replacementLetter || "&#9632;";

  if (censoredWords && censoredWords.length) {
    patterns = censoredWords.split("|").map(t => `(${escapeRegexp(t)})`);
@ -21,19 +23,35 @@ export function censor(text, censoredWords, censoredPattern) {
      censorRegexp = new RegExp("(\\b(?:" + patterns.join("|") + ")\\b)(?![^\\(]*\\))", "ig");

      if (censorRegexp) {
-        let m = censorRegexp.exec(text);

-        while (m && m[0]) {
-          if (m[0].length > originalText.length) { return originalText; } // regex is dangerous
-          const replacement = new Array(m[0].length+1).join('&#9632;');
-          text = text.replace(new RegExp(`(\\b${escapeRegexp(m[0])}\\b)(?![^\\(]*\\))`, "ig"), replacement);
-          m = censorRegexp.exec(text);
-        }
+        return function(text) {
+          let original = text;
+
+          try {
+            let m = censorRegexp.exec(text);
+
+            while (m && m[0]) {
+              if (m[0].length > original.length) { return original; } // regex is dangerous
+              const replacement = new Array(m[0].length+1).join(replacementLetter);
+              text = text.replace(new RegExp(`(\\b${escapeRegexp(m[0])}\\b)(?![^\\(]*\\))`, "ig"), replacement);
+              m = censorRegexp.exec(text);
+            }
+
+            return text;
+          } catch (e) {
+            return original;
+          }
+        };
+
      }
    } catch(e) {
-      return originalText;
+      // fall through
    }
  }

-  return text;
+  return function(t){ return t;};
+}
+
+export function censor(text, censoredWords, censoredPattern, replacementLetter) {
+  return censorFn(censoredWords, censoredPattern, replacementLetter)(text);
 }
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6
@ -0,0 +1,139 @@
+import { default as WhiteLister, whiteListFeature } from 'pretty-text/white-lister';
+import { sanitize } from 'pretty-text/sanitizer';
+
+function deprecate(feature, name){
+  return function() {
+    if (console && console.log) {
+      console.log(feature + ': ' + name + ' is deprecated, please use the new markdown it APIs');
+    }
+  };
+}
+
+function createHelper(featureName, opts, optionCallbacks, pluginCallbacks, getOptions) {
+  let helper = {};
+  helper.markdownIt = true;
+  helper.whiteList = info => whiteListFeature(featureName, info);
+  helper.registerInline = deprecate(featureName,'registerInline');
+  helper.replaceBlock = deprecate(featureName,'replaceBlock');
+  helper.addPreProcessor = deprecate(featureName,'addPreProcessor');
+  helper.inlineReplace = deprecate(featureName,'inlineReplace');
+  helper.postProcessTag = deprecate(featureName,'postProcessTag');
+  helper.inlineRegexp = deprecate(featureName,'inlineRegexp');
+  helper.inlineBetween = deprecate(featureName,'inlineBetween');
+  helper.postProcessText = deprecate(featureName,'postProcessText');
+  helper.onParseNode = deprecate(featureName,'onParseNode');
+  helper.registerBlock = deprecate(featureName,'registerBlock');
+  // hack to allow moving of getOptions
+  helper.getOptions = () => getOptions.f();
+
+  helper.registerOptions = function(callback){
+    optionCallbacks.push([featureName, callback]);
+  };
+
+  helper.registerPlugin = function(callback){
+    pluginCallbacks.push([featureName, callback]);
+  };
+
+  return helper;
+}
+
+// TODO we may just use a proper ruler from markdown it... this is a basic proxy
+class Ruler {
+  constructor() {
+    this.rules = [];
+  }
+
+  getRules() {
+    return this.rules;
+  }
+
+  push(name, rule) {
+    this.rules.push({name, rule});
+  }
+}
+
+// block bb code ruler for parsing of quotes / code / polls
+function setupBlockBBCode(md) {
+  md.block.bbcode_ruler = new Ruler();
+}
+
+export function setup(opts, siteSettings, state) {
+  if (opts.setup) {
+    return;
+  }
+
+  opts.markdownIt = true;
+
+  let optionCallbacks = [];
+  let pluginCallbacks = [];
+
+  // ideally I would like to change the top level API a bit, but in the mean time this will do
+  let getOptions = {
+    f: () => opts
+  };
+
+  const check = /discourse-markdown\/|markdown-it\//;
+  let features = [];
+
+  Object.keys(require._eak_seen).forEach(entry => {
+    if (check.test(entry)) {
+      const module = require(entry);
+      if (module && module.setup) {
+
+        const featureName = entry.split('/').reverse()[0];
+        features.push(featureName);
+        module.setup(createHelper(featureName, opts, optionCallbacks, pluginCallbacks, getOptions));
+      }
+    }
+  });
+
+  optionCallbacks.forEach(([,callback])=>{
+    callback(opts, siteSettings, state);
+  });
+
+  // enable all features by default
+  features.forEach(feature => {
+    if (!opts.features.hasOwnProperty(feature)) {
+      opts.features[feature] = true;
+    }
+  });
+
+  let copy = {};
+  Object.keys(opts).forEach(entry => {
+    copy[entry] = opts[entry];
+    delete opts[entry];
+  });
+
+  opts.discourse = copy;
+  getOptions.f = () => opts.discourse;
+
+  opts.engine = window.markdownit({
+    discourse: opts.discourse,
+    html: true,
+    breaks: opts.discourse.features.newline,
+    xhtmlOut: false,
+    linkify: true,
+    typographer: false
+  });
+
+  setupBlockBBCode(opts.engine);
+
+  pluginCallbacks.forEach(([feature, callback])=>{
+    if (opts.discourse.features[feature]) {
+      opts.engine.use(callback);
+    }
+  });
+
+  // top level markdown it notifier
+  opts.markdownIt = true;
+  opts.setup = true;
+
+  if (!opts.discourse.sanitizer) {
+    opts.sanitizer = opts.discourse.sanitizer = (!!opts.discourse.sanitize) ? sanitize : a=>a;
+  }
+}
+
+export function cook(raw, opts) {
+  const whiteLister = new WhiteLister(opts.discourse);
+  return opts.discourse.sanitizer(opts.engine.render(raw), whiteLister).trim();
+}
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6
@ -385,14 +385,25 @@ export function cook(raw, opts) {
  currentOpts = opts;

  hoisted = {};
-  raw = hoistCodeBlocksAndSpans(raw);

-  preProcessors.forEach(p => raw = p(raw));
+  if (!currentOpts.enableExperimentalMarkdownIt) {
+    raw = hoistCodeBlocksAndSpans(raw);
+    preProcessors.forEach(p => raw = p(raw));
+  }

  const whiteLister = new WhiteLister(opts);

-  const tree = parser.toHTMLTree(raw, 'Discourse');
-  let result = opts.sanitizer(parser.renderJsonML(parseTree(tree, opts)), whiteLister);
+  let result;
+
+  if (currentOpts.enableExperimentalMarkdownIt) {
+    result = opts.sanitizer(
+        require('pretty-text/engines/markdown-it/instance').default(opts).render(raw),
+        whiteLister
+    );
+  } else {
+    const tree = parser.toHTMLTree(raw, 'Discourse');
+    result = opts.sanitizer(parser.renderJsonML(parseTree(tree, opts)), whiteLister);
+  }

  // If we hoisted out anything, put it back
  const keys = Object.keys(hoisted);
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6
@ -21,6 +21,7 @@ const urlReplacerArgs = {
 };

 export function setup(helper) {
+  if (helper.markdownIt) { return; }
  helper.inlineRegexp(_.merge({start: 'http'}, urlReplacerArgs));
  helper.inlineRegexp(_.merge({start: 'www'}, urlReplacerArgs));
 }
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6
@ -102,6 +102,8 @@ export function builders(helper) {

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']);

  const { replaceBBCode, rawBBCode, removeEmptyLines, replaceBBCodeParamsRaw } = builders(helper);
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6
@ -35,6 +35,8 @@ function unhoist(obj,from,to){

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  function replaceMarkdown(match, tag) {
    const hash = guid();

--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6
@ -1,4 +1,7 @@
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
  helper.inlineRegexp({
    start: '#',
    matcher: /^#([\w-:]{1,101})/i,
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6
@ -8,6 +8,9 @@ registerOption((siteSettings, opts) => {
 });

 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
  helper.addPreProcessor(text => {
    const options = helper.getOptions();
    return censor(text, options.censoredWords, options.censoredPattern);
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6
@ -21,6 +21,8 @@ registerOption((siteSettings, opts) => {

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  helper.whiteList({
    custom(tag, name, value) {
      if (tag === 'code' && name === 'class') {
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6
@ -35,6 +35,8 @@ registerOption((siteSettings, opts, state) => {

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  helper.whiteList('img.emoji');

  function imageFor(code) {
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6
@ -21,6 +21,8 @@ function splitAtLast(tag, block, next, first) {

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  // If a row begins with HTML tags, don't parse it.
  helper.registerBlock('html', function(block, next) {
    let split, pos;
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6
@ -5,6 +5,8 @@
 **/
 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  // We have to prune @mentions that are within links.
  helper.onParseNode(event => {
    const node = event.node,
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6
@ -2,6 +2,9 @@
 // in the tree, replace any new lines with `br`s.

 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
  helper.postProcessText((text, event) => {
    const { options, insideCounts } = event;
    if (options.traditionalMarkdownLinebreaks || (insideCounts.pre > 0)) { return; }
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6
@ -25,6 +25,9 @@ function isOnOneLine(link, parent) {

 //  We only onebox stuff that is on its own line.
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
  helper.onParseNode(event => {
    const node = event.node,
    path = event.path;
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6
@ -9,6 +9,9 @@ registerOption((siteSettings, opts) => {


 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
  register(helper, 'quote', {noWrap: true, singlePara: true}, (contents, bbParams, options) => {

    const params = {'class': 'quote'};
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6
@ -18,6 +18,8 @@ registerOption((siteSettings, opts) => {

 export function setup(helper) {

+  if (helper.markdownIt) { return; }
+
  helper.whiteList(['table', 'table.md-table', 'tbody', 'thead', 'tr', 'th', 'td']);

  helper.replaceBlock({
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6
@ -0,0 +1,200 @@
+// parse a tag [test a=1 b=2] to a data structure
+// {tag: "test", attrs={a: "1", b: "2"}
+export function parseBBCodeTag(src, start, max) {
+
+  let i;
+  let tag;
+  let attrs = {};
+  let closed = false;
+  let length = 0;
+  let closingTag = false;
+
+  // closing tag
+  if (src.charCodeAt(start+1) === 47) {
+    closingTag = true;
+    start += 1;
+  }
+
+  for (i=start+1;i<max;i++) {
+    let letter = src[i];
+    if (!( (letter >= 'a' && letter <= 'z') ||
+         (letter >= 'A' && letter <= 'Z'))) {
+      break;
+    }
+  }
+
+  tag = src.slice(start+1, i);
+
+  if (!tag) {
+    return;
+  }
+
+  if (closingTag) {
+    if (src[i] === ']') {
+      return {tag, length: tag.length+3, closing: true};
+    }
+    return;
+  }
+
+  for (;i<max;i++) {
+    let letter = src[i];
+
+    if (letter === ']') {
+      closed = true;
+      break;
+    }
+  }
+
+  if (closed) {
+    length = i;
+
+    let raw = src.slice(start+tag.length+1, i);
+
+    // trivial parser that is going to have to be rewritten at some point
+    if (raw) {
+
+      // reading a key 0, reading a val = 1
+      let readingKey = true;
+      let startSplit = 0;
+      let key;
+
+      for(i=0; i<raw.length; i++) {
+        if (raw[i] === '=' || i === (raw.length-1)) {
+          // one more offset to allow room to capture last
+          if (raw[i] !== '=' || i === (raw.length-1)) {
+            i+=1;
+          }
+
+          let cur = raw.slice(startSplit, i).trim();
+          if (readingKey) {
+            key =  cur || '_default';
+          } else {
+            let val = raw.slice(startSplit, i).trim();
+            if (val && val.length > 0) {
+              val = val.replace(/^["'](.*)["']$/, '$1');
+              attrs[key] = val;
+            }
+          }
+          readingKey = !readingKey;
+          startSplit = i+1;
+        }
+      }
+    }
+
+    tag = tag.toLowerCase();
+
+    return {tag, attrs, length};
+  }
+}
+
+function applyBBCode(state, startLine, endLine, silent, md) {
+
+  var i, pos, nextLine,
+      old_parent, old_line_max, rule,
+      auto_closed = false,
+      start = state.bMarks[startLine] + state.tShift[startLine],
+      initial = start,
+      max = state.eMarks[startLine];
+
+
+  // [ === 91
+  if (91 !== state.src.charCodeAt(start)) { return false; }
+
+  let info = parseBBCodeTag(state.src, start, max);
+
+  if (!info) {
+    return false;
+  }
+
+  let rules = md.block.bbcode_ruler.getRules();
+
+  for(i=0;i<rules.length;i++) {
+    let r = rules[i].rule;
+
+    if (r.tag === info.tag) {
+      rule = r;
+      break;
+    }
+  }
+
+  if (!rule) { return false; }
+
+  // Since start is found, we can report success here in validation mode
+  if (silent) { return true; }
+
+  // Search for the end of the block
+  nextLine = startLine;
+
+  for (;;) {
+    nextLine++;
+    if (nextLine >= endLine) {
+      // unclosed block should be autoclosed by end of document.
+      // also block seems to be autoclosed by end of parent
+      break;
+    }
+
+    start = state.bMarks[nextLine] + state.tShift[nextLine];
+    max = state.eMarks[nextLine];
+
+    if (start < max && state.sCount[nextLine] < state.blkIndent) {
+      // non-empty line with negative indent should stop the list:
+      // - ```
+      //  test
+      break;
+    }
+
+
+    // bbcode close [ === 91
+    if (91 !== state.src.charCodeAt(start)) { continue; }
+
+    if (state.sCount[nextLine] - state.blkIndent >= 4) {
+      // closing fence should be indented less than 4 spaces
+      continue;
+    }
+
+
+    if (state.src.slice(start+2, max-1) !== rule.tag) { continue; }
+
+    if (pos < max) { continue; }
+
+    // found!
+    auto_closed = true;
+    break;
+  }
+
+  old_parent = state.parentType;
+  old_line_max = state.lineMax;
+
+  // this will prevent lazy continuations from ever going past our end marker
+  state.lineMax = nextLine;
+
+  rule.before.call(this, state, info.attrs, md, state.src.slice(initial, initial + info.length + 1));
+
+  let lastToken = state.tokens[state.tokens.length-1];
+  lastToken.map    = [ startLine, nextLine ];
+
+  state.md.block.tokenize(state, startLine + 1, nextLine);
+
+  rule.after.call(this, state, lastToken, md);
+
+  lastToken = state.tokens[state.tokens.length-1];
+
+  state.parentType = old_parent;
+
+  state.lineMax = old_line_max;
+  state.line = nextLine + (auto_closed ? 1 : 0);
+
+  return true;
+}
+
+
+
+export function setup(helper) {
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.block.ruler.after('fence', 'bbcode', (state, startLine, endLine, silent)=> {
+      return applyBBCode(state, startLine, endLine, silent, md);
+    });
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6
@ -0,0 +1,109 @@
+import { parseBBCodeTag } from 'pretty-text/engines/markdown-it/bbcode-block';
+
+const rules = {
+  'b': {tag: 'span', 'class': 'bbcode-b'},
+  'i': {tag: 'span', 'class': 'bbcode-i'},
+  'u': {tag: 'span', 'class': 'bbcode-u'},
+  's': {tag: 'span', 'class': 'bbcode-s'}
+};
+
+function tokanizeBBCode(state, silent) {
+
+  let pos = state.pos;
+
+  // 91 = [
+  if (silent || state.src.charCodeAt(pos) !== 91) {
+    return false;
+  }
+
+  const tagInfo = parseBBCodeTag(state.src, pos, state.posMax);
+
+  if (!tagInfo) {
+    return false;
+  }
+
+  const rule = rules[tagInfo.tag];
+  if (!rule) {
+    return false;
+  }
+
+  tagInfo.rule = rule;
+
+  let token = state.push('text', '' , 0);
+  token.content = state.src.slice(pos, pos+tagInfo.length);
+
+  state.delimiters.push({
+    bbInfo: tagInfo,
+    marker: 'bb' + tagInfo.tag,
+    open: !tagInfo.closing,
+    close: !!tagInfo.closing,
+    token: state.tokens.length - 1,
+    level: state.level,
+    end: -1,
+    jump: 0
+  });
+
+  state.pos = pos + tagInfo.length;
+  return true;
+}
+
+function processBBCode(state, silent) {
+  let i,
+      startDelim,
+      endDelim,
+      token,
+      tagInfo,
+      delimiters = state.delimiters,
+      max = delimiters.length;
+
+  if (silent) {
+    return;
+  }
+
+  for (i=0; i<max-1; i++) {
+    startDelim = delimiters[i];
+    tagInfo = startDelim.bbInfo;
+
+    if (!tagInfo) {
+      continue;
+    }
+
+    if (startDelim.end === -1) {
+      continue;
+    }
+
+    endDelim = delimiters[startDelim.end];
+
+    token = state.tokens[startDelim.token];
+    token.type = 'bbcode_' + tagInfo.tag + '_open';
+    token.attrs = [['class', tagInfo.rule['class']]];
+    token.tag = tagInfo.rule.tag;
+    token.nesting = 1;
+    token.markup = token.content;
+    token.content = '';
+
+    token = state.tokens[endDelim.token];
+    token.type = 'bbcode_' + tagInfo.tag + '_close';
+    token.tag = tagInfo.rule.tag;
+    token.nesting = -1;
+    token.markup = token.content;
+    token.content = '';
+  }
+  return false;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']);
+
+  helper.registerOptions(opts => {
+    opts.features['bbcode-inline'] = true;
+  });
+
+  helper.registerPlugin(md => {
+    md.inline.ruler.push('bbcode-inline', tokanizeBBCode);
+    md.inline.ruler2.before('text_collapse', 'bbcode-inline', processBBCode);
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6
@ -0,0 +1,58 @@
+import { inlineRegexRule } from 'pretty-text/engines/markdown-it/helpers';
+
+function emitter(matches, state) {
+    const options = state.md.options.discourse;
+    const [hashtag, slug] = matches;
+    const categoryHashtagLookup = options.categoryHashtagLookup;
+    const result = categoryHashtagLookup && categoryHashtagLookup(slug);
+
+    let token;
+
+    if (result) {
+      token = state.push('link_open', 'a', 1);
+      token.attrs = [['class', 'hashtag'], ['href', result[0]]];
+      token.block = false;
+
+      token = state.push('text', '', 0);
+      token.content = '#';
+
+      token = state.push('span_open', 'span', 1);
+      token.block = false;
+
+      token = state.push('text', '', 0);
+      token.content = result[1];
+
+      state.push('span_close', 'span', -1);
+
+      state.push('link_close', 'a', -1);
+    } else {
+
+      token = state.push('span_open', 'span', 1);
+      token.attrs = [['class', 'hashtag']];
+
+      token = state.push('text', '', 0);
+      token.content = hashtag;
+
+      token = state.push('span_close', 'span', -1);
+    }
+
+    return true;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md=>{
+
+    const rule = inlineRegexRule(md, {
+      start: '#',
+      matcher: /^#([\w-:]{1,101})/i,
+      skipInLink: true,
+      maxLength: 102,
+      emitter: emitter
+    });
+
+    md.inline.ruler.push('category-hashtag', rule);
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6
@ -0,0 +1,44 @@
+import { censorFn } from 'pretty-text/censored-words';
+
+function recurse(tokens, apply) {
+  let i;
+  for(i=0;i<tokens.length;i++) {
+    apply(tokens[i]);
+    if (tokens[i].children) {
+      recurse(tokens[i].children, apply);
+    }
+  }
+}
+
+function censorTree(state, censor) {
+  if (!state.tokens) {
+    return;
+  }
+
+  recurse(state.tokens, token => {
+    if (token.content) {
+      token.content = censor(token.content);
+    }
+  });
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.censoredWords = siteSettings.censored_words;
+    opts.censoredPattern = siteSettings.censored_pattern;
+  });
+
+  helper.registerPlugin(md => {
+    const words = md.options.discourse.censoredWords;
+    const patterns = md.options.discourse.censoredPattern;
+
+    if ((words && words.length > 0) || (patterns && patterns.length > 0)) {
+      const replacement = String.fromCharCode(9632);
+      const censor = censorFn(words, patterns, replacement);
+      md.core.ruler.push('censored', state => censorTree(state, censor));
+    }
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6
@ -0,0 +1,51 @@
+// we need a custom renderer for code blocks cause we have a slightly non compliant
+// format with special handling for text and so on
+
+const TEXT_CODE_CLASSES = ["text", "pre", "plain"];
+
+
+function render(tokens, idx, options, env, slf, md) {
+  let token = tokens[idx],
+      info = token.info ? md.utils.unescapeAll(token.info) : '',
+      langName = md.options.discourse.defaultCodeLang,
+      className,
+      escapedContent = md.utils.escapeHtml(token.content);
+
+  if (info) {
+    // strip off any additional languages
+    info = info.split(/\s+/g)[0];
+  }
+
+  const acceptableCodeClasses = md.options.discourse.acceptableCodeClasses;
+  if (acceptableCodeClasses && info && acceptableCodeClasses.indexOf(info) !== -1) {
+    langName = info;
+  }
+
+  className = TEXT_CODE_CLASSES.indexOf(langName) !== -1 ? 'lang-nohighlight' : 'lang-' + langName;
+
+  return `<pre><code class='${className}'>${escapedContent}</code></pre>\n`;
+}
+
+export function setup(helper) {
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.defaultCodeLang = siteSettings.default_code_lang;
+    opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']);
+  });
+
+  helper.whiteList({
+    custom(tag, name, value) {
+      if (tag === 'code' && name === 'class') {
+        const m = /^lang\-(.+)$/.exec(value);
+        if (m) {
+          return helper.getOptions().acceptableCodeClasses.indexOf(m[1]) !== -1;
+        }
+      }
+    }
+  });
+
+  helper.registerPlugin(md=>{
+    md.renderer.rules.fence = (tokens,idx,options,env,slf)=>render(tokens,idx,options,env,slf,md);
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6
@ -0,0 +1,241 @@
+import { buildEmojiUrl, isCustomEmoji } from 'pretty-text/emoji';
+import { translations } from 'pretty-text/emoji/data';
+import { textReplace } from 'pretty-text/engines/markdown-it/helpers';
+
+const MAX_NAME_LENGTH = 60;
+
+let translationTree = null;
+
+// This allows us to efficiently search for aliases
+// We build a data structure that allows us to quickly
+// search through our N next chars to see if any match
+// one of our alias emojis.
+//
+function buildTranslationTree() {
+  let tree = [];
+  let lastNode;
+
+  Object.keys(translations).forEach(function(key){
+    let i;
+    let node = tree;
+
+    for(i=0;i<key.length;i++) {
+      let code = key.charCodeAt(i);
+      let j;
+
+      let found = false;
+
+      for (j=0;j<node.length;j++){
+        if (node[j][0] === code) {
+          node = node[j][1];
+          found = true;
+          break;
+        }
+      }
+
+      if (!found) {
+        // token, children, value
+        let tmp = [code, []];
+        node.push(tmp);
+        lastNode = tmp;
+        node = tmp[1];
+      }
+    }
+
+    lastNode[1] = translations[key];
+  });
+
+  return tree;
+}
+
+
+function imageFor(code, opts) {
+  code = code.toLowerCase();
+  const url = buildEmojiUrl(code, opts);
+  if (url) {
+    const title = `:${code}:`;
+    const classes = isCustomEmoji(code, opts) ? "emoji emoji-custom" : "emoji";
+    return {url, title, classes};
+  }
+}
+
+function getEmojiName(content, pos, state) {
+
+  if (content.charCodeAt(pos) !== 58) {
+    return;
+  }
+
+  if (pos > 0) {
+    let prev = content.charCodeAt(pos-1);
+    if (!state.md.utils.isSpace(prev) && !state.md.utils.isPunctChar(String.fromCharCode(prev))) {
+      return;
+    }
+  }
+
+  pos++;
+  if (content.charCodeAt(pos) === 58) {
+    return;
+  }
+
+  let length = 0;
+  while(length < MAX_NAME_LENGTH) {
+    length++;
+
+    if (content.charCodeAt(pos+length) === 58) {
+      // check for t2-t6
+      if (content.substr(pos+length+1, 3).match(/t[2-6]:/)) {
+        length += 3;
+      }
+      break;
+    }
+
+    if (pos+length > content.length) {
+      return;
+    }
+  }
+
+  if (length === MAX_NAME_LENGTH) {
+    return;
+  }
+
+  return content.substr(pos, length);
+}
+
+// straight forward :smile: to emoji image
+function getEmojiTokenByName(name, state) {
+
+  let info;
+  if (info = imageFor(name, state.md.options.discourse)) {
+    let token = new state.Token('emoji', 'img', 0);
+    token.attrs = [['src', info.url],
+                   ['title', info.title],
+                   ['class', info.classes],
+                   ['alt', info.title]];
+
+    return token;
+  }
+}
+
+function getEmojiTokenByTranslation(content, pos, state) {
+
+  translationTree = translationTree || buildTranslationTree();
+
+  let currentTree = translationTree;
+
+  let i;
+  let search = true;
+  let found = false;
+  let start = pos;
+
+  while(search) {
+
+    search = false;
+    let code = content.charCodeAt(pos);
+
+    for (i=0;i<currentTree.length;i++) {
+      if(currentTree[i][0] === code) {
+        currentTree = currentTree[i][1];
+        pos++;
+        search = true;
+        if (typeof currentTree === "string") {
+          found = currentTree;
+        }
+        break;
+      }
+    }
+  }
+
+  if (!found) {
+    return;
+  }
+
+  // quick boundary check
+  if (start > 0) {
+    let leading = content.charAt(start-1);
+    if (!state.md.utils.isSpace(leading.charCodeAt(0)) && !state.md.utils.isPunctChar(leading)) {
+      return;
+    }
+  }
+
+  // check trailing for punct or space
+  if (pos < content.length) {
+    let trailing = content.charCodeAt(pos);
+    if (!state.md.utils.isSpace(trailing)){
+      return;
+    }
+  }
+
+  let token = getEmojiTokenByName(found, state);
+  if (token) {
+    return { pos, token };
+  }
+}
+
+function applyEmoji(content, state) {
+  let i;
+  let result = null;
+  let contentToken = null;
+
+  let start = 0;
+
+  let endToken = content.length;
+
+  for (i=0; i<content.length-1; i++) {
+    let offset = 0;
+    let emojiName = getEmojiName(content,i,state);
+    let token = null;
+
+    if (emojiName) {
+      token = getEmojiTokenByName(emojiName, state);
+      if (token) {
+        offset = emojiName.length+2;
+      }
+    }
+
+    if (!token) {
+      // handle aliases (note: we can't do this in inline cause ; is not a split point)
+      //
+      let info = getEmojiTokenByTranslation(content, i, state);
+
+      if (info) {
+        offset = info.pos - i;
+        token = info.token;
+      }
+    }
+
+    if (token) {
+      result = result || [];
+      if (i-start>0) {
+        contentToken = new state.Token('text', '', 0);
+        contentToken.content = content.slice(start,i);
+        result.push(contentToken);
+      }
+
+      result.push(token);
+      endToken = start = i + offset;
+    }
+  }
+
+  if (endToken < content.length) {
+    contentToken = new state.Token('text', '', 0);
+    contentToken.content = content.slice(endToken);
+    result.push(contentToken);
+  }
+
+  return result;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings, state)=>{
+    opts.features.emoji = !!siteSettings.enable_emoji;
+    opts.emojiSet = siteSettings.emoji_set || "";
+    opts.customEmoji = state.customEmoji;
+  });
+
+  helper.registerPlugin((md)=>{
+    md.core.ruler.push('emoji', state => textReplace(state, applyEmoji));
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6
@ -0,0 +1,106 @@
+// since the markdown.it interface is a bit on the verbose side
+// we can keep some general patterns here
+
+
+export default null;
+
+// creates a rule suitable for inline parsing and replacement
+//
+// example:
+// const rule = inlineRegexRule(md, {
+//   start: '#',
+//   matcher: /^#([\w-:]{1,101})/i,
+//   emitter: emitter
+// });
+export function inlineRegexRule(md, options) {
+
+  const start = options.start.charCodeAt(0);
+  const maxLength = (options.maxLength || 500) + 1;
+
+  return function(state) {
+    const pos = state.pos;
+
+    if (state.src.charCodeAt(pos) !== start) {
+      return false;
+    }
+
+    // test prev
+    if (pos > 0) {
+      let prev = state.src.charCodeAt(pos-1);
+      if (!md.utils.isSpace(prev) && !md.utils.isPunctChar(String.fromCharCode(prev))) {
+        return false;
+      }
+    }
+
+    // skip if in a link
+    if (options.skipInLink && state.tokens) {
+      let last = state.tokens[state.tokens.length-1];
+      if (last) {
+        if (last.type === 'link_open') {
+          return false;
+        }
+        if (last.type === 'html_inline' && last.content.substr(0,2) === "<a") {
+          return false;
+        }
+      }
+    }
+
+
+    const substr = state.src.slice(pos, Math.min(pos + maxLength,state.posMax));
+
+    const matches = options.matcher.exec(substr);
+    if (!matches) {
+      return false;
+    }
+
+    // got to test trailing boundary
+    const finalPos = pos+matches[0].length;
+    if (finalPos < state.posMax) {
+      const trailing = state.src.charCodeAt(finalPos);
+      if (!md.utils.isSpace(trailing) && !md.utils.isPunctChar(String.fromCharCode(trailing))) {
+        return false;
+      }
+    }
+
+    if (options.emitter(matches, state)) {
+      state.pos = Math.min(state.posMax, finalPos);
+      return true;
+    }
+
+    return false;
+
+  };
+}
+
+// based off https://github.com/markdown-it/markdown-it-emoji/blob/master/dist/markdown-it-emoji.js
+//
+export function textReplace(state, callback) {
+  var i, j, l, tokens, token,
+        blockTokens = state.tokens,
+        autolinkLevel = 0;
+
+  for (j = 0, l = blockTokens.length; j < l; j++) {
+    if (blockTokens[j].type !== 'inline') { continue; }
+    tokens = blockTokens[j].children;
+
+    // We scan from the end, to keep position when new tags added.
+    // Use reversed logic in links start/end match
+    for (i = tokens.length - 1; i >= 0; i--) {
+      token = tokens[i];
+
+      if (token.type === 'link_open' || token.type === 'link_close') {
+        if (token.info === 'auto') { autolinkLevel -= token.nesting; }
+      }
+
+      if (token.type === 'text' && autolinkLevel === 0) {
+        let split;
+        if(split = callback(token.content, state)) {
+          // replace current node
+          blockTokens[j].children = tokens = state.md.utils.arrayReplaceAt(
+            tokens, i, split
+          );
+        }
+      }
+    }
+  }
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6
@ -0,0 +1,88 @@
+const regex = /^(\w[\w.-]{0,59})\b/i;
+
+function applyMentions(state, silent, isSpace, isPunctChar, mentionLookup, getURL) {
+
+  let pos = state.pos;
+
+  // 64 = @
+  if (silent || state.src.charCodeAt(pos) !== 64) {
+    return false;
+  }
+
+  if (pos > 0) {
+    let prev = state.src.charCodeAt(pos-1);
+    if (!isSpace(prev) && !isPunctChar(String.fromCharCode(prev))) {
+      return false;
+    }
+  }
+
+  // skip if in a link
+  if (state.tokens) {
+    let last = state.tokens[state.tokens.length-1];
+    if (last) {
+      if (last.type === 'link_open') {
+        return false;
+      }
+      if (last.type === 'html_inline' && last.content.substr(0,2) === "<a") {
+        return false;
+      }
+    }
+  }
+
+  let maxMention = state.src.substr(pos+1, 60);
+
+  let matches = maxMention.match(regex);
+
+  if (!matches) {
+    return false;
+  }
+
+  let username = matches[1];
+
+  let type = mentionLookup && mentionLookup(username);
+
+  let tag = 'a';
+  let className = 'mention';
+  let href = null;
+
+  if (type === 'user') {
+    href = getURL('/u/') + username.toLowerCase();
+  } else if (type === 'group') {
+    href = getURL('/groups/') + username;
+    className = 'mention-group';
+  } else {
+    tag = 'span';
+  }
+
+  let token = state.push('mention_open', tag, 1);
+  token.attrs = [['class', className]];
+  if (href) {
+    token.attrs.push(['href', href]);
+  }
+
+  token = state.push('text', '', 0);
+  token.content = '@'+username;
+
+  state.push('mention_close', tag, -1);
+
+  state.pos = pos + username.length + 1;
+
+  return true;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.inline.ruler.push('mentions', (state,silent)=> applyMentions(
+          state,
+          silent,
+          md.utils.isSpace,
+          md.utils.isPunctChar,
+          md.options.discourse.mentionLookup,
+          md.options.discourse.getURL
+    ));
+  });
+}
+
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6
@ -0,0 +1,66 @@
+function applyOnebox(state, silent) {
+  if (silent || !state.tokens || state.tokens.length < 3) {
+    return;
+  }
+
+  let i;
+  for(i=1;i<state.tokens.length;i++) {
+    let token = state.tokens[i];
+
+    let prev = state.tokens[i-1];
+    let prevAccepted =  prev.type === "paragraph_open" && prev.level === 0;
+
+    if (token.type === "inline" && prevAccepted) {
+      let j;
+      for(j=0;j<token.children.length;j++){
+        let child = token.children[j];
+
+        if (child.type === "link_open") {
+
+          // look behind for soft or hard break
+          if (j > 0 && token.children[j-1].tag !== 'br') {
+            continue;
+          }
+
+          // look ahead for soft or hard break
+          let text = token.children[j+1];
+          let close = token.children[j+2];
+          let lookahead = token.children[j+3];
+
+          if (lookahead && lookahead.tag !== 'br') {
+            continue;
+          }
+
+          // check attrs only include a href
+          let attrs = child["attrs"];
+
+          if (!attrs || attrs.length !== 1 || attrs[0][0] !== "href") {
+            continue;
+          }
+
+          // check text matches href
+          if (text.type !== "text" || attrs[0][1] !== text.content) {
+            continue;
+          }
+
+          if (!close || close.type !== "link_close") {
+            continue;
+          }
+
+          // decorate...
+          attrs.push(["class", "onebox"]);
+        }
+      }
+    }
+  }
+}
+
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.core.ruler.after('linkify', 'onebox', applyOnebox);
+  });
+}
--- a/app/assets/javascripts/pretty-text/engines/markdown-it/quotes.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/markdown-it/quotes.js.es6
@ -0,0 +1,134 @@
+import { performEmojiUnescape } from 'pretty-text/emoji';
+
+const rule = {
+  tag: 'quote',
+
+  before: function(state, attrs, md) {
+
+    let options = md.options.discourse;
+
+    let quoteInfo = attrs['_default'];
+    let username, postNumber, topicId, avatarImg, full;
+
+    if (quoteInfo) {
+      let split = quoteInfo.split(/\,\s*/);
+      username = split[0];
+
+      let i;
+      for(i=1;i<split.length;i++) {
+        if (split[i].indexOf("post:") === 0) {
+          postNumber = parseInt(split[i].substr(5),10);
+          continue;
+        }
+
+        if (split[i].indexOf("topic:") === 0) {
+          topicId = parseInt(split[i].substr(6),10);
+          continue;
+        }
+
+        if (split[i].indexOf(/full:\s*true/) === 0) {
+          full = true;
+          continue;
+        }
+      }
+    }
+
+
+    let token    = state.push('bbcode_open', 'aside', 1);
+    token.attrs  = [['class', 'quote']];
+
+    if (postNumber) {
+      token.attrs.push(['data-post', postNumber]);
+    }
+
+    if (topicId) {
+      token.attrs.push(['data-topic', topicId]);
+    }
+
+    if (full) {
+      token.attrs.push(['data-full', 'true']);
+    }
+
+    if (options.lookupAvatarByPostNumber) {
+      // client-side, we can retrieve the avatar from the post
+      avatarImg = options.lookupAvatarByPostNumber(postNumber, topicId);
+    } else if (options.lookupAvatar) {
+      // server-side, we need to lookup the avatar from the username
+      avatarImg = options.lookupAvatar(username);
+    }
+
+    if (username) {
+      let offTopicQuote = options.topicId &&
+                          postNumber &&
+                          options.getTopicInfo &&
+                          topicId !== options.topicId;
+
+      // on topic quote
+      token = state.push('quote_header_open', 'div', 1);
+      token.attrs = [['class', 'title']];
+
+      token = state.push('quote_controls_open', 'div', 1);
+      token.attrs = [['class', 'quote-controls']];
+
+      token = state.push('quote_controls_close', 'div', -1);
+
+      if (avatarImg) {
+        token = state.push('html_inline', '', 0);
+        token.content = avatarImg;
+      }
+
+      if (offTopicQuote) {
+        const topicInfo = options.getTopicInfo(topicId);
+        if (topicInfo) {
+          var href = topicInfo.href;
+          if (postNumber > 0) { href += "/" + postNumber; }
+
+          let title = topicInfo.title;
+
+
+          if (options.enableEmoji) {
+            title = performEmojiUnescape(topicInfo.title, {
+              getURL: options.getURL, emojiSet: options.emojiSet
+            });
+          }
+
+          token = state.push('link_open', 'a', 1);
+          token.attrs = [[ 'href', href ]];
+          token.block = false;
+
+          token = state.push('html_inline', '', 0);
+          token.content = title;
+
+          token = state.push('link_close', 'a', -1);
+          token.block = false;
+        }
+      } else {
+        token = state.push('text', '', 0);
+        token.content = ` ${username}:`;
+      }
+
+      token = state.push('quote_header_close', 'div', -1);
+    }
+
+    token        = state.push('bbcode_open', 'blockquote', 1);
+  },
+
+  after: function(state) {
+    state.push('bbcode_close', 'blockquote', -1);
+    state.push('bbcode_close', 'aside', -1);
+  }
+};
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.enableEmoji = siteSettings.enable_emoji;
+    opts.emojiSet = siteSettings.emoji_set;
+  });
+
+  helper.registerPlugin(md=>{
+     md.block.bbcode_ruler.push('quotes', rule);
+  });
+}
--- a/app/assets/javascripts/pretty-text/pretty-text.js.es6
+++ b/app/assets/javascripts/pretty-text/pretty-text.js.es6
@ -1,4 +1,5 @@
 import { cook, setup } from 'pretty-text/engines/discourse-markdown';
+import { cook as cookIt, setup as setupIt } from 'pretty-text/engines/discourse-markdown-it';
 import { sanitize } from 'pretty-text/sanitizer';
 import WhiteLister from 'pretty-text/white-lister';

@ -10,8 +11,6 @@ export function registerOption(fn) {
 }

 export function buildOptions(state) {
-  setup();
-
  const {
    siteSettings,
    getURL,
@ -21,9 +20,14 @@ export function buildOptions(state) {
    categoryHashtagLookup,
    userId,
    getCurrentUser,
-    currentUser
+    currentUser,
+    lookupAvatarByPostNumber
  } = state;

+  if (!siteSettings.enable_experimental_markdown_it) {
+    setup();
+  }
+
  const features = {
    'bold-italics': true,
    'auto-link': true,
@ -33,7 +37,7 @@ export function buildOptions(state) {
    'html': true,
    'category-hashtag': true,
    'onebox': true,
-    'newline': true
+    'newline': !siteSettings.traditional_markdown_linebreaks
  };

  const options = {
@ -47,11 +51,18 @@ export function buildOptions(state) {
    userId,
    getCurrentUser,
    currentUser,
+    lookupAvatarByPostNumber,
    mentionLookup: state.mentionLookup,
-    allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null
+    allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null,
+    markdownIt: siteSettings.enable_experimental_markdown_it
  };

-  _registerFns.forEach(fn => fn(siteSettings, options, state));
+  if (siteSettings.enable_experimental_markdown_it) {
+    setupIt(options, siteSettings, state);
+  } else {
+    // TODO deprecate this
+    _registerFns.forEach(fn => fn(siteSettings, options, state));
+  }

  return options;
 }
@ -61,13 +72,22 @@ export default class {
    this.opts = opts || {};
    this.opts.features = this.opts.features || {};
    this.opts.sanitizer = (!!this.opts.sanitize) ? (this.opts.sanitizer || sanitize) : identity;
-    setup();
+    // We used to do a failsafe call to setup here
+    // under new engine we always expect setup to be called by buildOptions.
+    // setup();
  }

  cook(raw) {
    if (!raw || raw.length === 0) { return ""; }

-    const result = cook(raw, this.opts);
+    let result;
+
+    if (this.opts.markdownIt) {
+      result = cookIt(raw, this.opts);
+    } else {
+      result = cook(raw, this.opts);
+    }
+
    return result ? result : "";
  }

--- a/app/assets/javascripts/pretty-text/white-lister.js.es6
+++ b/app/assets/javascripts/pretty-text/white-lister.js.es6
@ -155,6 +155,7 @@ whiteListFeature('default', [
  'kbd',
  'li',
  'ol',
+  'ol[start]',
  'p',
  'pre',
  's',
--- a/app/views/common/_discourse_javascript.html.erb
+++ b/app/views/common/_discourse_javascript.html.erb
@ -17,12 +17,6 @@
    <% end %>
    window.onerror(e && e.message, null,null,null,e);
  });
-
-<% if Rails.env.development? || Rails.env.test? %>
-  //Ember.ENV.RAISE_ON_DEPRECATION = true
-  //Ember.LOG_STACKTRACE_ON_DEPRECATION = true
-<% end %>
-
 </script>

 <script>
@ -48,19 +42,22 @@
    Discourse.Environment = '<%= Rails.env %>';
    Discourse.SiteSettings = ps.get('siteSettings');
    Discourse.LetterAvatarVersion = '<%= LetterAvatar.version %>';
+    <%- if SiteSetting.enable_experimental_markdown_it %>
+    Discourse.MarkdownItURL = '<%= asset_url('markdown-it-bundle.js') %>';
+    <%- end %>
    I18n.defaultLocale = '<%= SiteSetting.default_locale %>';
    Discourse.start();
    Discourse.set('assetVersion','<%= Discourse.assets_digest %>');
    Discourse.Session.currentProp("disableCustomCSS", <%= loading_admin? %>);
    <%- if params["safe_mode"] %>
-        Discourse.Session.currentProp("safe_mode", <%= normalized_safe_mode.inspect.html_safe %>);
+    Discourse.Session.currentProp("safe_mode", <%= normalized_safe_mode.inspect.html_safe %>);
    <%- end %>
    Discourse.HighlightJSPath = <%= HighlightJs.path.inspect.html_safe %>;
    <%- if SiteSetting.enable_s3_uploads %>
      <%- if SiteSetting.s3_cdn_url.present? %>
-        Discourse.S3CDN = '<%= SiteSetting.s3_cdn_url %>';
+    Discourse.S3CDN = '<%= SiteSetting.s3_cdn_url %>';
      <%- end %>
-      Discourse.S3BaseUrl = '<%= Discourse.store.absolute_base_url %>';
+    Discourse.S3BaseUrl = '<%= Discourse.store.absolute_base_url %>';
    <%- end %>
  })();
 </script>
--- a/bin/notify_file_change
+++ b/bin/notify_file_change
@ -19,7 +19,7 @@ fi
 # 3. Add the following to your .vimrc
 #
 # function s:notify_file_change()
-#   let root = RailsRoot()
+#   let root = rails#app().path()
 #   let notify = root . "/bin/notify_file_change"
 #   if executable(notify)
 #     if executable('socat')
--- a/config/application.rb
+++ b/config/application.rb
@ -83,6 +83,7 @@ module Discourse
                                 browser-update.js break_string.js ember_jquery.js
                                 pretty-text-bundle.js wizard-application.js
                                 wizard-vendor.js plugin.js plugin-third-party.js
+                                 markdown-it-bundle.js
                                 }

    # Precompile all available locales
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@ -490,6 +490,10 @@ posting:
  delete_removed_posts_after:
    client: true
    default: 24
+  enable_experimental_markdown_it:
+    hidden: true
+    client: true
+    default: false
  traditional_markdown_linebreaks:
    client: true
    default: false
--- a/lib/autospec/manager.rb
+++ b/lib/autospec/manager.rb
@ -295,8 +295,10 @@ class Autospec::Manager
        if @queue.first && @queue.first[0] == "focus"
          focus = @queue.shift
          @queue.unshift([file, spec, runner])
-          if focus[1].include?(spec) || file != spec
-            @queue.unshift(focus)
+          unless spec.include?(":") && focus[1].include?(spec.split(":")[0])
+            if focus[1].include?(spec) || file != spec
+              @queue.unshift(focus)
+            end
          end
        else
          @queue.unshift([file, spec, runner])
--- a/lib/autospec/qunit_runner.rb
+++ b/lib/autospec/qunit_runner.rb
@ -159,7 +159,12 @@ module Autospec
        if m = /moduleFor\(['"]([^'"]+)/i.match(line)
          return m[1]
        end
+        if m = /moduleForComponent\(['"]([^"']+)/i.match(line)
+          return m[1]
+        end
      end
+
+      nil
    end

  end
--- a/lib/autospec/rspec_runner.rb
+++ b/lib/autospec/rspec_runner.rb
@ -19,6 +19,10 @@ module Autospec

    watch(%r{^spec/fabricators/.+_fabricator\.rb$})     { "spec" }

+    watch(%r{^app/assets/javascripts/pretty-text/.*\.js\.es6$}) { "spec/components/pretty_text_spec.rb"}
+
+    watch(%r{^plugins/.*/spec/.*\.rb})
+
    RELOADERS = Set.new
    def self.reload(pattern); RELOADERS << pattern; end
    def reloaders; RELOADERS; end
--- a/lib/autospec/simple_runner.rb
+++ b/lib/autospec/simple_runner.rb
@ -13,7 +13,12 @@ module Autospec
              "-f", "Autospec::Formatter", specs.split].flatten.join(" ")
      # launch rspec
      Dir.chdir(Rails.root) do
-        @pid = Process.spawn({"RAILS_ENV" => "test"}, "bin/rspec #{args}")
+        env = {"RAILS_ENV" => "test"}
+        if specs.split(' ').any?{|s| s =~ /^(.\/)?plugins/}
+          env["LOAD_PLUGINS"] = "1"
+          puts "Loading plugins while running specs"
+        end
+        @pid = Process.spawn(env, "bin/rspec #{args}")
        _, status = Process.wait2(@pid)
        status.exitstatus
      end
--- a/lib/html_normalize.rb
+++ b/lib/html_normalize.rb
@ -0,0 +1,150 @@
+# frozen_string_literal: true
+#
+# this class is used to normalize html output for internal comparisons in specs
+#
+require 'oga'
+
+class HtmlNormalize
+
+  def self.normalize(html)
+    parsed = Oga.parse_html(html.strip, strict: true)
+    if parsed.children.length != 1
+      puts parsed.children.count
+      raise "expecting a single child"
+    end
+    new(parsed.children.first).format
+  end
+
+  SELF_CLOSE = Set.new(%w{area base br col command embed hr img input keygen line meta param source track wbr})
+
+  BLOCK = Set.new(%w{
+    html
+    body
+    aside
+    p
+    h1 h2 h3 h4 h5 h6
+    ol ul
+    address
+    blockquote
+    dl
+    div
+    fieldset
+    form
+    hr
+    noscript
+    table
+    pre
+  })
+
+  def initialize(doc)
+    @doc = doc
+  end
+
+  def format
+    buffer = String.new
+    dump_node(@doc, 0, buffer)
+    buffer.strip!
+    buffer
+  end
+
+  def inline?(node)
+    Oga::XML::Text === node || !BLOCK.include?(node.name.downcase)
+  end
+
+  def dump_node(node, indent=0, buffer)
+
+    if Oga::XML::Text === node
+      if node.parent&.name
+        buffer << node.text
+      end
+      return
+    end
+
+    name = node.name.downcase
+
+    block = BLOCK.include?(name)
+
+    buffer << " " * indent * 2 if block
+
+    buffer << "<" << name
+
+    attrs = node&.attributes
+    if (attrs && attrs.length > 0)
+      attrs.sort!{|x,y| x.name <=> y.name}
+      attrs.each do |a|
+        buffer << " "
+        buffer << a.name
+        buffer << "='"
+        buffer << a.value
+        buffer << "'"
+      end
+    end
+
+    buffer << ">"
+
+    if block
+      buffer << "\n"
+    end
+
+    children = node.children
+    children = trim(children) if block
+
+    inline_buffer = nil
+
+    children&.each do |child|
+      if block && inline?(child)
+        inline_buffer ||= String.new
+        dump_node(child, indent+1, inline_buffer)
+      else
+        if inline_buffer
+          buffer << " " * (indent+1) * 2
+          buffer << inline_buffer.strip
+          inline_buffer = nil
+        else
+          dump_node(child, indent+1, buffer)
+        end
+      end
+    end
+
+    if inline_buffer
+      buffer << " " * (indent+1) * 2
+      buffer << inline_buffer.strip
+      inline_buffer = nil
+    end
+
+    if block
+      buffer << "\n" unless buffer[-1] == "\n"
+      buffer << " " * indent * 2
+    end
+
+    unless SELF_CLOSE.include?(name)
+      buffer << "</" << name
+      buffer << ">\n"
+    end
+  end
+
+  def trim(nodes)
+    start = 0
+    finish = nodes.length
+
+    nodes.each do |n|
+      if Oga::XML::Text === n && n.text.blank?
+        start += 1
+      else
+        break
+      end
+    end
+
+    nodes.reverse_each do |n|
+      if Oga::XML::Text === n && n.text.blank?
+        finish -= 1
+      else
+        break
+      end
+    end
+
+    nodes[start...finish]
+  end
+
+
+end
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@ -50,19 +50,10 @@ module PrettyText
    end
  end

-  def self.create_es6_context
-    ctx = MiniRacer::Context.new(timeout: 15000)
-
-    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
-
-    if Rails.env.development? || Rails.env.test?
-      ctx.attach("console.log", proc { |l| p l })
-    end
-
-    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
-    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
-    manifest = File.read("#{Rails.root}/app/assets/javascripts/pretty-text-bundle.js")
+  def self.ctx_load_manifest(ctx, name)
+    manifest = File.read("#{Rails.root}/app/assets/javascripts/#{name}")
    root_path = "#{Rails.root}/app/assets/javascripts/"
+
    manifest.each_line do |l|
      l = l.chomp
      if l =~ /\/\/= require (\.\/)?(.*)$/
@ -74,6 +65,22 @@ module PrettyText
        end
      end
    end
+  end
+
+  def self.create_es6_context
+    ctx = MiniRacer::Context.new(timeout: 15000)
+
+    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
+
+    if Rails.env.development? || Rails.env.test?
+      ctx.attach("console.log", proc { |l| p l })
+    end
+
+    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
+    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
+    ctx_load_manifest(ctx, "pretty-text-bundle.js")
+
+    root_path = "#{Rails.root}/app/assets/javascripts/"

    apply_es6_file(ctx, root_path, "discourse/lib/utilities")

@ -140,52 +147,62 @@ module PrettyText
        paths[:S3BaseUrl] = Discourse.store.absolute_base_url
      end

-      context.eval("__optInput = {};")
-      context.eval("__optInput.siteSettings = #{SiteSetting.client_settings_json};")
-      context.eval("__paths = #{paths.to_json};")
-
-      if opts[:topicId]
-        context.eval("__optInput.topicId = #{opts[:topicId].to_i};")
+      if SiteSetting.enable_experimental_markdown_it
+        unless context.eval("window.markdownit")
+          ctx_load_manifest(context, "markdown-it-bundle.js")
+        end
      end

-      context.eval("__optInput.userId = #{opts[:user_id].to_i};") if opts[:user_id]
-
-      context.eval("__optInput.getURL = __getURL;")
-      context.eval("__optInput.getCurrentUser = __getCurrentUser;")
-      context.eval("__optInput.lookupAvatar = __lookupAvatar;")
-      context.eval("__optInput.getTopicInfo = __getTopicInfo;")
-      context.eval("__optInput.categoryHashtagLookup = __categoryLookup;")
-      context.eval("__optInput.mentionLookup = __mentionLookup;")
-
      custom_emoji = {}
      Emoji.custom.map { |e| custom_emoji[e.name] = e.url }
-      context.eval("__optInput.customEmoji = #{custom_emoji.to_json};")

-      context.eval('__textOptions = __buildOptions(__optInput);')
+      buffer = <<~JS
+        __optInput = {};
+        __optInput.siteSettings = #{SiteSetting.client_settings_json};
+        __paths = #{paths.to_json};
+        __optInput.getURL = __getURL;
+        __optInput.getCurrentUser = __getCurrentUser;
+        __optInput.lookupAvatar = __lookupAvatar;
+        __optInput.getTopicInfo = __getTopicInfo;
+        __optInput.categoryHashtagLookup = __categoryLookup;
+        __optInput.mentionLookup = __mentionLookup;
+        __optInput.customEmoji = #{custom_emoji.to_json};
+      JS
+
+      if opts[:topicId]
+        buffer << "__optInput.topicId = #{opts[:topicId].to_i};\n"
+      end
+
+      if opts[:user_id]
+        buffer << "__optInput.userId = #{opts[:user_id].to_i};\n"
+      end
+
+      buffer << "__textOptions = __buildOptions(__optInput);\n"

      # Be careful disabling sanitization. We allow for custom emails
      if opts[:sanitize] == false
-        context.eval('__textOptions.sanitize = false;')
+        buffer << ('__textOptions.sanitize = false;')
      end

-      opts = context.eval("__pt = new __PrettyText(__textOptions);")
+      buffer << ("__pt = new __PrettyText(__textOptions);")
+      opts = context.eval(buffer)

      DiscourseEvent.trigger(:markdown_context, context)
      baked = context.eval("__pt.cook(#{text.inspect})")
    end

-    if baked.blank? && !(opts || {})[:skip_blank_test]
-      # we may have a js engine issue
-      test = markdown("a", skip_blank_test: true)
-      if test.blank?
-        Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
-        reset_context
-        opts ||= {}
-        opts = opts.dup
-        opts[:skip_blank_test] = true
-        baked = markdown(text, opts)
-      end
-    end
+    # if baked.blank? && !(opts || {})[:skip_blank_test]
+    #   # we may have a js engine issue
+    #   test = markdown("a", skip_blank_test: true)
+    #   if test.blank?
+    #     Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
+    #     reset_context
+    #     opts ||= {}
+    #     opts = opts.dup
+    #     opts[:skip_blank_test] = true
+    #     baked = markdown(text, opts)
+    #   end
+    # end

    baked
  end
--- a/plugins/discourse-details/assets/javascripts/lib/discourse-markdown/details.js.es6
+++ b/plugins/discourse-details/assets/javascripts/lib/discourse-markdown/details.js.es6
@ -20,6 +20,23 @@ registerOption((siteSettings, opts) => {
  opts.features.details = true;
 });

+const rule = {
+  tag: 'details',
+  before: function(state, attrs) {
+    state.push('bbcode_open', 'details', 1);
+    state.push('bbcode_open', 'summary', 1);
+
+    let token = state.push('text', '', 0);
+    token.content = attrs['_default'] || '';
+
+    state.push('bbcode_close', 'summary', -1);
+  },
+
+  after: function(state) {
+    state.push('bbcode_close', 'details', -1);
+  }
+};
+
 export function setup(helper) {
  helper.whiteList([
    'summary',
@ -29,5 +46,11 @@ export function setup(helper) {
    'details.elided'
  ]);

-  helper.addPreProcessor(text => replaceDetails(text));
+  if (helper.markdownIt) {
+    helper.registerPlugin(md => {
+      md.block.bbcode_ruler.push('details', rule);
+    });
+  } else {
+    helper.addPreProcessor(text => replaceDetails(text));
+  }
 }
--- a/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6
+++ b/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6
@ -14,6 +14,274 @@ registerOption((siteSettings, opts) => {
  opts.pollMaximumOptions = siteSettings.poll_maximum_options;
 });

+function getHelpText(count, min, max) {
+
+  // default values
+  if (isNaN(min) || min < 1) { min = 1; }
+  if (isNaN(max) || max > count) { max = count; }
+
+  // add some help text
+  let help;
+
+  if (max > 0) {
+    if (min === max) {
+      if (min > 1) {
+        help = I18n.t("poll.multiple.help.x_options", { count: min });
+      }
+    } else if (min > 1) {
+      if (max < count) {
+        help = I18n.t("poll.multiple.help.between_min_and_max_options", { min: min, max: max });
+      } else {
+        help = I18n.t("poll.multiple.help.at_least_min_options", { count: min });
+      }
+    } else if (max <= count) {
+      help = I18n.t("poll.multiple.help.up_to_max_options", { count: max });
+    }
+  }
+
+  return help;
+}
+
+function replaceToken(tokens, target, list) {
+  let pos = tokens.indexOf(target);
+  tokens.splice(pos, 1, ...list);
+  list[0].map = target.map;
+}
+
+// analyzes the block to that we have poll options
+function getListItems(tokens, startToken) {
+
+  let i = tokens.length-1;
+  let listItems = [];
+  let buffer = [];
+
+  for(;tokens[i]!==startToken;i--) {
+    if (i === 0) {
+      return;
+    }
+
+    let token = tokens[i];
+    if (token.level === 0) {
+      if (token.tag !== 'ol' && token.tag !== 'ul') {
+        return;
+      }
+    }
+
+    if (token.level === 1 && token.nesting === 1) {
+      if (token.tag === 'li') {
+        listItems.push([token, buffer.reverse().join(' ')]);
+      } else {
+        return;
+      }
+    }
+
+    if (token.level === 1 && token.nesting === 1 && token.tag === 'li') {
+      buffer = [];
+    } else {
+      if (token.type === 'text' || token.type === 'inline') {
+        buffer.push(token.content);
+      }
+    }
+  }
+
+  return listItems.reverse();
+}
+
+function invalidPoll(state, tag) {
+  let token = state.push('text', '', 0);
+  token.content = '[/' + tag + ']';
+}
+
+const rule = {
+  tag: 'poll',
+
+  before: function(state, attrs, md, raw){
+    let token = state.push('text', '', 0);
+    token.content = raw;
+    token.bbcode_attrs = attrs;
+    token.bbcode_type = 'poll_open';
+  },
+
+  after: function(state, openToken, md, raw) {
+
+    let items = getListItems(state.tokens, openToken);
+
+    const attrs = openToken.bbcode_attrs;
+
+    // default poll attributes
+    const attributes = [["class", "poll"]];
+    attributes.push([DATA_PREFIX + "status", "open"]);
+
+    WHITELISTED_ATTRIBUTES.forEach(name => {
+      if (attrs[name]) {
+        attributes[DATA_PREFIX + name] = attrs[name];
+      }
+    });
+
+    if (!attrs.name) {
+      attributes.push([DATA_PREFIX + "name", DEFAULT_POLL_NAME]);
+    }
+
+    // we might need these values later...
+    let min = parseInt(attributes[DATA_PREFIX + "min"], 10);
+    let max = parseInt(attributes[DATA_PREFIX + "max"], 10);
+    let step = parseInt(attributes[DATA_PREFIX + "step"], 10);
+
+    let header = [];
+
+    let token = new state.Token('poll_open', 'div', 1);
+    token.block = true;
+    token.attrs = attributes;
+    header.push(token);
+
+    token = new state.Token('poll_open', 'div', 1);
+    token.block = true;
+    header.push(token);
+
+    token = new state.Token('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-container']];
+
+    header.push(token);
+
+    // generate the options when the type is "number"
+    if (attributes[DATA_PREFIX + "type"] === "number") {
+      // default values
+      if (isNaN(min)) { min = 1; }
+      if (isNaN(max)) { max = md.options.discourse.pollMaximumOptions; }
+      if (isNaN(step)) { step = 1; }
+
+      if (items.length > 0) {
+        return invalidPoll(state, raw);
+      }
+
+      // dynamically generate options
+      token = new state.Token('bullet_list_open', 'ul', 1);
+      header.push(token);
+
+      for (let o = min; o <= max; o += step) {
+        token = new state.Token('list_item_open', '', 1);
+        items.push([token,  String(o)]);
+        header.push(token);
+
+        token = new state.Token('text', '', 0);
+        token.content = String(o);
+        header.push(token);
+
+        token = new state.Token('list_item_close', '', -1);
+        header.push(token);
+      }
+      token = new state.Token('bullet_item_close', '', -1);
+      header.push(token);
+    }
+
+    if (items.length < 2) {
+      return invalidPoll(state, raw);
+    }
+
+    // flag items so we add hashes
+    for (let o = 0; o < items.length; o++) {
+      token = items[o][0];
+      let text = items[o][1];
+
+      token.attrs = token.attrs || [];
+      let md5Hash = md5(JSON.stringify([text]));
+      token.attrs.push([DATA_PREFIX + 'option-id', md5Hash]);
+    }
+
+    replaceToken(state.tokens, openToken, header);
+
+    state.push('poll_close', 'div', -1);
+
+    token = state.push('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-info']];
+
+    state.push('paragraph_open', 'p', 1);
+
+    token = state.push('span_open', 'span', 1);
+    token.block = false;
+    token.attrs = [['class', 'info-number']];
+    token = state.push('text', '', 0);
+    token.content = '0';
+    state.push('span_close', 'span', -1);
+
+    token = state.push('span_open', 'span', 1);
+    token.block = false;
+    token.attrs = [['class', 'info-text']];
+    token = state.push('text', '', 0);
+    token.content = I18n.t("poll.voters", { count: 0 });
+    state.push('span_close', 'span', -1);
+
+    state.push('paragraph_close', 'p', -1);
+
+    // multiple help text
+    if (attributes[DATA_PREFIX + "type"] === "multiple") {
+      let help = getHelpText(items.length, min, max);
+      if (help) {
+        state.push('paragraph_open', 'p', 1);
+        token = state.push('html_inline', '', 0);
+        token.content = help;
+        state.push('paragraph_close', 'p', -1);
+      }
+    }
+
+    if (attributes[DATA_PREFIX + 'public'] === 'true') {
+      state.push('paragraph_open', 'p', 1);
+      token = state.push('text', '', 0);
+      token.content = I18n.t('poll.public.title');
+      state.push('paragraph_close', 'p', -1);
+    }
+
+    state.push('poll_close', 'div', -1);
+    state.push('poll_close', 'div', -1);
+
+    token = state.push('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-buttons']];
+
+    if (attributes[DATA_PREFIX + 'type'] === 'multiple') {
+      token = state.push('link_open', 'a', 1);
+      token.block = false;
+      token.attrs = [
+        ['class', 'button cast-votes'],
+        ['title', I18n.t('poll.cast-votes.title')]
+      ];
+
+      token = state.push('text', '', 0);
+      token.content = I18n.t('poll.cast-votes.label');
+
+      state.push('link_close', 'a', -1);
+    }
+
+    token = state.push('link_open', 'a', 1);
+    token.block = false;
+    token.attrs = [
+      ['class', 'button toggle-results'],
+      ['title', I18n.t('poll.show-results.title')]
+    ];
+
+    token = state.push('text', '', 0);
+    token.content = I18n.t("poll.show-results.label");
+
+    state.push('link_close', 'a', -1);
+
+    state.push('poll_close', 'div', -1);
+    state.push('poll_close', 'div', -1);
+  }
+};
+
+function newApiInit(helper) {
+  helper.registerOptions((opts, siteSettings) => {
+    const currentUser = (opts.getCurrentUser && opts.getCurrentUser(opts.userId)) || opts.currentUser;
+    const staff = currentUser && currentUser.staff;
+
+    opts.features.poll = !!siteSettings.poll_enabled || staff;
+    opts.pollMaximumOptions = siteSettings.poll_maximum_options;
+  });
+
+  helper.registerPlugin(md => {
+    md.block.bbcode_ruler.push('poll', rule);
+  });
+}
+
 export function setup(helper) {
  helper.whiteList([
    'div.poll',
@ -28,6 +296,11 @@ export function setup(helper) {
    'li[data-*]'
  ]);

+  if (helper.markdownIt) {
+    newApiInit(helper);
+    return;
+  }
+
  helper.replaceBlock({
    start: /\[poll((?:\s+\w+=[^\s\]]+)*)\]([\s\S]*)/igm,
    stop: /\[\/poll\]/igm,
--- a/plugins/poll/spec/lib/pretty_text_spec.rb
+++ b/plugins/poll/spec/lib/pretty_text_spec.rb
@ -0,0 +1,101 @@
+require 'rails_helper'
+require 'html_normalize'
+
+describe PrettyText do
+
+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  context 'markdown it' do
+    before do
+      SiteSetting.enable_experimental_markdown_it = true
+    end
+
+    it 'works correctly for new vs old engine with trivial cases' do
+      md = <<~MD
+        [poll]
+        1. test 1
+        2. test 2
+        [/poll]
+      MD
+
+      new_engine = n(PrettyText.cook(md))
+
+      SiteSetting.enable_experimental_markdown_it = false
+      old_engine = n(PrettyText.cook(md))
+
+      expect(new_engine).to eq(old_engine)
+    end
+
+    it 'does not break poll options when going from loose to tight' do
+      md = <<~MD
+        [poll type=multiple]
+        1. test 1 :) <b>test</b>
+        2. test 2
+        [/poll]
+      MD
+
+      tight_cooked = PrettyText.cook(md)
+
+      md = <<~MD
+        [poll type=multiple]
+
+        1. test 1 :) <b>test</b>
+
+        2. test 2
+
+        [/poll]
+      MD
+
+      loose_cooked = PrettyText.cook(md)
+
+      tight_hashes = tight_cooked.scan(/data-poll-option-id=['"]([^'"]+)/)
+      loose_hashes = loose_cooked.scan(/data-poll-option-id=['"]([^'"]+)/)
+
+      expect(tight_hashes).to eq(loose_hashes)
+    end
+
+    it 'can correctly cook polls' do
+      md = <<~MD
+        [poll type=multiple]
+        1. test 1 :) <b>test</b>
+        2. test 2
+        [/poll]
+      MD
+
+      cooked = PrettyText.cook md
+
+      expected = <<~MD
+        <div class="poll" data-poll-status="open" data-poll-name="poll">
+        <div>
+        <div class="poll-container">
+        <ol>
+        <li data-poll-option-id='b6475cbf6acb8676b20c60582cfc487a'>test 1 <img alt=':slight_smile:' class='emoji' src='/images/emoji/emoji_one/slight_smile.png?v=5' title=':slight_smile:'> <b>test</b>
+        </li>
+        <li data-poll-option-id='7158af352698eb1443d709818df097d4'>test 2</li>
+        </li>
+        </ol>
+        </div>
+        <div class="poll-info">
+        <p>
+        <span class="info-number">0</span>
+        <span class="info-text">voters</span>
+        </p>
+        <p>
+        Choose up to <strong>2</strong> options</p>
+        </div>
+        </div>
+        <div class="poll-buttons">
+        <a title="Cast your votes">Vote now!</a>
+        <a title="Display the poll results">Show results</a>
+        </div>
+        </div>
+      MD
+
+      # note, hashes should remain stable even if emoji changes cause text content is hashed
+      expect(n cooked).to eq(n expected)
+
+    end
+  end
+end
--- a/script/benchmarks/markdown/.gitignore
+++ b/script/benchmarks/markdown/.gitignore
@ -0,0 +1 @@
+tmp/*
--- a/script/benchmarks/markdown/bench.rb
+++ b/script/benchmarks/markdown/bench.rb
@ -0,0 +1,54 @@
+require 'benchmark/ips'
+require File.expand_path('../../../../config/environment', __FILE__)
+
+
+tests = [
+  ["tiny post", "**hello**"],
+  ["giant post", File.read("giant_post.md")],
+  ["most features", File.read("most_features.md")],
+  ["lots of mentions", File.read("lots_of_mentions.md")]
+]
+
+SiteSetting.enable_experimental_markdown_it = true
+PrettyText.cook("")
+PrettyText.v8.eval("window.commonmark = window.markdownit('commonmark')")
+
+# Benchmark.ips do |x|
+#   x.report("markdown") do
+#     PrettyText.markdown("x")
+#   end
+#
+#   x.report("cook") do
+#     PrettyText.cook("y")
+#   end
+# end
+#
+# exit
+
+Benchmark.ips do |x|
+  [true,false].each do |sanitize|
+    {
+      "markdown js" =>
+        lambda{SiteSetting.enable_experimental_markdown_it = false},
+
+      "markdown it" =>
+        lambda{SiteSetting.enable_experimental_markdown_it = true}
+    }.each do |name, before|
+      before.call
+
+      tests.each do |test, text|
+        x.report("#{name} #{test} sanitize: #{sanitize}") do
+          PrettyText.markdown(text, sanitize: sanitize)
+        end
+      end
+    end
+  end
+
+
+  tests.each do |test, text|
+    x.report("markdown it no extensions commonmark #{test}") do
+      PrettyText.v8.eval("window.commonmark.render(#{text.inspect})")
+    end
+  end
+end
+
--- a/script/benchmarks/markdown/giant_post.md
+++ b/script/benchmarks/markdown/giant_post.md
@ -0,0 +1,583 @@
+    [discourse]# ./launcher bootstrap app
+    which: no docker.io in (/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin)
+    
+    WARNING: We are about to start downloading the Discourse base image
+    This process may take anywhere between a few minutes to an hour, depending on your network speed
+    
+    Please be patient
+    
+    Unable to find image 'samsaffron/discourse:1.0.13' locally
+    1.0.13: Pulling from samsaffron/discourse
+    ........
+    
+    Fast-forward
+     .travis.yml                                        |   3 -
+     Gemfile                                            |  25 +-
+     Gemfile.lock                                       | 298 ++++++-----
+     README.md                                          |  23 +-
+     .../admin/components/embedding-setting.js.es6      |   5 +
+     .../screened_ip_address_form_component.js          |  23 +-
+     .../admin/controllers/admin-site-settings.js.es6   |  28 +-
+     .../admin/controllers/admin-user-badges.js.es6     |   4 +-
+     .../javascripts/admin/models/admin-user.js.es6     |  12 +-
+     .../javascripts/admin/models/staff_action_log.js   |   3 +-
+     app/assets/javascripts/admin/templates/admin.hbs   |   2 +-
+     .../templates/components/embedding-setting.hbs     |   2 +-
+     .../admin/templates/components/site-setting.hbs    |   2 +-
+     .../javascripts/admin/templates/dashboard.hbs      |   2 +-
+     .../javascripts/admin/templates/embedding.hbs      |   9 +-
+     .../admin/templates/modal/admin_agree_flag.hbs     |   6 +-
+     .../admin/templates/modal/admin_delete_flag.hbs    |   2 +-
+     .../javascripts/admin/templates/plugins-index.hbs  |  19 +-
+     .../admin/templates/site-settings-category.hbs     |   2 +-
+     .../javascripts/admin/templates/site-settings.hbs  |   6 +-
+     .../javascripts/admin/templates/site-text-edit.hbs |   2 +-
+     .../javascripts/admin/templates/user-index.hbs     |  36 +-
+     app/assets/javascripts/discourse.js                |   2 +-
+     .../javascripts/discourse/adapters/rest.js.es6     |  14 +-
+     .../discourse/components/actions-summary.js.es6    |   4 +-
+     .../discourse/components/category-chooser.js.es6   |   2 +-
+     .../discourse/components/d-editor-modal.js.es6     |  52 ++
+     .../discourse/components/d-editor.js.es6           | 263 ++++++++++
+     .../javascripts/discourse/components/d-link.js.es6 |   7 +-
+     .../discourse/components/date-picker.js.es6        |  22 +-
+     .../components/desktop-notification-config.js.es6  |  60 ++-
+     .../discourse/components/image-uploader.js.es6     |   9 +-
+     .../discourse/components/menu-panel.js.es6         |   3 +-
+     .../discourse/components/notification-item.js.es6  |  10 +-
+     .../discourse/components/pagedown-editor.js.es6    |  23 -
+     .../discourse/components/post-gutter.js.es6        |  53 +-
+     .../discourse/components/post-menu.js.es6          |   9 +-
+     .../components/private-message-map.js.es6          |   2 +-
+     .../discourse/components/who-liked.js.es6          |   2 +-
+     .../discourse/controllers/change-owner.js.es6      |   5 +-
+     .../discourse/controllers/composer.js.es6          |   4 +-
+     .../controllers/discovery-sortable.js.es6          |   6 +-
+     .../discourse/controllers/discovery/topics.js.es6  |  11 +-
+     .../discourse/controllers/edit-category.js.es6     |  17 +-
+     .../discourse/controllers/full-page-search.js.es6  |   2 +-
+     .../javascripts/discourse/controllers/login.js.es6 |  37 +-
+     .../discourse/controllers/quote-button.js.es6      |  38 +-
+     .../javascripts/discourse/controllers/topic.js.es6 |  52 +-
+     .../discourse/controllers/user-card.js.es6         |   1 +
+     .../javascripts/discourse/controllers/user.js.es6  |  58 ++-
+     .../javascripts/discourse/dialects/dialect.js      |   2 +-
+     .../discourse/dialects/quote_dialect.js            |  33 +-
+     .../discourse/initializers/enable-emoji.js.es6     |  10 +-
+     .../discourse/initializers/load-all-helpers.js.es6 |  17 +-
+     .../subscribe-user-notifications.js.es6            |  12 +-
+     .../javascripts/discourse/lib/Markdown.Editor.js   |   3 +-
+     .../javascripts/discourse/lib/autocomplete.js.es6  |  15 +-
+     .../discourse/lib/desktop-notifications.js.es6     |  11 +-
+     .../discourse/lib/emoji/emoji-groups.js.es6        |  57 ++
+     .../discourse/lib/emoji/emoji-toolbar.js.es6       | 259 ++++------
+     .../discourse/lib/key-value-store.js.es6           |  16 +-
+     .../discourse/lib/keyboard-shortcuts.js.es6        |  15 +-
+     .../javascripts/discourse/models/post.js.es6       |  12 +-
+     .../javascripts/discourse/models/topic-list.js.es6 |  56 +-
+     .../javascripts/discourse/models/topic.js.es6      |   2 +
+     .../javascripts/discourse/models/user.js.es6       |   4 +
+     .../pre-initializers/dynamic-route-builders.js.es6 |  47 +-
+     .../pre-initializers/sniff-capabilities.js.es6     |  21 +-
+     .../discourse/routes/app-route-map.js.es6          |  31 +-
+     .../discourse/routes/badges-show.js.es6            |   2 +-
+     .../discourse/routes/build-category-route.js.es6   |  44 +-
+     .../discourse/routes/build-static-route.js.es6     |  18 +
+     .../discourse/routes/build-topic-route.js.es6      |  20 +-
+     .../javascripts/discourse/routes/discovery.js.es6  |   4 +
+     .../discourse/routes/forgot-password.js.es6        |  27 +-
+     .../javascripts/discourse/routes/login.js.es6      |  31 +-
+     .../templates/components/category-unread.hbs       |   2 +-
+     .../templates/components/d-editor-modal.hbs        |   7 +
+     .../discourse/templates/components/d-editor.hbs    |  32 ++
+     .../templates/components/edit-category-images.hbs  |   2 +-
+     .../components/edit-category-topic-template.hbs    |   2 +-
+     .../discourse/templates/components/home-logo.hbs   |   3 -
+     .../templates/components/pagedown-editor.hbs       |   4 -
+     .../discourse/templates/components/user-menu.hbs   |   4 +-
+     .../javascripts/discourse/templates/composer.hbs   |   4 +-
+     .../discourse/templates/discovery/topics.hbs       |  23 +-
+     .../templates/emoji-selector-autocomplete.raw.hbs  |  10 +-
+     .../javascripts/discourse/templates/header.hbs     |   2 +-
+     .../discourse/templates/login-preferences.hbs      |   8 +
+     .../templates/mobile/discovery/topics.hbs          |  17 +-
+     .../templates/mobile/list/topic_list_item.raw.hbs  |  49 +-
+     .../discourse/templates/modal/create-account.hbs   |  17 +-
+     .../discourse/templates/modal/dismiss-read.hbs     |  10 +
+     .../discourse/templates/navigation/category.hbs    |   7 +-
+     .../javascripts/discourse/templates/post.hbs       |  13 +-
+     .../discourse/templates/queued-posts.hbs           |   2 +-
+     .../javascripts/discourse/templates/user/about.hbs |   2 +-
+     .../discourse/templates/user/preferences.hbs       |   3 +-
+     .../javascripts/discourse/templates/user/user.hbs  |  13 +-
+     .../discourse/views/cloaked-collection.js.es6      |   3 +
+     .../javascripts/discourse/views/composer.js.es6    |  94 ++--
+     .../discourse/views/embedded-post.js.es6           |   2 +
+     app/assets/javascripts/discourse/views/post.js.es6 |  13 +-
+     .../discourse/views/quote-button.js.es6            |  26 +-
+     .../discourse/views/topic-entrance.js.es6          |  21 +-
+     .../discourse/views/upload-selector.js.es6         |   6 +-
+     app/assets/javascripts/main_include.js             |   6 +-
+     app/assets/stylesheets/common.scss                 |   1 +
+     .../stylesheets/common/admin/admin_base.scss       |   8 -
+     app/assets/stylesheets/common/base/compose.scss    |  12 +-
+     app/assets/stylesheets/common/base/discourse.scss  |  18 -
+     app/assets/stylesheets/common/base/emoji.scss      |   6 +-
+     app/assets/stylesheets/common/base/header.scss     |   6 -
+     app/assets/stylesheets/common/base/modal.scss      |   4 -
+     app/assets/stylesheets/common/base/onebox.scss     |   2 +-
+     .../stylesheets/common/base/topic-admin-menu.scss  |   2 +-
+     app/assets/stylesheets/common/base/topic-post.scss |   2 -
+     app/assets/stylesheets/common/base/topic.scss      |   8 +-
+     app/assets/stylesheets/common/base/upload.scss     |   8 +-
+     .../stylesheets/common/components/badges.css.scss  |   8 +-
+     app/assets/stylesheets/common/d-editor.scss        |  94 ++++
+     app/assets/stylesheets/desktop/compose.scss        |  10 +-
+     app/assets/stylesheets/desktop/discourse.scss      |   2 +-
+     app/assets/stylesheets/desktop/topic-post.scss     |   8 +-
+     app/assets/stylesheets/desktop/topic.scss          |   5 +-
+     app/assets/stylesheets/desktop/upload.scss         |   2 +-
+     app/assets/stylesheets/desktop/user-card.scss      |   2 +-
+     app/assets/stylesheets/desktop/user.scss           |   8 -
+     app/assets/stylesheets/mobile.scss                 |   1 +
+     app/assets/stylesheets/mobile/alert.scss           |   2 +
+     app/assets/stylesheets/mobile/banner.scss          |   4 +-
+     app/assets/stylesheets/mobile/discourse.scss       |   3 +-
+     app/assets/stylesheets/mobile/emoji.scss           |   3 +
+     app/assets/stylesheets/mobile/header.scss          |   4 +-
+     app/assets/stylesheets/mobile/topic-list.scss      |  32 +-
+     app/assets/stylesheets/mobile/topic-post.scss      |  16 +-
+     app/assets/stylesheets/mobile/topic.scss           |   3 +-
+     app/assets/stylesheets/mobile/user.scss            |   8 +-
+     app/controllers/admin/diagnostics_controller.rb    |  13 +
+     app/controllers/admin/email_controller.rb          |   6 +
+     app/controllers/admin/embedding_controller.rb      |   4 +
+     app/controllers/application_controller.rb          |  15 +-
+     app/controllers/categories_controller.rb           |  32 +-
+     app/controllers/list_controller.rb                 |  29 +-
+     app/controllers/manifest_json_controller.rb        |  15 +
+     app/controllers/permalinks_controller.rb           |   2 +-
+     app/controllers/post_action_users_controller.rb    |  22 +
+     app/controllers/post_actions_controller.rb         |  13 +-
+     app/controllers/posts_controller.rb                |  30 +-
+     app/controllers/robots_txt_controller.rb           |   9 +-
+     app/controllers/topics_controller.rb               |   2 +-
+     .../users/omniauth_callbacks_controller.rb         |  25 +-
+     app/controllers/users_controller.rb                |  26 +-
+     app/helpers/application_helper.rb                  |  22 +-
+     app/jobs/regular/post_alert.rb                     |   2 +-
+     app/jobs/regular/process_post.rb                   |   4 +-
+     app/jobs/scheduled/periodical_updates.rb           |   4 +-
+     app/mailers/user_notifications.rb                  |   1 +
+     app/models/admin_dashboard_data.rb                 |   1 +
+     app/models/anon_site_json_cache_observer.rb        |  12 +
+     app/models/badge.rb                                |  28 +-
+     app/models/category.rb                             |  19 +-
+     app/models/category_group.rb                       |   2 +
+     app/models/color_scheme.rb                         |  22 +-
+     app/models/group.rb                                |   7 +
+     app/models/permalink.rb                            |   2 +-
+     app/models/post.rb                                 |  21 +-
+     app/models/post_action.rb                          |   6 +-
+     app/models/post_action_type.rb                     |  10 +
+     app/models/post_analyzer.rb                        |   2 +-
+     app/models/report.rb                               |  13 +-
+     app/models/screened_ip_address.rb                  |   1 +
+     app/models/site.rb                                 |  63 ++-
+     app/models/topic.rb                                |  65 ++-
+     app/models/topic_featured_users.rb                 |   4 +-
+     app/models/topic_link.rb                           |  14 +-
+     app/models/topic_link_click.rb                     |   2 +-
+     app/models/topic_tracking_state.rb                 |  16 +-
+     app/models/topic_user.rb                           |  15 -
+     app/models/upload.rb                               |   5 +-
+     app/models/user.rb                                 |   4 +-
+     app/models/user_history.rb                         |  13 +-
+     app/models/user_profile.rb                         |   2 +
+     app/models/user_profile_view.rb                    |  47 ++
+     app/serializers/admin_detailed_user_serializer.rb  |   2 +-
+     app/serializers/application_serializer.rb          |  25 +
+     app/serializers/badge_serializer.rb                |  15 +-
+     app/serializers/basic_post_serializer.rb           |   6 +-
+     app/serializers/post_action_user_serializer.rb     |   7 +-
+     app/serializers/post_serializer.rb                 |   2 +-
+     app/serializers/site_serializer.rb                 |  26 +-
+     app/serializers/topic_view_serializer.rb           |   8 +-
+     app/serializers/user_history_serializer.rb         |   1 +
+     app/serializers/user_serializer.rb                 |   7 +-
+     app/services/badge_granter.rb                      |  10 +-
+     app/services/post_alerter.rb                       |   3 +-
+     app/services/random_topic_selector.rb              |  16 +-
+     app/services/staff_action_logger.rb                |  62 +++
+     app/views/common/_discourse_javascript.html.erb    |   5 +
+     app/views/layouts/_head.html.erb                   |  11 +-
+     app/views/layouts/application.html.erb             |   1 +
+     app/views/list/list.erb                            |   3 +-
+     app/views/posts/latest.rss.erb                     |   2 +-
+     app/views/topics/plain.html.erb                    |   4 +-
+     app/views/topics/show.html.erb                     |   4 +-
+     app/views/user_notifications/digest.html.erb       |   4 +-
+     .../users/omniauth_callbacks/complete.html.erb     |   4 +-
+     app/views/users/show.html.erb                      |   4 +-
+     config/application.rb                              |   5 +-
+     config/database.yml                                |   2 +
+     config/discourse_defaults.conf                     |   8 +-
+     config/environments/production.rb                  |   2 +-
+     config/environments/profile.rb                     |   2 +-
+     config/environments/test.rb                        |   2 +-
+     config/initializers/04-message_bus.rb              |   2 +-
+     config/initializers/i18n.rb                        |  15 +-
+     config/locales/client.ar.yml                       |  45 +-
+     config/locales/client.bs_BA.yml                    |  28 +-
+     config/locales/client.cs.yml                       |   9 -
+     config/locales/client.da.yml                       | 168 +++++-
+     config/locales/client.de.yml                       |  65 ++-
+     config/locales/client.en.yml                       |  42 +-
+     config/locales/client.es.yml                       |  35 +-
+     config/locales/client.fa_IR.yml                    |   9 -
+     config/locales/client.fi.yml                       | 125 ++++-
+     config/locales/client.fr.yml                       |  56 +-
+     config/locales/client.he.yml                       | 575 +++++++++++----------
+     config/locales/client.it.yml                       | 133 +++--
+     config/locales/client.ja.yml                       |   9 -
+     config/locales/client.ko.yml                       | 496 +++++++++---------
+     config/locales/client.nb_NO.yml                    |  16 +-
+     config/locales/client.nl.yml                       |  43 +-
+     config/locales/client.pl_PL.yml                    |  66 ++-
+     config/locales/client.pt.yml                       |  37 +-
+     config/locales/client.pt_BR.yml                    | 215 +++++++-
+     config/locales/client.ro.yml                       |   8 -
+     config/locales/client.ru.yml                       | 172 ++++--
+     config/locales/client.sq.yml                       |   9 -
+     config/locales/client.sv.yml                       |   9 -
+     config/locales/client.te.yml                       |   8 -
+     config/locales/client.tr_TR.yml                    |  11 +-
+     config/locales/client.uk.yml                       |   6 -
+     config/locales/client.zh_CN.yml                    |  22 +-
+     config/locales/client.zh_TW.yml                    |  11 +-
+     config/locales/server.ar.yml                       |  61 ++-
+     config/locales/server.bs_BA.yml                    |   2 -
+     config/locales/server.cs.yml                       |   1 -
+     config/locales/server.da.yml                       | 107 +++-
+     config/locales/server.de.yml                       | 238 ++++++++-
+     config/locales/server.en.yml                       |  37 +-
+     config/locales/server.es.yml                       | 134 ++++-
+     config/locales/server.fa_IR.yml                    |   2 -
+     config/locales/server.fi.yml                       |  85 ++-
+     config/locales/server.fr.yml                       |   4 -
+     config/locales/server.he.yml                       |  21 +-
+     config/locales/server.it.yml                       |  44 +-
+     config/locales/server.ja.yml                       |   2 -
+     config/locales/server.ko.yml                       |   2 -
+     config/locales/server.nb_NO.yml                    |   1 -
+     config/locales/server.nl.yml                       |   9 +-
+     config/locales/server.pl_PL.yml                    | 236 ++++++++-
+     config/locales/server.pt.yml                       |  31 +-
+     config/locales/server.pt_BR.yml                    |  14 +-
+     config/locales/server.ru.yml                       |  48 +-
+     config/locales/server.sq.yml                       |   2 -
+     config/locales/server.sv.yml                       |   3 +-
+     config/locales/server.te.yml                       |   1 -
+     config/locales/server.tr_TR.yml                    |   4 -
+     config/locales/server.zh_CN.yml                    |  23 +-
+     config/locales/server.zh_TW.yml                    |  38 +-
+     config/routes.rb                                   |   7 +-
+     config/site_settings.yml                           |  26 +-
+     .../20150914021445_create_user_profile_views.rb    |  15 +
+     .../20150914034541_add_views_to_user_profile.rb    |   5 +
+     ...0917071017_add_category_id_to_user_histories.rb |   6 +
+     .../20150924022040_add_fancy_title_to_topic.rb     |   5 +
+     .../20150925000915_exclude_whispers_from_badges.rb |  22 +
+     docs/INSTALL-cloud.md                              |  59 +--
+     docs/VAGRANT.md                                    |   2 +-
+     lib/cooked_post_processor.rb                       |  27 +-
+     lib/discourse.rb                                   |   6 +-
+     lib/discourse_redis.rb                             |   7 +-
+     lib/edit_rate_limiter.rb                           |   6 +-
+     lib/email.rb                                       |   9 +-
+     lib/email/message_builder.rb                       |   2 +-
+     lib/email/renderer.rb                              |   4 +-
+     lib/email/sender.rb                                |   4 +-
+     lib/email/styles.rb                                |  14 +-
+     lib/file_store/base_store.rb                       |   6 +-
+     lib/freedom_patches/i18n_fallbacks.rb              |   2 +
+     lib/freedom_patches/pool_drainer.rb                |  11 +-
+     lib/guardian.rb                                    |   2 +-
+     lib/guardian/category_guardian.rb                  |   7 +-
+     lib/guardian/post_guardian.rb                      |   2 +-
+     lib/html_prettify.rb                               | 407 +++++++++++++++
+     lib/onebox/engine/discourse_local_onebox.rb        |  11 +-
+     lib/oneboxer.rb                                    |  20 +-
+     lib/plugin/auth_provider.rb                        |  21 +-
+     lib/plugin/instance.rb                             |  10 +-
+     lib/post_creator.rb                                |  21 +-
+     lib/post_revisor.rb                                |  10 +-
+     lib/pretty_text.rb                                 |  21 +
+     lib/rate_limiter.rb                                |  11 +-
+     lib/rate_limiter/limit_exceeded.rb                 |  25 +-
+     lib/search.rb                                      |   2 +-
+     lib/site_setting_extension.rb                      |   1 +
+     lib/tasks/assets.rake                              |  27 +-
+     lib/tasks/db.rake                                  |   2 +
+     lib/tasks/posts.rake                               |  20 +-
+     lib/topic_creator.rb                               |   4 +-
+     lib/topic_query.rb                                 |   9 +-
+     lib/topic_view.rb                                  |   6 +-
+     lib/version.rb                                     |   4 +-
+     plugins/poll/config/locales/client.ar.yml          |  74 ++-
+     plugins/poll/config/locales/client.bs_BA.yml       |  13 +-
+     plugins/poll/config/locales/client.cs.yml          |   3 -
+     plugins/poll/config/locales/client.da.yml          |  14 +-
+     plugins/poll/config/locales/client.de.yml          |  12 +-
+     plugins/poll/config/locales/client.en.yml          |  12 +-
+     plugins/poll/config/locales/client.es.yml          |  12 +-
+     plugins/poll/config/locales/client.fa_IR.yml       |   3 -
+     plugins/poll/config/locales/client.fi.yml          |  12 +-
+     plugins/poll/config/locales/client.fr.yml          |   3 -
+     plugins/poll/config/locales/client.he.yml          |  12 +-
+     plugins/poll/config/locales/client.id.yml          |   3 -
+     plugins/poll/config/locales/client.it.yml          |  12 +-
+     plugins/poll/config/locales/client.ja.yml          |   3 -
+     plugins/poll/config/locales/client.ko.yml          |   3 -
+     plugins/poll/config/locales/client.nb_NO.yml       |   3 -
+     plugins/poll/config/locales/client.nl.yml          |   9 +-
+     plugins/poll/config/locales/client.pl_PL.yml       |  15 +-
+     plugins/poll/config/locales/client.pt.yml          |  12 +-
+     plugins/poll/config/locales/client.pt_BR.yml       |  18 +-
+     plugins/poll/config/locales/client.ro.yml          |   3 -
+     plugins/poll/config/locales/client.ru.yml          |  46 +-
+     plugins/poll/config/locales/client.sq.yml          |   3 -
+     plugins/poll/config/locales/client.sv.yml          |   3 -
+     plugins/poll/config/locales/client.tr_TR.yml       |   3 -
+     plugins/poll/config/locales/client.zh_CN.yml       |   9 +-
+     plugins/poll/config/locales/server.ar.yml          |  54 +-
+     plugins/poll/config/locales/server.bs_BA.yml       |   5 +-
+     plugins/poll/config/locales/server.cs.yml          |   2 -
+     plugins/poll/config/locales/server.da.yml          |   8 +-
+     plugins/poll/config/locales/server.de.yml          |   8 +-
+     plugins/poll/config/locales/server.en.yml          |  11 +-
+     plugins/poll/config/locales/server.es.yml          |   8 +-
+     plugins/poll/config/locales/server.fa_IR.yml       |   2 -
+     plugins/poll/config/locales/server.fi.yml          |   8 +-
+     plugins/poll/config/locales/server.fr.yml          |   2 -
+     plugins/poll/config/locales/server.he.yml          |   8 +-
+     plugins/poll/config/locales/server.it.yml          |   8 +-
+     plugins/poll/config/locales/server.ja.yml          |   4 +-
+     plugins/poll/config/locales/server.ko.yml          |   2 -
+     plugins/poll/config/locales/server.nb_NO.yml       |   2 -
+     plugins/poll/config/locales/server.nl.yml          |   8 +-
+     plugins/poll/config/locales/server.pl_PL.yml       |  10 +-
+     plugins/poll/config/locales/server.pt.yml          |   8 +-
+     plugins/poll/config/locales/server.pt_BR.yml       |  11 +-
+     plugins/poll/config/locales/server.ru.yml          |  52 +-
+     plugins/poll/config/locales/server.sq.yml          |   2 -
+     plugins/poll/config/locales/server.sv.yml          |   2 -
+     plugins/poll/config/locales/server.tr_TR.yml       |   2 -
+     plugins/poll/config/locales/server.zh_CN.yml       |   6 +-
+     .../db/migrate/20151016163051_merge_polls_votes.rb |  20 +
+     plugins/poll/plugin.rb                             |  62 ++-
+     .../poll/spec/controllers/posts_controller_spec.rb |  57 +-
+     public/403.ar.html                                 |   4 +-
+     public/403.it.html                                 |   2 +-
+     public/403.zh_CN.html                              |   2 +-
+     public/500.zh_CN.html                              |   2 +-
+     public/images/welcome/reply-post-2x.png            | Bin 549 -> 430 bytes
+     .../welcome/topic-notification-control-2x.png      | Bin 39580 -> 50219 bytes
+     public/javascripts/pikaday.js                      |   6 +-
+     script/import_scripts/base.rb                      |  21 +-
+     script/import_scripts/lithium.rb                   |  22 +-
+     script/import_scripts/mbox.rb                      |  52 +-
+     script/import_scripts/mybb.rb                      |  38 +-
+     .../import_scripts/phpbb3/database/database_3_0.rb |   2 +-
+     .../phpbb3/database/database_base.rb               |   2 +-
+     script/import_scripts/phpbb3/importer.rb           |   8 +-
+     .../phpbb3/importers/message_importer.rb           |  11 +-
+     .../phpbb3/importers/post_importer.rb              |   4 +
+     .../phpbb3/importers/user_importer.rb              |  11 +-
+     script/import_scripts/vbulletin.rb                 |   4 +-
+     spec/components/cooked_post_processor_spec.rb      |  23 +-
+     spec/components/email/receiver_spec.rb             |   8 +-
+     spec/components/email/sender_spec.rb               |   5 +-
+     spec/components/guardian_spec.rb                   |  20 +-
+     spec/components/html_prettify_spec.rb              |  30 ++
+     .../onebox/engine/discourse_local_onebox_spec.rb   |   7 +-
+     spec/components/post_creator_spec.rb               |  36 +-
+     spec/components/pretty_text_spec.rb                |  15 +-
+     spec/components/topic_creator_spec.rb              |  33 +-
+     spec/controllers/categories_controller_spec.rb     |  13 +
+     spec/controllers/manifest_json_controller_spec.rb  |  12 +
+     spec/controllers/permalinks_controller_spec.rb     |  10 +
+     .../post_action_users_controller_spec.rb           |  34 ++
+     spec/controllers/post_actions_controller_spec.rb   |  35 --
+     spec/controllers/posts_controller_spec.rb          |  41 +-
+     spec/controllers/session_controller_spec.rb        |   1 +
+     spec/controllers/users_controller_spec.rb          |  30 +-
+     spec/fabricators/category_group_fabricator.rb      |   5 +
+     spec/fabricators/post_fabricator.rb                |  17 +-
+     spec/fixtures/emails/paragraphs.cooked             |   2 +-
+     spec/helpers/i18n_fallbacks_spec.rb                |  52 ++
+     spec/models/category_spec.rb                       |   9 +
+     spec/models/color_scheme_spec.rb                   |   9 +-
+     spec/models/post_action_spec.rb                    |  16 +
+     spec/models/screened_ip_address_spec.rb            |  61 ++-
+     spec/models/site_spec.rb                           |   3 +
+     spec/models/topic_link_spec.rb                     |   2 +-
+     spec/models/topic_spec.rb                          |  21 +-
+     spec/models/topic_tracking_state_spec.rb           |  16 -
+     spec/models/user_email_observer_spec.rb            |  43 +-
+     spec/models/user_profile_view_spec.rb              |  39 ++
+     spec/models/user_spec.rb                           |  14 +-
+     spec/services/post_alerter_spec.rb                 |  14 +
+     spec/services/staff_action_logger_spec.rb          |  77 +++
+     .../acceptance/category-edit-test.js.es6           |   2 +-
+     .../controllers/admin-user-badges-test.js.es6      |  11 +-
+     test/javascripts/components/d-editor-test.js.es6   | 461 +++++++++++++++++
+     test/javascripts/components/d-link-test.js.es6     |   4 -
+     test/javascripts/helpers/component-test.js.es6     |  12 +-
+     test/javascripts/lib/discourse-test.js.es6         |   7 +
+     test/javascripts/lib/markdown-test.js.es6          |   4 +
+     test/javascripts/models/post-stream-test.js.es6    |   2 +-
+     test/javascripts/test_helper.js                    |   3 +
+     test/stylesheets/test_helper.css                   |   8 +
+     .../lib/discourse_imgur/locale/server.ar.yml       |   4 +-
+     vendor/gems/rails_multisite/.gitignore             |  17 -
+     vendor/gems/rails_multisite/Gemfile                |  14 -
+     vendor/gems/rails_multisite/Guardfile              |   9 -
+     vendor/gems/rails_multisite/LICENSE                |  22 -
+     vendor/gems/rails_multisite/README.md              |  29 --
+     vendor/gems/rails_multisite/Rakefile               |   7 -
+     vendor/gems/rails_multisite/lib/rails_multisite.rb |   3 -
+     .../lib/rails_multisite/connection_management.rb   | 190 -------
+     .../rails_multisite/lib/rails_multisite/railtie.rb |  23 -
+     .../rails_multisite/lib/rails_multisite/version.rb |   3 -
+     vendor/gems/rails_multisite/lib/tasks/db.rake      |  31 --
+     .../gems/rails_multisite/lib/tasks/generators.rake |  26 -
+     .../gems/rails_multisite/rails_multisite.gemspec   |  20 -
+     .../spec/connection_management_rack_spec.rb        |  47 --
+     .../spec/connection_management_spec.rb             |  99 ----
+     .../rails_multisite/spec/fixtures/database.yml     |   6 -
+     .../gems/rails_multisite/spec/fixtures/two_dbs.yml |   6 -
+     vendor/gems/rails_multisite/spec/spec_helper.rb    |  38 --
+     456 files changed, 7519 insertions(+), 3458 deletions(-)
+     create mode 100644 app/assets/javascripts/discourse/components/d-editor-modal.js.es6
+     create mode 100644 app/assets/javascripts/discourse/components/d-editor.js.es6
+     delete mode 100644 app/assets/javascripts/discourse/components/pagedown-editor.js.es6
+     create mode 100644 app/assets/javascripts/discourse/lib/emoji/emoji-groups.js.es6
+     create mode 100644 app/assets/javascripts/discourse/routes/build-static-route.js.es6
+     create mode 100644 app/assets/javascripts/discourse/templates/components/d-editor-modal.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/components/d-editor.hbs
+     delete mode 100644 app/assets/javascripts/discourse/templates/components/pagedown-editor.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/login-preferences.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/modal/dismiss-read.hbs
+     create mode 100644 app/assets/stylesheets/common/d-editor.scss
+     create mode 100644 app/assets/stylesheets/mobile/emoji.scss
+     create mode 100644 app/controllers/manifest_json_controller.rb
+     create mode 100644 app/controllers/post_action_users_controller.rb
+     create mode 100644 app/models/anon_site_json_cache_observer.rb
+     create mode 100644 app/models/user_profile_view.rb
+     create mode 100644 db/migrate/20150914021445_create_user_profile_views.rb
+     create mode 100644 db/migrate/20150914034541_add_views_to_user_profile.rb
+     create mode 100644 db/migrate/20150917071017_add_category_id_to_user_histories.rb
+     create mode 100644 db/migrate/20150924022040_add_fancy_title_to_topic.rb
+     create mode 100644 db/migrate/20150925000915_exclude_whispers_from_badges.rb
+     create mode 100644 lib/html_prettify.rb
+     create mode 100644 plugins/poll/db/migrate/20151016163051_merge_polls_votes.rb
+     create mode 100644 spec/components/html_prettify_spec.rb
+     create mode 100644 spec/controllers/manifest_json_controller_spec.rb
+     create mode 100644 spec/controllers/post_action_users_controller_spec.rb
+     create mode 100644 spec/fabricators/category_group_fabricator.rb
+     create mode 100644 spec/helpers/i18n_fallbacks_spec.rb
+     create mode 100644 spec/models/user_profile_view_spec.rb
+     create mode 100644 test/javascripts/components/d-editor-test.js.es6
+     create mode 100644 test/javascripts/lib/discourse-test.js.es6
+     delete mode 100644 vendor/gems/rails_multisite/.gitignore
+     delete mode 100644 vendor/gems/rails_multisite/Gemfile
+     delete mode 100644 vendor/gems/rails_multisite/Guardfile
+     delete mode 100644 vendor/gems/rails_multisite/LICENSE
+     delete mode 100644 vendor/gems/rails_multisite/README.md
+     delete mode 100755 vendor/gems/rails_multisite/Rakefile
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/connection_management.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/railtie.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/version.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/tasks/db.rake
+     delete mode 100644 vendor/gems/rails_multisite/lib/tasks/generators.rake
+     delete mode 100644 vendor/gems/rails_multisite/rails_multisite.gemspec
+     delete mode 100644 vendor/gems/rails_multisite/spec/connection_management_rack_spec.rb
+     delete mode 100644 vendor/gems/rails_multisite/spec/connection_management_spec.rb
+     delete mode 100644 vendor/gems/rails_multisite/spec/fixtures/database.yml
+     delete mode 100644 vendor/gems/rails_multisite/spec/fixtures/two_dbs.yml
+     delete mode 100644 vendor/gems/rails_multisite/spec/spec_helper.rb
+    
+    I, [2015-10-23T15:53:58.134756 #42]  INFO -- : > cd /var/www/discourse && git fetch origin tests-passed
+    From https://github.com/discourse/discourse
+     * branch            tests-passed -> FETCH_HEAD
+    I, [2015-10-23T15:54:04.068910 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.069344 #42]  INFO -- : > cd /var/www/discourse && git checkout tests-passed
+    Switched to a new branch 'tests-passed'
+    I, [2015-10-23T15:54:04.200168 #42]  INFO -- : Branch tests-passed set up to track remote branch tests-passed from origin.
+    
+    I, [2015-10-23T15:54:04.200518 #42]  INFO -- : > cd /var/www/discourse && mkdir -p tmp/pids
+    I, [2015-10-23T15:54:04.205068 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.205515 #42]  INFO -- : > cd /var/www/discourse && mkdir -p tmp/sockets
+    I, [2015-10-23T15:54:04.209201 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.209397 #42]  INFO -- : > cd /var/www/discourse && touch tmp/.gitkeep
+    I, [2015-10-23T15:54:04.213544 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.213795 #42]  INFO -- : > cd /var/www/discourse && mkdir -p                    /shared/log/rails
+    I, [2015-10-23T15:54:04.217537 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.217765 #42]  INFO -- : > cd /var/www/discourse && bash -c "touch -a           /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr}.log"
+    I, [2015-10-23T15:54:04.222814 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.223049 #42]  INFO -- : > cd /var/www/discourse && bash -c "ln    -s           /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr}.log /var/www/discourse/log"
+    I, [2015-10-23T15:54:04.229000 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.229490 #42]  INFO -- : > cd /var/www/discourse && bash -c "mkdir -p           /shared/{uploads,backups}"
+    I, [2015-10-23T15:54:04.236051 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.236497 #42]  INFO -- : > cd /var/www/discourse && bash -c "ln    -s           /shared/{uploads,backups} /var/www/discourse/public"
+    I, [2015-10-23T15:54:04.242560 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.242977 #42]  INFO -- : > cd /var/www/discourse && chown -R discourse:www-data /shared/log/rails /shared/uploads /shared/backups
+    I, [2015-10-23T15:54:04.249820 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.250574 #42]  INFO -- : Replacing # redis with sv start redis || exit 1 in /etc/service/unicorn/run
+    I, [2015-10-23T15:54:04.254889 #42]  INFO -- : > cd /var/www/discourse/plugins && mkdir -p plugins
+    I, [2015-10-23T15:54:04.261446 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.261965 #42]  INFO -- : > cd /var/www/discourse/plugins && git clone https://github.com/discourse/docker_manager.git
+    Cloning into 'docker_manager'...
+    I, [2015-10-23T15:54:06.823958 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.826947 #42]  INFO -- : > cp /var/www/discourse/config/nginx.sample.conf /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.834182 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.834792 #42]  INFO -- : > rm /etc/nginx/sites-enabled/default
+    I, [2015-10-23T15:54:06.839072 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.839549 #42]  INFO -- : > mkdir -p /var/nginx/cache
+    I, [2015-10-23T15:54:06.843773 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.845326 #42]  INFO -- : Replacing pid /run/nginx.pid; with daemon off; in /etc/nginx/nginx.conf
+    I, [2015-10-23T15:54:06.846966 #42]  INFO -- : Replacing (?m-ix:upstream[^\}]+\}) with upstream discourse { server 127.0.0.1:3000; } in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.848224 #42]  INFO -- : Replacing (?-mix:server_name.+$) with server_name _ ; in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.849445 #42]  INFO -- : Replacing (?-mix:client_max_body_size.+$) with client_max_body_size $upload_size ; in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.851315 #42]  INFO -- : > echo "done configuring web"
+    I, [2015-10-23T15:54:06.856465 #42]  INFO -- : done configuring web
+    
+    I, [2015-10-23T15:54:06.858840 #42]  INFO -- : > cd /var/www/discourse && gem update bundler
+    ERROR:  While executing gem ... (Gem::RemoteFetcher::FetchError)
+        hostname "rubygems.global.ssl.fastly.net" does not match the server certificate (https://rubygems.global.ssl.fastly.net/specs.4.8.gz)
+    I, [2015-10-23T15:54:27.329080 #42]  INFO -- : Updating installed gems
+    
+    I, [2015-10-23T15:54:27.330007 #42]  INFO -- : Terminating async processes
+    I, [2015-10-23T15:54:27.330217 #42]  INFO -- : Sending INT to HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/9.3/bin/postmaster -D /etc/postgresql/9.3/main pid: 112
+    2015-10-23 15:54:27 UTC [112-2] LOG:  received fast shutdown request
+    2015-10-23 15:54:27 UTC [112-3] LOG:  aborting any active transactions
+    2015-10-23 15:54:27 UTC [119-2] LOG:  autovacuum launcher shutting down
+    2015-10-23 15:54:27 UTC [116-1] LOG:  shutting down
+    I, [2015-10-23T15:54:27.330394 #42]  INFO -- : Sending TERM to exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf pid: 240
+    240:signal-handler (1445615667) Received SIGTERM scheduling shutdown...
+    240:M 23 Oct 15:54:27.359 # User requested shutdown...
+    240:M 23 Oct 15:54:27.359 * Saving the final RDB snapshot before exiting.
+    240:M 23 Oct 15:54:27.371 * DB saved on disk
+    240:M 23 Oct 15:54:27.371 # Redis is now ready to exit, bye bye...
+    2015-10-23 15:54:31 UTC [116-2] LOG:  database system is shut down
+    
+    
+    FAILED
+    --------------------
+    RuntimeError: cd /var/www/discourse && gem update bundler failed with return #<Process::Status: pid 335 exit 1>
+    Location of failure: /pups/lib/pups/exec_command.rb:105:in `spawn'
+    exec failed with the params {"cd"=>"$home", "hook"=>"web", "cmd"=>["gem update bundler", "chown -R discourse $home"]}
+    035a935af484328809d3399e2bfca421f5de3165b113fedc7c8dfe76dd7a07f1
+    ** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one
--- a/script/benchmarks/markdown/lots_of_mentions.md
+++ b/script/benchmarks/markdown/lots_of_mentions.md
@ -0,0 +1,84 @@
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
--- a/script/benchmarks/markdown/most_features.md
+++ b/script/benchmarks/markdown/most_features.md
@ -0,0 +1,201 @@
+Based off https://markdown-it.github.io/ with every feature we support...
+
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
+ Create a list by starting a line with `+`, `-`, or `*`
+ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
+ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+
+```text
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/discourse 
+
+
+## Images
+
+![Minion](/uploads/default/original/1X/f038dc6544a178b470b3014e92377b4dc996b991.png)
+![](/uploads/default/original/1X/974402975b9ec316057a9e331bbade74d225bc46.jpg)
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: /uploads/default/original/1X/7bd599f0af2da1f370ea7b49ebec6e73c32d722b.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink:  :cry: :laughing: :yum: :surfing_woman:t4:   
+>
+> Shortcuts (emoticons) ;) :) 
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### Polls
+
+[poll type=number min=1 max=20 step=1 public=true]
+[/poll]
+
+[details=Summary]This is a spoiler[/details]
+
+
+Multiline spoiler
+
+[details=Summary]
+
+This is a spoiler
+
+[/details]
+
+
+### Mentions
+Mentions ... @sam 
+
+
+### Categories
+
+ #site-feedback
+
+### Inline bbcode
+
+Hello [code]I am code[/code] 
+
+
+
+## A few paragraphs of **bacon**
+
+Bacon ipsum dolor amet boudin ham hock burgdoggen, strip steak leberkas corned beef pork chop rump short loin porchetta shank venison andouille spare ribs turkey. Boudin tri-tip picanha chicken, porchetta beef ribs hamburger leberkas shankle flank pork spare ribs cupim biltong. Meatball pig leberkas sirloin beef tenderloin tongue picanha ham biltong ribeye chicken. Ham beef chuck frankfurter bresaola pig. Beef turkey ground round kevin pork belly doner jowl. Chicken burgdoggen shankle brisket short ribs capicola beef pancetta.
+
+Rump t-bone beef ribs, cupim pork loin bresaola drumstick frankfurter capicola. Doner pastrami shank ribeye turkey ham hock meatloaf sirloin biltong pig ball tip beef ribs short loin shoulder. Meatball ribeye pastrami shank strip steak porchetta burgdoggen jowl short ribs sausage tail beef landjaeger capicola swine. Alcatra pork chop pork loin turkey, rump tenderloin landjaeger meatball swine ham hock strip steak sirloin. Strip steak drumstick tenderloin ground round, tongue ball tip t-bone tri-tip. Tenderloin doner boudin, sausage beef filet mignon short ribs.
+
+Meatloaf pork loin pork belly porchetta landjaeger frankfurter fatback chicken. Short loin boudin bacon pastrami ball tip. Chicken burgdoggen bresaola chuck porchetta. Swine spare ribs cupim, shoulder rump boudin shank pork belly porchetta chicken pancetta beef meatloaf. Prosciutto shoulder hamburger, pig corned beef picanha filet mignon shankle t-bone jowl rump. Tri-tip pork burgdoggen flank salami short loin cow fatback pig ball tip kielbasa venison ham hock.
+
+Flank jowl pastrami beef swine pork loin. Tail strip steak leberkas t-bone sausage, bresaola rump pastrami meatloaf short ribs prosciutto bacon cupim cow. Beef ribs shoulder ham hock beef meatloaf. Doner sausage porchetta, tongue pork chop jerky boudin meatball shoulder hamburger ribeye beef ribs. Pastrami turkey flank tri-tip, sausage ball tip rump ground round shankle.
--- a/spec/components/html_normalize_spec.rb
+++ b/spec/components/html_normalize_spec.rb
@ -0,0 +1,69 @@
+require 'rails_helper'
+require 'html_normalize'
+
+describe HtmlNormalize do
+
+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  it "handles self closing tags" do
+
+    source = <<-HTML
+<div>
+  <span><img src='testing'>
+  boo</span>
+</div>
+HTML
+    expect(n source).to eq(source.strip)
+  end
+
+  it "Can handle aside" do
+
+      source = <<~HTML
+        <aside class="quote" data-topic="2" data-post="1">
+          <div class="title">
+          <div class="quote-controls"></div>
+          <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/x/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a></div>
+          <blockquote>
+          <p>ddd</p>
+          </blockquote></aside>
+HTML
+      expected = <<~HTML
+        <aside class="quote" data-post="1" data-topic="2">
+          <div class="title">
+          <div class="quote-controls"></div>
+          <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/x/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a>
+          </div>
+          <blockquote>
+            <p>ddd</p>
+          </blockquote>
+        </aside>
+HTML
+
+      expect(n expected).to eq(n source)
+  end
+
+  it "Can normalize attributes" do
+
+    source = "<a class='a b' name='sam'>b</a>"
+    same = "<a name='sam' class='a b' >b</a>"
+
+    expect(n source).to eq(n same)
+  end
+
+  it "Can indent divs nicely" do
+    source = "<div> <div><div>hello world</div> </div>     </div>"
+    expected = <<~HTML
+        <div>
+          <div>
+            <div>
+              hello world
+            </div>
+          </div>
+        </div>
+HTML
+
+    expect(n source).to eq(expected.strip)
+  end
+end
--- a/spec/components/pretty_text_spec.rb
+++ b/spec/components/pretty_text_spec.rb
@ -1,8 +1,17 @@
 require 'rails_helper'
 require 'pretty_text'
+require 'html_normalize'

 describe PrettyText do

+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  def cook(*args)
+    n(PrettyText.cook(*args))
+  end
+
  let(:wrapped_image) { "<div class=\"lightbox-wrapper\"><a href=\"//localhost:3000/uploads/default/4399/33691397e78b4d75.png\" class=\"lightbox\" title=\"Screen Shot 2014-04-14 at 9.47.10 PM.png\"><img src=\"//localhost:3000/uploads/default/_optimized/bd9/b20/bbbcd6a0c0_655x500.png\" width=\"655\" height=\"500\"><div class=\"meta\">\n<span class=\"filename\">Screen Shot 2014-04-14 at 9.47.10 PM.png</span><span class=\"informations\">966x737 1.47 MB</span><span class=\"expand\"></span>\n</div></a></div>" }
  let(:wrapped_image_excerpt) {  }

@ -489,4 +498,146 @@ HTML
    end
  end

+  context "markdown it" do
+
+    before do
+      SiteSetting.enable_experimental_markdown_it = true
+    end
+
+    # it "replaces skin toned emoji" do
+    #   expect(PrettyText.cook("hello 👱🏿‍♀️")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/blonde_woman/6.png?v=5\" title=\":blonde_woman:t6:\" class=\"emoji\" alt=\":blonde_woman:t6:\"></p>")
+    #   expect(PrettyText.cook("hello 👩‍🎤")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_singer.png?v=5\" title=\":woman_singer:\" class=\"emoji\" alt=\":woman_singer:\"></p>")
+    #   expect(PrettyText.cook("hello 👩🏾‍🎓")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_student/5.png?v=5\" title=\":woman_student:t5:\" class=\"emoji\" alt=\":woman_student:t5:\"></p>")
+    #   expect(PrettyText.cook("hello 🤷‍♀️")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_shrugging.png?v=5\" title=\":woman_shrugging:\" class=\"emoji\" alt=\":woman_shrugging:\"></p>")
+    # end
+    #
+
+    it "produces tag links" do
+      Fabricate(:topic, {tags: [Fabricate(:tag, name: 'known')]})
+      expect(PrettyText.cook("x #unknown::tag #known::tag")).to match_html("<p>x <span class=\"hashtag\">#unknown::tag</span> <a class=\"hashtag\" href=\"http://test.localhost/tags/known\">#<span>known</span></a></p>")
+    end
+
+    it "can handle mixed lists" do
+      # known bug in old md engine
+      cooked = PrettyText.cook("* a\n\n1. b")
+      expect(cooked).to match_html("<ul>\n<li>a</li>\n</ul><ol>\n<li>b</li>\n</ol>")
+    end
+
+    it "can handle traditional vs non traditional newlines" do
+      SiteSetting.traditional_markdown_linebreaks = true
+      expect(PrettyText.cook("1\n2")).to match_html "<p>1 2</p>"
+
+      SiteSetting.traditional_markdown_linebreaks = false
+      expect(PrettyText.cook("1\n2")).to match_html "<p>1<br>\n2</p>"
+    end
+
+    it "can handle mentions" do
+      Fabricate(:user, username: "sam")
+      expect(PrettyText.cook("hi @sam! hi")).to match_html '<p>hi <a class="mention" href="/u/sam">@sam</a>! hi</p>'
+    end
+
+    it "can handle mentions inside a hyperlink" do
+    expect(PrettyText.cook("<a> @inner</a> ")).to match_html '<p><a> @inner</a></p>'
+    end
+
+    it "can handle mentions inside a hyperlink" do
+      expect(PrettyText.cook("[link @inner](http://site.com)")).to match_html '<p><a href="http://site.com" rel="nofollow noopener">link @inner</a></p>'
+    end
+
+    it "can handle a list of mentions" do
+      expect(PrettyText.cook("@a,@b")).to match_html('<p><span class="mention">@a</span>,<span class="mention">@b</span></p>')
+    end
+
+    it "can handle emoji by name" do
+
+      expected = <<HTML
+<p><img src="/images/emoji/emoji_one/smile.png?v=5\" title=":smile:" class="emoji" alt=":smile:"><img src="/images/emoji/emoji_one/sunny.png?v=5" title=":sunny:" class="emoji" alt=":sunny:"></p>
+HTML
+      expect(PrettyText.cook(":smile::sunny:")).to eq(expected.strip)
+    end
+
+    it "handles emoji boundaries correctly" do
+      cooked = PrettyText.cook("a,:man:t2:,b")
+      expected = '<p>a,<img src="/images/emoji/emoji_one/man/2.png?v=5" title=":man:t2:" class="emoji" alt=":man:t2:">,b</p>'
+      expect(cooked).to match(expected.strip)
+    end
+
+    it "can handle emoji by translation" do
+      expected = '<p><img src="/images/emoji/emoji_one/wink.png?v=5" title=":wink:" class="emoji" alt=":wink:"></p>'
+      expect(PrettyText.cook(";)")).to eq(expected)
+    end
+
+    it "can handle multiple emojis by translation" do
+      cooked = PrettyText.cook(":) ;) :)")
+      expect(cooked.split("img").length-1).to eq(3)
+    end
+
+    it "handles emoji boundries correctly" do
+      expect(PrettyText.cook(",:)")).to include("emoji")
+      expect(PrettyText.cook(":-)\n")).to include("emoji")
+      expect(PrettyText.cook("a :)")).to include("emoji")
+      expect(PrettyText.cook(":),")).not_to include("emoji")
+      expect(PrettyText.cook("abcde ^:;-P")).to include("emoji")
+    end
+
+
+    it 'can include code class correctly' do
+      expect(PrettyText.cook("```cpp\ncpp\n```")).to match_html("<pre><code class='lang-cpp'>cpp\n</code></pre>")
+    end
+
+    it 'indents code correctly' do
+      code = "X\n```\n\n    #\n    x\n```"
+      cooked = PrettyText.cook(code)
+      expect(cooked).to match_html("<p>X</p>\n<pre><code class=\"lang-auto\">\n    #\n    x\n</code></pre>")
+    end
+
+    it 'can censor words correctly' do
+      SiteSetting.censored_words = 'apple|banana'
+      expect(PrettyText.cook('yay banana yay')).not_to include('banana')
+      expect(PrettyText.cook('yay `banana` yay')).not_to include('banana')
+      expect(PrettyText.cook("yay \n\n```\nbanana\n````\n yay")).not_to include('banana')
+      expect(PrettyText.cook("# banana")).not_to include('banana')
+      expect(PrettyText.cook("# banana")).to include("\u25a0\u25a0")
+    end
+
+    it 'handles onebox correctly' do
+      # we expect 2 oneboxes
+      expect(PrettyText.cook("http://a.com\nhttp://b.com").split("onebox").length).to eq(3)
+      expect(PrettyText.cook("http://a.com\n\nhttp://b.com").split("onebox").length).to eq(3)
+
+      expect(PrettyText.cook("a\nhttp://a.com")).to include('onebox')
+      expect(PrettyText.cook("> http://a.com")).not_to include('onebox')
+      expect(PrettyText.cook("a\nhttp://a.com a")).not_to include('onebox')
+      expect(PrettyText.cook("a\nhttp://a.com\na")).to include('onebox')
+      expect(PrettyText.cook("http://a.com")).to include('onebox')
+      expect(PrettyText.cook("a.com")).not_to include('onebox')
+      expect(PrettyText.cook("http://a.com ")).to include('onebox')
+      expect(PrettyText.cook("http://a.com a")).not_to include('onebox')
+      expect(PrettyText.cook("- http://a.com")).not_to include('onebox')
+    end
+
+    it "can handle bbcode" do
+      expect(PrettyText.cook("a[b]b[/b]c")).to eq('<p>a<span class="bbcode-b">b</span>c</p>')
+      expect(PrettyText.cook("a[i]b[/i]c")).to eq('<p>a<span class="bbcode-i">b</span>c</p>')
+    end
+
+    it "do off topic quoting with emoji unescape" do
+
+      topic = Fabricate(:topic, title: "this is a test topic :slight_smile:")
+      expected = <<HTML
+<aside class="quote" data-topic="#{topic.id}" data-post="2">
+<div class="title">
+  <div class="quote-controls"></div>
+  <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/#{topic.id}/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a>
+</div>
+<blockquote>
+  <p>ddd</p>
+</blockquote>
+</aside>
+HTML
+
+      expect(cook("[quote=\"EvilTrout, post:2, topic:#{topic.id}\"]\nddd\n[/quote]", topic_id: 1)).to eq(n(expected))
+    end
+  end
+
 end
--- a/test/javascripts/components/d-editor-test.js.es6
+++ b/test/javascripts/components/d-editor-test.js.es6
@ -21,7 +21,7 @@ componentTest('preview sanitizes HTML', {
  template: '{{d-editor value=value}}',

  test(assert) {
-    this.set('value', `"><svg onload="prompt(/xss/)"></svg>`);
+    fillIn('.d-editor-input', `"><svg onload="prompt(/xss/)"></svg>`);
    andThen(() => {
      assert.equal(this.$('.d-editor-preview').html().trim(), '<p>\"&gt;</p>');
    });
--- a/test/javascripts/test_helper.js
+++ b/test/javascripts/test_helper.js
@ -22,6 +22,7 @@
 //= require vendor
 //= require ember-shim
 //= require pretty-text-bundle
+//= require markdown-it-bundle
 //= require application
 //= require plugin
 //= require htmlparser.js
--- a/vendor/assets/javascripts/markdown-it.js
+++ b/vendor/assets/javascripts/markdown-it.js