import AllowLister from "pretty-text/allow-lister"; import { cloneJSON } from "discourse-common/lib/object"; import deprecated from "discourse-common/lib/deprecated"; import guid from "pretty-text/guid"; import { sanitize } from "pretty-text/sanitizer"; export const ATTACHMENT_CSS_CLASS = "attachment"; function deprecate(feature, name) { return function () { if (window.console && window.console.log) { window.console.log( feature + ": " + name + " is deprecated, please use the new markdown it APIs" ); } }; } // We have some custom extensions and extension points for markdown-it, including // the helper (passed in via setup) that has registerOptions, registerPlugin etc. // as well as our postProcessText rule to replace text with a regex. // // Take a look at https://meta.discourse.org/t/developers-guide-to-markdown-extensions/66023 // for more detailed information. function createHelper( featureName, opts, optionCallbacks, pluginCallbacks, customMarkdownCookFnCallbacks, getOptions, allowListed ) { let helper = {}; helper.markdownIt = true; helper.allowList = (info) => allowListed.push([featureName, info]); helper.whiteList = (info) => { deprecated("`whiteList` has been replaced with `allowList`", { since: "2.6.0.beta.4", dropFrom: "2.7.0", }); helper.allowList(info); }; helper.registerInline = deprecate(featureName, "registerInline"); helper.replaceBlock = deprecate(featureName, "replaceBlock"); helper.addPreProcessor = deprecate(featureName, "addPreProcessor"); helper.inlineReplace = deprecate(featureName, "inlineReplace"); helper.postProcessTag = deprecate(featureName, "postProcessTag"); helper.inlineRegexp = deprecate(featureName, "inlineRegexp"); helper.inlineBetween = deprecate(featureName, "inlineBetween"); helper.postProcessText = deprecate(featureName, "postProcessText"); helper.onParseNode = deprecate(featureName, "onParseNode"); helper.registerBlock = deprecate(featureName, "registerBlock"); // hack to allow moving of getOptions helper.getOptions = () => getOptions.f(); helper.registerOptions = (callback) => { optionCallbacks.push([featureName, callback]); }; helper.registerPlugin = (callback) => { pluginCallbacks.push([featureName, callback]); }; helper.buildCookFunction = (callback) => { customMarkdownCookFnCallbacks.push([featureName, callback]); }; return helper; } // TODO we may just use a proper ruler from markdown it... this is a basic proxy class Ruler { constructor() { this.rules = []; } getRules() { return this.rules; } getRuleForTag(tag) { this.ensureCache(); if (this.cache.hasOwnProperty(tag)) { return this.cache[tag]; } } ensureCache() { if (this.cache) { return; } this.cache = {}; for (let i = this.rules.length - 1; i >= 0; i--) { let info = this.rules[i]; this.cache[info.rule.tag] = info; } } push(name, rule) { this.rules.push({ name, rule }); this.cache = null; } } // block bb code ruler for parsing of quotes / code / polls function setupBlockBBCode(md) { md.block.bbcode = { ruler: new Ruler() }; } // inline bbcode ruler for parsing of spoiler tags, discourse-chart etc function setupInlineBBCode(md) { md.inline.bbcode = { ruler: new Ruler() }; } // rule for text replacement via regex, used for @mentions, category hashtags, etc. function setupTextPostProcessRuler(md) { const TextPostProcessRuler = requirejs( "pretty-text/engines/discourse-markdown/text-post-process" ).TextPostProcessRuler; md.core.textPostProcess = { ruler: new TextPostProcessRuler() }; } // hoists html_raw tokens out of the render flow and replaces them // with a GUID. this GUID is then replaced with the final raw HTML // content in unhoistForCooked function renderHoisted(tokens, idx, options) { const content = tokens[idx].content; if (content && content.length > 0) { let id = guid(); options.discourse.hoisted[id] = content; return id; } else { return ""; } } function setupUrlDecoding(md) { // this fixed a subtle issue where %20 is decoded as space in // automatic urls md.utils.lib.mdurl.decode.defaultChars = ";/?:@&=+$,# "; } // html_raw tokens, funnily enough, render raw HTML via renderHoisted and // unhoistForCooked function setupHoister(md) { md.renderer.rules.html_raw = renderHoisted; } // videoHTML and audioHTML follow the same HTML syntax // as oneboxer.rb when dealing with these formats function videoHTML(token) { const src = token.attrGet("src"); const origSrc = token.attrGet("data-orig-src"); const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : ""; return `
`; } function audioHTML(token) { const src = token.attrGet("src"); const origSrc = token.attrGet("data-orig-src"); const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : ""; return ``; } const IMG_SIZE_REGEX = /^([1-9]+[0-9]*)x([1-9]+[0-9]*)(\s*,\s*(x?)([1-9][0-9]{0,2}?)([%x]?))?$/; function renderImageOrPlayableMedia(tokens, idx, options, env, slf) { const token = tokens[idx]; const alt = slf.renderInlineAsText(token.children, options, env); const split = alt.split("|"); const altSplit = [split[0]]; // markdown-it supports returning HTML instead of continuing to render the current token // see https://github.com/markdown-it/markdown-it/blob/master/docs/architecture.md#renderer // handles |video and |audio alt transformations for image tags if (split[1] === "video") { if ( options.discourse.previewing && !options.discourse.limitedSiteSettings.enableDiffhtmlPreview ) { return `
`; } else { return videoHTML(token); } } else if (split[1] === "audio") { return audioHTML(token); } // parsing ![myimage|500x300]() or ![myimage|75%]() or ![myimage|500x300, 75%] for (let i = 1, match, data; i < split.length; ++i) { if ((match = split[i].match(IMG_SIZE_REGEX)) && match[1] && match[2]) { let width = match[1]; let height = match[2]; // calculate using percentage if (match[5] && match[6] && match[6] === "%") { let percent = parseFloat(match[5]) / 100.0; width = parseInt(width * percent, 10); height = parseInt(height * percent, 10); } // calculate using only given width if (match[5] && match[6] && match[6] === "x") { let wr = parseFloat(match[5]) / width; width = parseInt(match[5], 10); height = parseInt(height * wr, 10); } // calculate using only given height if (match[5] && match[4] && match[4] === "x" && !match[6]) { let hr = parseFloat(match[5]) / height; height = parseInt(match[5], 10); width = parseInt(width * hr, 10); } if (token.attrIndex("width") === -1) { token.attrs.push(["width", width]); } if (token.attrIndex("height") === -1) { token.attrs.push(["height", height]); } if ( options.discourse.previewing && match[6] !== "x" && match[4] !== "x" ) { token.attrs.push(["class", "resizable"]); } } else if ((data = extractDataAttribute(split[i]))) { token.attrs.push(data); } else if (split[i] === "thumbnail") { token.attrs.push(["data-thumbnail", "true"]); } else { altSplit.push(split[i]); } } token.attrs[token.attrIndex("alt")][1] = altSplit.join("|"); return slf.renderToken(tokens, idx, options); } // we have taken over the ![]() syntax in markdown to // be able to render a video or audio URL as well as the // image using |video and |audio in the text inside [] function setupImageAndPlayableMediaRenderer(md) { md.renderer.rules.image = renderImageOrPlayableMedia; } function renderAttachment(tokens, idx, options, env, slf) { const linkToken = tokens[idx]; const textToken = tokens[idx + 1]; const split = textToken.content.split("|"); const contentSplit = []; for (let i = 0, data; i < split.length; ++i) { if (split[i] === ATTACHMENT_CSS_CLASS) { linkToken.attrs.unshift(["class", split[i]]); } else if ((data = extractDataAttribute(split[i]))) { linkToken.attrs.push(data); } else { contentSplit.push(split[i]); } } if (contentSplit.length > 0) { textToken.content = contentSplit.join("|"); } return slf.renderToken(tokens, idx, options); } function setupAttachments(md) { md.renderer.rules.link_open = renderAttachment; } function buildCustomMarkdownCookFunction(engineOpts, defaultEngineOpts) { // everything except the engine for opts can just point to the other // opts references, they do not change and we don't need to worry about // mutating them. note that this may need to be updated when additional // opts are added to the pipeline const newOpts = {}; newOpts.allowListed = defaultEngineOpts.allowListed; newOpts.pluginCallbacks = defaultEngineOpts.pluginCallbacks; newOpts.sanitizer = defaultEngineOpts.sanitizer; newOpts.discourse = {}; const featureConfig = cloneJSON(defaultEngineOpts.discourse.features); // everything from the discourse part of defaultEngineOpts can be cloned except // the features, because these can be a limited subset and we don't want to // change the original object reference for (const [key, value] of Object.entries(defaultEngineOpts.discourse)) { if (key !== "features") { newOpts.discourse[key] = value; } } if (engineOpts.featuresOverride !== undefined) { overrideMarkdownFeatures(featureConfig, engineOpts.featuresOverride); } newOpts.discourse.features = featureConfig; const markdownitOpts = { discourse: newOpts.discourse, html: defaultEngineOpts.engine.options.html, breaks: defaultEngineOpts.engine.options.breaks, xhtmlOut: defaultEngineOpts.engine.options.xhtmlOut, linkify: defaultEngineOpts.engine.options.linkify, typographer: defaultEngineOpts.engine.options.typographer, }; newOpts.engine = createMarkdownItEngineWithOpts( markdownitOpts, engineOpts.markdownItRules ); // we have to do this to make sure plugin callbacks, allow list, and helper // functions are all set up correctly for the new engine setupMarkdownEngine(newOpts, featureConfig); // we don't need the whole engine as a consumer, just a cook function // will do return function customRenderFn(contentToRender) { return cook(contentToRender, newOpts); }; } function createMarkdownItEngineWithOpts(markdownitOpts, ruleOverrides) { if (ruleOverrides !== undefined) { // Preset for "zero", https://github.com/markdown-it/markdown-it/blob/master/lib/presets/zero.js return globalThis.markdownit("zero", markdownitOpts).enable(ruleOverrides); } return globalThis.markdownit(markdownitOpts); } function overrideMarkdownFeatures(features, featureOverrides) { if (featureOverrides !== undefined) { Object.keys(features).forEach((feature) => { features[feature] = featureOverrides.includes(feature); }); } } function setupMarkdownEngine(opts, featureConfig) { const quotation_marks = opts.discourse.limitedSiteSettings.markdownTypographerQuotationMarks; if (quotation_marks) { opts.engine.options.quotes = quotation_marks.split("|"); } opts.engine.linkify.tlds( (opts.discourse.limitedSiteSettings.markdownLinkifyTlds || "").split("|") ); setupUrlDecoding(opts.engine); setupHoister(opts.engine); setupImageAndPlayableMediaRenderer(opts.engine); setupAttachments(opts.engine); setupBlockBBCode(opts.engine); setupInlineBBCode(opts.engine); setupTextPostProcessRuler(opts.engine); opts.pluginCallbacks.forEach(([feature, callback]) => { if (featureConfig[feature]) { opts.engine.use(callback); } }); // top level markdown it notifier opts.markdownIt = true; opts.setup = true; if (!opts.discourse.sanitizer || !opts.sanitizer) { const allowLister = new AllowLister(opts.discourse); opts.allowListed.forEach(([feature, info]) => { allowLister.allowListFeature(feature, info); }); opts.sanitizer = opts.discourse.sanitizer = !!opts.discourse.sanitize ? (a) => sanitize(a, allowLister) : (a) => a; } } function unhoistForCooked(hoisted, cooked) { const keys = Object.keys(hoisted); if (keys.length) { let found = true; const unhoist = function (key) { cooked = cooked.replace(new RegExp(key, "g"), function () { found = true; return hoisted[key]; }); }; while (found) { found = false; keys.forEach(unhoist); } } return cooked; } export function extractDataAttribute(str) { let sep = str.indexOf("="); if (sep === -1) { return null; } const key = `data-${str.slice(0, sep)}`.toLowerCase(); if (!/^[A-Za-z]+[\w\-\:\.]*$/.test(key)) { return null; } const value = str.slice(sep + 1); return [key, value]; } let Helpers; export function setup(opts, siteSettings, state) { if (opts.setup) { return; } // we got to require this late cause bundle is not loaded in pretty-text Helpers = Helpers || requirejs("pretty-text/engines/discourse-markdown/helpers"); opts.markdownIt = true; let optionCallbacks = []; let pluginCallbacks = []; let customMarkdownCookFnCallbacks = []; // ideally I would like to change the top level API a bit, but in the mean time this will do let getOptions = { f: () => opts, }; const check = /discourse-markdown\/|markdown-it\//; let features = []; let allowListed = []; // all of the modules under discourse-markdown or markdown-it // directories are considered additional markdown "features" which // may define their own rules Object.keys(require._eak_seen).forEach((entry) => { if (check.test(entry)) { const module = requirejs(entry); if (module && module.setup) { const id = entry.split("/").reverse()[0]; let priority = module.priority || 0; features.unshift({ id, setup: module.setup, priority }); } } }); features .sort((a, b) => a.priority - b.priority) .forEach((markdownFeature) => { markdownFeature.setup( createHelper( markdownFeature.id, opts, optionCallbacks, pluginCallbacks, customMarkdownCookFnCallbacks, getOptions, allowListed ) ); }); Object.entries(state.allowListed || {}).forEach((entry) => { allowListed.push(entry); }); optionCallbacks.forEach(([, callback]) => { callback(opts, siteSettings, state); }); // enable all features by default features.forEach((feature) => { if (!opts.features.hasOwnProperty(feature.id)) { opts.features[feature.id] = true; } }); if (opts.featuresOverride !== undefined) { overrideMarkdownFeatures(opts.features, opts.featuresOverride); } let copy = {}; Object.keys(opts).forEach((entry) => { copy[entry] = opts[entry]; delete opts[entry]; }); copy.helpers = { textReplace: Helpers.textReplace, }; opts.discourse = copy; getOptions.f = () => opts.discourse; opts.discourse.limitedSiteSettings = { secureMedia: siteSettings.secure_media, enableDiffhtmlPreview: siteSettings.enable_diffhtml_preview, traditionalMarkdownLinebreaks: siteSettings.traditional_markdown_linebreaks, enableMarkdownLinkify: siteSettings.enable_markdown_linkify, enableMarkdownTypographer: siteSettings.enable_markdown_typographer, markdownTypographerQuotationMarks: siteSettings.markdown_typographer_quotation_marks, markdownLinkifyTlds: siteSettings.markdown_linkify_tlds, }; const markdownitOpts = { discourse: opts.discourse, html: true, breaks: !opts.discourse.limitedSiteSettings.traditionalMarkdownLinebreaks, xhtmlOut: false, linkify: opts.discourse.limitedSiteSettings.enableMarkdownLinkify, typographer: opts.discourse.limitedSiteSettings.enableMarkdownTypographer, }; opts.engine = createMarkdownItEngineWithOpts( markdownitOpts, opts.discourse.markdownItRules ); opts.pluginCallbacks = pluginCallbacks; opts.allowListed = allowListed; setupMarkdownEngine(opts, opts.discourse.features); customMarkdownCookFnCallbacks.forEach(([, callback]) => { callback(opts, (engineOpts, afterBuild) => afterBuild(buildCustomMarkdownCookFunction(engineOpts, opts)) ); }); } export function cook(raw, opts) { // we still have to hoist html_raw nodes so they bypass the allowlister // this is the case for oneboxes let hoisted = {}; opts.discourse.hoisted = hoisted; const rendered = opts.engine.render(raw); let cooked = opts.discourse.sanitizer(rendered).trim(); cooked = unhoistForCooked(hoisted, cooked); delete opts.discourse.hoisted; return cooked; }