github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 05:47:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: remove markdown-it-bundle and custom build code (#23859)

Browse Source 
With Embroider, we can rely on async `import()` to do the splitting
for us.

This commit extracts from `pretty-text` all the parts that are
meant to be loaded async into a new `discourse-markdown-it` package
that is also a V2 addon (meaning that all files are presumed unused
until they are imported, aka "static").

Mostly I tried to keep the very discourse specific stuff (accessing
site settings and loading plugin features) inside discourse proper,
while the new package aims to have some resembalance of a general
purpose library, a MarkdownIt++ if you will. It is far from perfect
because of how all the "options" stuff work but I think it's a good
start for more refactorings (clearing up the interfaces) to happen
later.

With this, pretty-text and app/lib/text are mostly a kitchen sink
of loosely related text processing utilities.

After the refactor, a lot more code related to setting up the
engine are now loaded lazily, which should be a pretty nice win. I
also noticed that we are currently pulling in the `xss` library at
initial load to power the "sanitize" stuff, but I suspect with a
similar refactoring effort those usages can be removed too. (See
also #23790).

This PR does not attempt to fix the sanitize issue, but I think it
sets things up on the right trajectory for that to happen later.

Co-authored-by: David Taylor <david@taylorhq.com>
This commit is contained in:
Godfrey Chan 2023-11-06 08:59:49 -08:00 committed by GitHub
parent 76b75fae36
commit 9a1695ccc1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 1249 additions and 990 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/assets/javascripts/discourse-markdown-it/addon-main.cjs Normal file
View File

@ -0,0 +1,4 @@
'use strict';
const { addonV1Shim } = require('@embroider/addon-shim');
module.exports = addonV1Shim(__dirname);

45
app/assets/javascripts/discourse-markdown-it/package.json Normal file
View File

@ -0,0 +1,45 @@
{
"name": "discourse-markdown-it",
"version": "1.0.0",
"private": true,
"description": "Discourse's markdown-it features",
"author": "Discourse <team@discourse.org>",
"license": "GPL-2.0-only",
"keywords": [
"ember-addon"
],
"exports": {
".": "./src/index.js",
"./*": "./src/*.js",
"./addon-main.js": "./addon-main.cjs"
},
"files": [
"addon-main.cjs",
"src"
],
"dependencies": {
"@embroider/addon-shim": "^1.0.0",
"discourse-common": "1.0.0",
"ember-auto-import": "^2.6.3",
"markdown-it": "^13.0.2"
},
"peerDependencies": {
"discourse-i18n": "1.0.0",
"pretty-text": "1.0.0",
"xss": "*"
},
"engines": {
"node": "16.* || >= 18",
"npm": "please-use-yarn",
"yarn": ">= 1.21.1"
},
"ember": {
"edition": "octane"
},
"ember-addon": {
"version": 2,
"type": "addon",
"main": "addon-main.cjs",
"app-js": {}
}
}

348
app/assets/javascripts/discourse-markdown-it/src/engine.js Normal file
View File

@ -0,0 +1,348 @@
import markdownit from "markdown-it";
import AllowLister from "pretty-text/allow-lister";
import guid from "pretty-text/guid";
import { sanitize } from "pretty-text/sanitizer";
import { TextPostProcessRuler } from "./features/text-post-process";
// note, this will mutate options due to the way the API is designed
// may need a refactor
export default function makeEngine(
options,
markdownItOptions,
markdownItRules
) {
const engine = makeMarkdownIt(markdownItOptions, markdownItRules);
const quotes =
options.discourse.limitedSiteSettings.markdownTypographerQuotationMarks;
if (quotes) {
engine.options.quotes = quotes.split("|");
}
const tlds = options.discourse.limitedSiteSettings.markdownLinkifyTlds || "";
engine.linkify.tlds(tlds.split("|"));
setupUrlDecoding(engine);
setupHoister(engine);
setupImageAndPlayableMediaRenderer(engine);
setupAttachments(engine);
setupBlockBBCode(engine);
setupInlineBBCode(engine);
setupTextPostProcessRuler(engine);
options.engine = engine;
for (const [feature, callback] of options.pluginCallbacks) {
if (options.discourse.features[feature]) {
if (callback === null || callback === undefined) {
// eslint-disable-next-line no-console
console.log("BAD MARKDOWN CALLBACK FOUND");
// eslint-disable-next-line no-console
console.log(`FEATURE IS: ${feature}`);
}
engine.use(callback);
}
}
// top level markdown it notifier
options.markdownIt = true;
options.setup = true;
if (!options.discourse.sanitizer || !options.sanitizer) {
const allowLister = new AllowLister(options.discourse);
options.allowListed.forEach(([feature, info]) => {
allowLister.allowListFeature(feature, info);
});
options.sanitizer = options.discourse.sanitizer = !!options.discourse
.sanitize
? (a) => sanitize(a, allowLister)
: (a) => a;
}
}
export function cook(raw, options) {
// we still have to hoist html_raw nodes so they bypass the allowlister
// this is the case for oneboxes and also certain plugins that require
// raw HTML rendering within markdown bbcode rules
options.discourse.hoisted ??= {};
const rendered = options.engine.render(raw);
let cooked = options.discourse.sanitizer(rendered).trim();
// opts.discourse.hoisted guid keys will be deleted within here to
// keep the object empty
cooked = unhoistForCooked(options.discourse.hoisted, cooked);
return cooked;
}
function makeMarkdownIt(markdownItOptions, markdownItRules) {
if (markdownItRules) {
// Preset for "zero", https://github.com/markdown-it/markdown-it/blob/master/lib/presets/zero.js
return markdownit("zero", markdownItOptions).enable(markdownItRules);
} else {
return markdownit(markdownItOptions);
}
}
function setupUrlDecoding(engine) {
// this fixed a subtle issue where %20 is decoded as space in
// automatic urls
engine.utils.lib.mdurl.decode.defaultChars = ";/?:@&=+$,# ";
}
// hoists html_raw tokens out of the render flow and replaces them
// with a GUID. this GUID is then replaced with the final raw HTML
// content in unhoistForCooked
function renderHoisted(tokens, idx, options) {
const content = tokens[idx].content;
if (content && content.length > 0) {
let id = guid();
options.discourse.hoisted[id] = content;
return id;
} else {
return "";
}
}
function unhoistForCooked(hoisted, cooked) {
const keys = Object.keys(hoisted);
if (keys.length) {
let found = true;
const unhoist = function (key) {
cooked = cooked.replace(new RegExp(key, "g"), function () {
found = true;
return hoisted[key];
});
delete hoisted[key];
};
while (found) {
found = false;
keys.forEach(unhoist);
}
}
return cooked;
}
// html_raw tokens, funnily enough, render raw HTML via renderHoisted and
// unhoistForCooked
function setupHoister(engine) {
engine.renderer.rules.html_raw = renderHoisted;
}
// exported for test only
export function extractDataAttribute(str) {
let sep = str.indexOf("=");
if (sep === -1) {
return null;
}
const key = `data-${str.slice(0, sep)}`.toLowerCase();
if (!/^[A-Za-z]+[\w\-\:\.]*$/.test(key)) {
return null;
}
const value = str.slice(sep + 1);
return [key, value];
}
// videoHTML and audioHTML follow the same HTML syntax
// as oneboxer.rb when dealing with these formats
function videoHTML(token) {
const src = token.attrGet("src");
const origSrc = token.attrGet("data-orig-src");
const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
return `<div class="video-placeholder-container" data-video-src="${src}" ${dataOrigSrcAttr}>
</div>`;
}
function audioHTML(token) {
const src = token.attrGet("src");
const origSrc = token.attrGet("data-orig-src");
const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
return `<audio preload="metadata" controls>
<source src="${src}" ${dataOrigSrcAttr}>
<a href="${src}">${src}</a>
</audio>`;
}
const IMG_SIZE_REGEX =
/^([1-9]+[0-9]*)x([1-9]+[0-9]*)(\s*,\s*(x?)([1-9][0-9]{0,2}?)([%x]?))?$/;
function renderImageOrPlayableMedia(tokens, idx, options, env, slf) {
const token = tokens[idx];
const alt = slf.renderInlineAsText(token.children, options, env);
const split = alt.split("|");
const altSplit = [split[0]];
// markdown-it supports returning HTML instead of continuing to render the current token
// see https://github.com/markdown-it/markdown-it/blob/master/docs/architecture.md#renderer
// handles |video and |audio alt transformations for image tags
if (split[1] === "video") {
if (
options.discourse.previewing &&
!options.discourse.limitedSiteSettings.enableDiffhtmlPreview
) {
return `<div class="onebox-placeholder-container">
<span class="placeholder-icon video"></span>
</div>`;
} else {
return videoHTML(token);
}
} else if (split[1] === "audio") {
return audioHTML(token);
}
// parsing ![myimage|500x300]() or ![myimage|75%]() or ![myimage|500x300, 75%]
for (let i = 1, match, data; i < split.length; ++i) {
if ((match = split[i].match(IMG_SIZE_REGEX)) && match[1] && match[2]) {
let width = match[1];
let height = match[2];
// calculate using percentage
if (match[5] && match[6] && match[6] === "%") {
let percent = parseFloat(match[5]) / 100.0;
width = parseInt(width * percent, 10);
height = parseInt(height * percent, 10);
}
// calculate using only given width
if (match[5] && match[6] && match[6] === "x") {
let wr = parseFloat(match[5]) / width;
width = parseInt(match[5], 10);
height = parseInt(height * wr, 10);
}
// calculate using only given height
if (match[5] && match[4] && match[4] === "x" && !match[6]) {
let hr = parseFloat(match[5]) / height;
height = parseInt(match[5], 10);
width = parseInt(width * hr, 10);
}
if (token.attrIndex("width") === -1) {
token.attrs.push(["width", width]);
}
if (token.attrIndex("height") === -1) {
token.attrs.push(["height", height]);
}
if (
options.discourse.previewing &&
match[6] !== "x" &&
match[4] !== "x"
) {
token.attrs.push(["class", "resizable"]);
}
} else if ((data = extractDataAttribute(split[i]))) {
token.attrs.push(data);
} else if (split[i] === "thumbnail") {
token.attrs.push(["data-thumbnail", "true"]);
} else {
altSplit.push(split[i]);
}
}
const altValue = altSplit.join("|").trim();
if (altValue === "") {
token.attrSet("role", "presentation");
} else {
token.attrSet("alt", altValue);
}
return slf.renderToken(tokens, idx, options);
}
// we have taken over the ![]() syntax in markdown to
// be able to render a video or audio URL as well as the
// image using |video and |audio in the text inside []
function setupImageAndPlayableMediaRenderer(engine) {
engine.renderer.rules.image = renderImageOrPlayableMedia;
}
// discourse-encrypt wants this?
export const ATTACHMENT_CSS_CLASS = "attachment";
function renderAttachment(tokens, idx, options, env, slf) {
const linkToken = tokens[idx];
const textToken = tokens[idx + 1];
const split = textToken.content.split("|");
const contentSplit = [];
for (let i = 0, data; i < split.length; ++i) {
if (split[i] === ATTACHMENT_CSS_CLASS) {
linkToken.attrs.unshift(["class", split[i]]);
} else if ((data = extractDataAttribute(split[i]))) {
linkToken.attrs.push(data);
} else {
contentSplit.push(split[i]);
}
}
if (contentSplit.length > 0) {
textToken.content = contentSplit.join("|");
}
return slf.renderToken(tokens, idx, options);
}
function setupAttachments(engine) {
engine.renderer.rules.link_open = renderAttachment;
}
// TODO we may just use a proper ruler from markdown it... this is a basic proxy
class Ruler {
constructor() {
this.rules = [];
}
getRules() {
return this.rules;
}
getRuleForTag(tag) {
this.ensureCache();
if (this.cache.hasOwnProperty(tag)) {
return this.cache[tag];
}
}
ensureCache() {
if (this.cache) {
return;
}
this.cache = {};
for (let i = this.rules.length - 1; i >= 0; i--) {
let info = this.rules[i];
this.cache[info.rule.tag] = info;
}
}
push(name, rule) {
this.rules.push({ name, rule });
this.cache = null;
}
}
// block bb code ruler for parsing of quotes / code / polls
function setupBlockBBCode(engine) {
engine.block.bbcode = { ruler: new Ruler() };
}
// inline bbcode ruler for parsing of spoiler tags, discourse-chart etc
function setupInlineBBCode(engine) {
engine.inline.bbcode = { ruler: new Ruler() };
}
// rule for text replacement via regex, used for @mentions, category hashtags, etc.
function setupTextPostProcessRuler(engine) {
engine.core.textPostProcess = { ruler: new TextPostProcessRuler() };
}

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/anchor.js → app/assets/javascripts/discourse-markdown-it/src/features/anchor.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-block.js → app/assets/javascripts/discourse-markdown-it/src/features/bbcode-block.js
View File

2
app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js → app/assets/javascripts/discourse-markdown-it/src/features/bbcode-inline.js
View File

@ -1,4 +1,4 @@
import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
import { parseBBCodeTag } from "./bbcode-block";
function tokenizeBBCode(state, silent, ruler) {
let pos = state.pos;

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js → app/assets/javascripts/discourse-markdown-it/src/features/censored.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js → app/assets/javascripts/discourse-markdown-it/src/features/code.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/custom-typographer-replacements.js → app/assets/javascripts/discourse-markdown-it/src/features/custom-typographer-replacements.js
View File

2
app/assets/javascripts/pretty-text/engines/discourse-markdown/d-wrap.js → app/assets/javascripts/discourse-markdown-it/src/features/d-wrap.js
View File

@ -1,4 +1,4 @@
import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
import { parseBBCodeTag } from "./bbcode-block";
const WRAP_CLASS = "d-wrap";

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js → app/assets/javascripts/discourse-markdown-it/src/features/emoji.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/hashtag-autocomplete.js → app/assets/javascripts/discourse-markdown-it/src/features/hashtag-autocomplete.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/html-img.js → app/assets/javascripts/discourse-markdown-it/src/features/html-img.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/image-controls.js → app/assets/javascripts/discourse-markdown-it/src/features/image-controls.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/image-grid.js → app/assets/javascripts/discourse-markdown-it/src/features/image-grid.js
View File

49
app/assets/javascripts/discourse-markdown-it/src/features/index.js Normal file
View File

@ -0,0 +1,49 @@
import * as anchor from "./anchor";
import * as bbcodeBlock from "./bbcode-block";
import * as bbcodeInline from "./bbcode-inline";
import * as censored from "./censored";
import * as code from "./code";
import * as customTypographerReplacements from "./custom-typographer-replacements";
import * as dWrap from "./d-wrap";
import * as emoji from "./emoji";
import * as hashtagAutocomplete from "./hashtag-autocomplete";
import * as htmlImg from "./html-img";
import * as imageControls from "./image-controls";
import * as imageGrid from "./image-grid";
import * as mentions from "./mentions";
import * as newline from "./newline";
import * as onebox from "./onebox";
import * as paragraph from "./paragraph";
import * as quotes from "./quotes";
import * as table from "./table";
import * as textPostProcess from "./text-post-process";
import * as uploadProtocol from "./upload-protocol";
import * as watchedWords from "./watched-words";
export default [
feature("watched-words", watchedWords),
feature("upload-protocol", uploadProtocol),
feature("text-post-process", textPostProcess),
feature("table", table),
feature("quotes", quotes),
feature("paragraph", paragraph),
feature("onebox", onebox),
feature("newline", newline),
feature("mentions", mentions),
feature("image-grid", imageGrid),
feature("image-controls", imageControls),
feature("html-img", htmlImg),
feature("hashtag-autocomplete", hashtagAutocomplete),
feature("emoji", emoji),
feature("d-wrap", dWrap),
feature("custom-typographer-replacements", customTypographerReplacements),
feature("code", code),
feature("censored", censored),
feature("bbcode-inline", bbcodeInline),
feature("bbcode-block", bbcodeBlock),
feature("anchor", anchor),
];
function feature(id, { setup, priority = 0 }) {
return { id, setup, priority };
}

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js → app/assets/javascripts/discourse-markdown-it/src/features/mentions.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js → app/assets/javascripts/discourse-markdown-it/src/features/newline.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js → app/assets/javascripts/discourse-markdown-it/src/features/onebox.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/paragraph.js → app/assets/javascripts/discourse-markdown-it/src/features/paragraph.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js → app/assets/javascripts/discourse-markdown-it/src/features/quotes.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js → app/assets/javascripts/discourse-markdown-it/src/features/table.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/text-post-process.js → app/assets/javascripts/discourse-markdown-it/src/features/text-post-process.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/upload-protocol.js → app/assets/javascripts/discourse-markdown-it/src/features/upload-protocol.js
View File

0
app/assets/javascripts/pretty-text/engines/discourse-markdown/watched-words.js → app/assets/javascripts/discourse-markdown-it/src/features/watched-words.js
View File

74
app/assets/javascripts/discourse-markdown-it/src/index.js Normal file
View File

@ -0,0 +1,74 @@
import { cook as cookIt } from "./engine";
import DEFAULT_FEATURES from "./features";
import buildOptions from "./options";
import setup from "./setup";
function NOOP(ident) {
return ident;
}
export default class DiscourseMarkdownIt {
static withDefaultFeatures() {
return this.withFeatures(DEFAULT_FEATURES);
}
static withCustomFeatures(features) {
return this.withFeatures([...DEFAULT_FEATURES, ...features]);
}
static withFeatures(features) {
const withOptions = (options) => this.withOptions(features, options);
return { withOptions };
}
static withOptions(features, rawOptions) {
const { options, siteSettings, state } = buildOptions(rawOptions);
// note, this will mutate options due to the way the API is designed
// may need a refactor
setup(features, options, siteSettings, state);
return new DiscourseMarkdownIt(options);
}
static minimal() {
return this.withFeatures([]).withOptions({ siteSettings: {} });
}
constructor(options) {
if (!options.setup) {
throw new Error(
"Cannot construct DiscourseMarkdownIt from raw options, " +
"use DiscourseMarkdownIt.withOptions() instead"
);
}
this.options = options;
}
disableSanitizer() {
this.options.sanitizer = this.options.discourse.sanitizer = NOOP;
}
cook(raw) {
if (!raw || raw.length === 0) {
return "";
}
let result;
result = cookIt(raw, this.options);
return result ? result : "";
}
parse(markdown, env = {}) {
return this.options.engine.parse(markdown, env);
}
sanitize(html) {
return this.options.sanitizer(html).trim();
}
get linkify() {
return this.options.engine.linkify;
}
}

83
app/assets/javascripts/discourse-markdown-it/src/options.js Normal file
View File

@ -0,0 +1,83 @@
import { deepMerge } from "discourse-common/lib/object";
// the options are passed here and must be explicitly allowed with
// the const options & state below
export default function buildOptions(state) {
const {
siteSettings,
getURL,
lookupAvatar,
lookupPrimaryUserGroup,
getTopicInfo,
topicId,
forceQuoteLink,
userId,
getCurrentUser,
currentUser,
lookupAvatarByPostNumber,
lookupPrimaryUserGroupByPostNumber,
formatUsername,
emojiUnicodeReplacer,
lookupUploadUrls,
previewing,
censoredRegexp,
disableEmojis,
customEmojiTranslation,
watchedWordsReplace,
watchedWordsLink,
emojiDenyList,
featuresOverride,
markdownItRules,
additionalOptions,
hashtagTypesInPriorityOrder,
hashtagIcons,
hashtagLookup,
} = state;
let features = {};
if (state.features) {
features = deepMerge(features, state.features);
}
const options = {
sanitize: true,
getURL,
features,
lookupAvatar,
lookupPrimaryUserGroup,
getTopicInfo,
topicId,
forceQuoteLink,
userId,
getCurrentUser,
currentUser,
lookupAvatarByPostNumber,
lookupPrimaryUserGroupByPostNumber,
formatUsername,
emojiUnicodeReplacer,
lookupUploadUrls,
censoredRegexp,
customEmojiTranslation,
allowedHrefSchemes: siteSettings.allowed_href_schemes
? siteSettings.allowed_href_schemes.split("|")
: null,
allowedIframes: siteSettings.allowed_iframes
? siteSettings.allowed_iframes.split("|")
: [],
markdownIt: true,
previewing,
disableEmojis,
watchedWordsReplace,
watchedWordsLink,
emojiDenyList,
featuresOverride,
markdownItRules,
additionalOptions,
hashtagTypesInPriorityOrder,
hashtagIcons,
hashtagLookup,
};
return { options, siteSettings, state };
}

359
app/assets/javascripts/discourse-markdown-it/src/setup.js Normal file
View File

@ -0,0 +1,359 @@
import { textReplace } from "pretty-text/text-replace";
import deprecated from "discourse-common/lib/deprecated";
import { cloneJSON } from "discourse-common/lib/object";
import makeEngine, { cook } from "./engine";
// note, this will mutate options due to the way the API is designed
// may need a refactor
export default function setupIt(features, options, siteSettings, state) {
Setup.run(features, options, siteSettings, state);
}
class Setup {
static run(features, options, siteSettings, state) {
if (options.setup) {
// Already setup
return;
}
const setup = new Setup(options);
features.sort((a, b) => a.priority - b.priority);
for (const feature of features) {
setup.#setupFeature(feature.id, feature.setup);
}
for (const entry of Object.entries(state.allowListed ?? {})) {
setup.allowList(entry);
}
setup.#runOptionsCallbacks(siteSettings, state);
setup.#enableMarkdownFeatures();
setup.#finalizeGetOptions(siteSettings);
setup.#makeEngine();
setup.#buildCookFunctions();
}
#context;
#options;
#allowListed = [];
#customMarkdownCookFunctionCallbacks = [];
#loadedFeatures = [];
#optionCallbacks = [];
#pluginCallbacks = [];
constructor(options) {
options.markdownIt = true;
this.#options = options;
// hack to allow moving of getOptions – see #finalizeGetOptions
this.#context = { options };
}
allowList(entry) {
this.#allowListed.push(entry);
}
registerOptions(entry) {
this.#optionCallbacks.push(entry);
}
registerPlugin(entry) {
this.#pluginCallbacks.push(entry);
}
buildCookFunction(entry) {
this.#customMarkdownCookFunctionCallbacks.push(entry);
}
#setupFeature(featureName, callback) {
// When we provide the API object to the setup callback, we expect them to
// make use of it synchronously. However, it is possible that the could
// close over the API object, intentionally or unintentionally, and cause
// memory leaks or unexpectedly call API methods at a later time with
// unpredictable results. This make sure to "gut" the API object after the
// callback is executed so that it cannot leak memory or be used later.
let loaned = this;
const doSetup = (methodName, ...args) => {
if (loaned === null) {
throw new Error(
`${featureName}: ${methodName} can only be called during setup()!`
);
}
if (loaned[methodName]) {
return loaned[methodName](...args);
}
};
callback(new API(featureName, this.#context, doSetup));
this.#loadedFeatures.push(featureName);
// revoke access to the Setup object
loaned = null;
}
#runOptionsCallbacks(siteSettings, state) {
this.#drain(this.#optionCallbacks, ([, callback]) =>
callback(this.#options, siteSettings, state)
);
}
#enableMarkdownFeatures({ features, featuresOverride } = this.#options) {
// TODO: `options.features` could in theory contain additional keys for
// features that aren't loaded. The way the previous code was written
// incidentally means we would iterate over a super set of both. To be
// pedantic we kept that behavior here, but I'm not sure if that's really
// necessary.
const allFeatures = new Set([
...this.#drain(this.#loadedFeatures),
...Object.keys(features),
]);
if (featuresOverride) {
for (const feature of allFeatures) {
features[feature] = featuresOverride.includes(feature);
}
} else {
// enable all features by default
for (let feature of allFeatures) {
features[feature] ??= true;
}
}
}
#finalizeGetOptions(siteSettings) {
// This is weird but essentially we want to remove `options.*` in-place
// into `options.discourse.*`, then, we want to change `context.options`
// to point at `options.discourse`. This ensures features that held onto
// the API object during setup will continue to get the right stuff when
// they call `getOptions()`.
const options = this.#options;
const discourse = {};
for (const [key, value] of Object.entries(options)) {
discourse[key] = value;
delete options[key];
}
discourse.helpers = { textReplace };
discourse.limitedSiteSettings = {
secureUploads: siteSettings.secure_uploads,
enableDiffhtmlPreview: siteSettings.enable_diffhtml_preview,
traditionalMarkdownLinebreaks:
siteSettings.traditional_markdown_linebreaks,
enableMarkdownLinkify: siteSettings.enable_markdown_linkify,
enableMarkdownTypographer: siteSettings.enable_markdown_typographer,
markdownTypographerQuotationMarks:
siteSettings.markdown_typographer_quotation_marks,
markdownLinkifyTlds: siteSettings.markdown_linkify_tlds,
};
this.#context.options = options.discourse = discourse;
}
#makeEngine() {
const options = this.#options;
const { discourse } = options;
const { markdownItRules, limitedSiteSettings } = discourse;
const {
enableMarkdownLinkify,
enableMarkdownTypographer,
traditionalMarkdownLinebreaks,
} = limitedSiteSettings;
options.allowListed = this.#drain(this.#allowListed);
options.pluginCallbacks = this.#drain(this.#pluginCallbacks);
const markdownItOptions = {
discourse,
html: true,
breaks: !traditionalMarkdownLinebreaks,
xhtmlOut: false,
linkify: enableMarkdownLinkify,
typographer: enableMarkdownTypographer,
};
makeEngine(options, markdownItOptions, markdownItRules);
}
#buildCookFunctions() {
const options = this.#options;
// the callback argument we pass to the callbacks
let callbackArg = (engineOptions, afterBuild) =>
afterBuild(this.#buildCookFunction(engineOptions, options));
this.#drain(this.#customMarkdownCookFunctionCallbacks, ([, callback]) => {
callback(options, callbackArg);
});
}
#buildCookFunction(engineOptions, defaultOptions) {
// everything except the engine for opts can just point to the other
// opts references, they do not change and we don't need to worry about
// mutating them. note that this may need to be updated when additional
// opts are added to the pipeline
const options = {};
options.allowListed = defaultOptions.allowListed;
options.pluginCallbacks = defaultOptions.pluginCallbacks;
options.sanitizer = defaultOptions.sanitizer;
// everything from the discourse part of defaultOptions can be cloned except
// the features, because these can be a limited subset and we don't want to
// change the original object reference
const features = cloneJSON(defaultOptions.discourse.features);
options.discourse = {
...defaultOptions.discourse,
features,
};
this.#enableMarkdownFeatures({
features,
featuresOverride: engineOptions.featuresOverride,
});
const markdownItOptions = {
discourse: options.discourse,
html: defaultOptions.engine.options.html,
breaks: defaultOptions.engine.options.breaks,
xhtmlOut: defaultOptions.engine.options.xhtmlOut,
linkify: defaultOptions.engine.options.linkify,
typographer: defaultOptions.engine.options.typographer,
};
makeEngine(options, markdownItOptions, engineOptions.markdownItRules);
return function customCookFunction(raw) {
return cook(raw, options);
};
}
#drain(items, callback) {
if (callback) {
let item = items.shift();
while (item) {
callback(item);
item = items.shift();
}
} else {
const cloned = [...items];
items.length = 0;
return cloned;
}
}
}
class API {
#name;
#context;
#setup;
#deprecate;
constructor(featureName, context, setup) {
this.#name = featureName;
this.#context = context;
this.#setup = setup;
this.#deprecate = (methodName, ...args) => {
if (window.console && window.console.log) {
window.console.log(
featureName +
": " +
methodName +
" is deprecated, please use the new markdown it APIs"
);
}
return setup(methodName, ...args);
};
}
get markdownIt() {
return true;
}
// this the only method we expect to be called post-setup()
getOptions() {
return this.#context.options;
}
allowList(info) {
this.#setup("allowList", [this.#name, info]);
}
whiteList(info) {
deprecated("`whiteList` has been replaced with `allowList`", {
since: "2.6.0.beta.4",
dropFrom: "2.7.0",
id: "discourse.markdown-it.whitelist",
});
this.allowList(info);
}
registerOptions(callback) {
this.#setup("registerOptions", [this.#name, callback]);
}
registerPlugin(callback) {
this.#setup("registerPlugin", [this.#name, callback]);
}
buildCookFunction(callback) {
this.#setup("buildCookFunction", [this.#name, callback]);
}
// deprecate methods – "deprecate" is a bit of a misnomer here since the
// methods don't actually do anything anymore
registerInline() {
this.#deprecate("registerInline");
}
replaceBlock() {
this.#deprecate("replaceBlock");
}
addPreProcessor() {
this.#deprecate("addPreProcessor");
}
inlineReplace() {
this.#deprecate("inlineReplace");
}
postProcessTag() {
this.#deprecate("postProcessTag");
}
inlineRegexp() {
this.#deprecate("inlineRegexp");
}
inlineBetween() {
this.#deprecate("inlineBetween");
}
postProcessText() {
this.#deprecate("postProcessText");
}
onParseNode() {
this.#deprecate("onParseNode");
}
registerBlock() {
this.#deprecate("registerBlock");
}
}

97
app/assets/javascripts/discourse/app/lib/text.js
View File

@ -1,44 +1,17 @@
import { htmlSafe } from "@ember/template";
import AllowLister from "pretty-text/allow-lister";
import { buildEmojiUrl, performEmojiUnescape } from "pretty-text/emoji";
import PrettyText, { buildOptions } from "pretty-text/pretty-text";
import { sanitize as textSanitize } from "pretty-text/sanitizer";
import { Promise } from "rsvp";
import loadScript from "discourse/lib/load-script";
import { MentionsParser } from "discourse/lib/mentions-parser";
import { formatUsername } from "discourse/lib/utilities";
import Session from "discourse/models/session";
import deprecated from "discourse-common/lib/deprecated";
import { getURLWithCDN } from "discourse-common/lib/get-url";
import { helperContext } from "discourse-common/lib/helpers";
function getOpts(opts) {
let context = helperContext();
opts = Object.assign(
{
getURL: getURLWithCDN,
currentUser: context.currentUser,
censoredRegexp: context.site.censored_regexp,
customEmojiTranslation: context.site.custom_emoji_translation,
emojiDenyList: context.site.denied_emojis,
siteSettings: context.siteSettings,
formatUsername,
watchedWordsReplace: context.site.watched_words_replace,
watchedWordsLink: context.site.watched_words_link,
additionalOptions: context.site.markdown_additional_options,
},
opts
);
return buildOptions(opts);
async function withEngine(name, ...args) {
const engine = await import("discourse/static/markdown-it");
return engine[name](...args);
}
export function cook(text, options) {
return loadMarkdownIt().then(() => {
const cooked = createPrettyText(options).cook(text);
return htmlSafe(cooked);
});
export async function cook(text, options) {
return await withEngine("cook", text, options);
}
// todo drop this function after migrating everything to cook()
@ -48,66 +21,38 @@ export function cookAsync(text, options) {
dropFrom: "3.2.0.beta5",
id: "discourse.text.cook-async",
});
return cook(text, options);
}
// Warm up pretty text with a set of options and return a function
// which can be used to cook without rebuilding pretty-text every time
export function generateCookFunction(options) {
return loadMarkdownIt().then(() => {
const prettyText = createPrettyText(options);
return (text) => prettyText.cook(text);
});
// Warm up the engine with a set of options and return a function
// which can be used to cook without rebuilding the engine every time
export async function generateCookFunction(options) {
return await withEngine("generateCookFunction", options);
}
export function generateLinkifyFunction(options) {
return loadMarkdownIt().then(() => {
const prettyText = createPrettyText(options);
return prettyText.opts.engine.linkify;
});
export async function generateLinkifyFunction(options) {
return await withEngine("generateLinkifyFunction", options);
}
// TODO: this one is special, it attempts to do something even without
// the engine loaded. Currently, this is what is forcing the xss library
// to be included on initial page load. The API/behavior also seems a bit
// different than the async version.
export function sanitize(text, options) {
return textSanitize(text, new AllowLister(options));
}
export function sanitizeAsync(text, options) {
return loadMarkdownIt().then(() => {
return createPrettyText(options).sanitize(text);
});
export async function sanitizeAsync(text, options) {
return await withEngine("sanitize", text, options);
}
export function parseAsync(md, options = {}, env = {}) {
return loadMarkdownIt().then(() => {
return createPrettyText(options).parse(md, env);
});
export async function parseAsync(md, options = {}, env = {}) {
return await withEngine("parse", md, options, env);
}
export async function parseMentions(markdown, options) {
await loadMarkdownIt();
const prettyText = createPrettyText(options);
const mentionsParser = new MentionsParser(prettyText);
return mentionsParser.parse(markdown);
}
function loadMarkdownIt() {
return new Promise((resolve) => {
let markdownItURL = Session.currentProp("markdownItURL");
if (markdownItURL) {
loadScript(markdownItURL)
.then(() => resolve())
.catch((e) => {
// eslint-disable-next-line no-console
console.error(e);
});
} else {
resolve();
}
});
}
function createPrettyText(options) {
return new PrettyText(getOpts(options));
return await withEngine("parseMentions", markdown, options);
}
function emojiOptions() {

25
app/assets/javascripts/discourse/app/static/markdown-it/features.js Normal file
View File

@ -0,0 +1,25 @@
export default function loadPluginFeatures() {
const features = [];
for (let moduleName of Object.keys(requirejs.entries)) {
if (moduleName.startsWith("discourse/plugins/")) {
// all of the modules under discourse-markdown or markdown-it
// directories are considered additional markdown "features" which
// may define their own rules
if (
moduleName.includes("/discourse-markdown/") ||
moduleName.includes("/markdown-it/")
) {
const module = requirejs(moduleName);
if (module && module.setup) {
const id = moduleName.split("/").reverse()[0];
const { setup, priority = 0 } = module;
features.unshift({ id, setup, priority });
}
}
}
}
return features;
}

60
app/assets/javascripts/discourse/app/static/markdown-it/index.js Normal file
View File

@ -0,0 +1,60 @@
import { htmlSafe } from "@ember/template";
import { importSync } from "@embroider/macros";
import loaderShim from "discourse-common/lib/loader-shim";
import DiscourseMarkdownIt from "discourse-markdown-it";
import loadPluginFeatures from "./features";
import MentionsParser from "./mentions-parser";
import buildOptions from "./options";
// Shims the `parseBBCodeTag` utility function back to its old location. For
// now, there is no deprecation with this as we don't have a new location for
// them to import from (well, we do, but we don't want to expose the new code
// to loader.js and we want to make sure the code is loaded lazily).
//
// TODO: find a new home for this – the code is rather small so we could just
// throw it into the synchronous pretty-text package and call it good, but we
// should probably look into why plugins are needing to call this utility in
// the first place, and provide better infrastructure for registering bbcode
// additions instead.
loaderShim("pretty-text/engines/discourse-markdown/bbcode-block", () =>
importSync("./parse-bbcode-tag")
);
function buildEngine(options) {
return DiscourseMarkdownIt.withCustomFeatures(
loadPluginFeatures()
).withOptions(buildOptions(options));
}
// Use this to easily create an instance with proper options
export function cook(text, options) {
return htmlSafe(buildEngine(options).cook(text));
}
// Warm up the engine with a set of options and return a function
// which can be used to cook without rebuilding the engine every time
export function generateCookFunction(options) {
const engine = buildEngine(options);
return (text) => engine.cook(text);
}
export function generateLinkifyFunction(options) {
const engine = buildEngine(options);
return engine.linkify;
}
export function sanitize(text, options) {
const engine = buildEngine(options);
return engine.sanitize(text);
}
export function parse(md, options = {}, env = {}) {
const engine = buildEngine(options);
return engine.parse(md, env);
}
export function parseMentions(markdown, options) {
const engine = buildEngine(options);
const mentionsParser = new MentionsParser(engine);
return mentionsParser.parse(markdown);
}

8
app/assets/javascripts/discourse/app/lib/mentions-parser.js → app/assets/javascripts/discourse/app/static/markdown-it/mentions-parser.js
View File

@ -1,10 +1,10 @@
export class MentionsParser {
constructor(prettyText) {
this.prettyText = prettyText;
export default class MentionsParser {
constructor(engine) {
this.engine = engine;
}
parse(markdown) {
const tokens = this.prettyText.parse(markdown);
const tokens = this.engine.parse(markdown);
const mentions = this.#parse(tokens);
return [...new Set(mentions)];
}

21
app/assets/javascripts/discourse/app/static/markdown-it/options.js Normal file
View File

@ -0,0 +1,21 @@
import { formatUsername } from "discourse/lib/utilities";
import { getURLWithCDN } from "discourse-common/lib/get-url";
import { helperContext } from "discourse-common/lib/helpers";
export default function buildOptions(options) {
let context = helperContext();
return {
getURL: getURLWithCDN,
currentUser: context.currentUser,
censoredRegexp: context.site.censored_regexp,
customEmojiTranslation: context.site.custom_emoji_translation,
emojiDenyList: context.site.denied_emojis,
siteSettings: context.siteSettings,
formatUsername,
watchedWordsReplace: context.site.watched_words_replace,
watchedWordsLink: context.site.watched_words_link,
additionalOptions: context.site.markdown_additional_options,
...options,
};
}

1
app/assets/javascripts/discourse/app/static/markdown-it/parse-bbcode-tag.js Normal file
View File

@ -0,0 +1 @@
export { parseBBCodeTag } from "discourse-markdown-it/features/bbcode-block";

14
app/assets/javascripts/discourse/ember-cli-build.js
View File

@ -93,10 +93,6 @@ module.exports = function (defaults) {
const wizardTree = app.project.findAddonByName("wizard").treeForAddonBundle();
const markdownItBundleTree = app.project
.findAddonByName("pretty-text")
.treeForMarkdownItBundle();
const testStylesheetTree = mergeTrees([
discourseScss(`${discourseRoot}/app/assets/stylesheets`, "qunit.scss"),
discourseScss(
@ -126,19 +122,19 @@ module.exports = function (defaults) {
inputFiles: ["**/*.js"],
outputFile: `assets/wizard.js`,
}),
concat(markdownItBundleTree, {
inputFiles: ["**/*.js"],
outputFile: `assets/markdown-it-bundle.js`,
}),
generateScriptsTree(app),
discoursePluginsTree,
testStylesheetTree,
];
const appTree = compatBuild(app, Webpack, {
staticAppPaths: ["static"],
packagerOptions: {
webpackConfig: {
devtool: "source-map",
output: {
publicPath: "auto",
},
externals: [
function ({ request }, callback) {
if (
@ -147,8 +143,6 @@ module.exports = function (defaults) {
(request === "jquery" ||
request.startsWith("admin/") ||
request.startsWith("wizard/") ||
(request.startsWith("pretty-text/engines/") &&
request !== "pretty-text/engines/discourse-markdown-it") ||
request.startsWith("discourse/plugins/") ||
request.startsWith("discourse/theme-"))
) {

1
app/assets/javascripts/discourse/package.json
View File

@ -84,6 +84,7 @@
"dialog-holder": "1.0.0",
"discourse-common": "1.0.0",
"discourse-i18n": "1.0.0",
"discourse-markdown-it": "1.0.0",
"discourse-plugins": "1.0.0",
"ember-auto-import": "^2.6.3",
"ember-buffered-proxy": "^2.1.1",

1
app/assets/javascripts/discourse/tests/index.html
View File

@ -61,7 +61,6 @@
<script src="{{rootURL}}assets/test-i18n.js" data-embroider-ignore></script>
<script src="{{rootURL}}assets/test-site-settings.js" data-embroider-ignore></script>
<script src="{{rootURL}}assets/markdown-it-bundle.js" data-embroider-ignore></script>
<script src="{{rootURL}}assets/admin.js" data-embroider-ignore></script>
<script src="{{rootURL}}assets/wizard.js" data-embroider-ignore></script>

2
app/assets/javascripts/discourse/tests/test-helper.js
View File

@ -1 +1 @@
// We don't currently use this file, but it is require'd by ember-cli's test bundle
import "discourse/static/markdown-it";

4
app/assets/javascripts/discourse/tests/unit/lib/build-quote-test.js
View File

@ -1,8 +1,8 @@
import { getOwner } from "@ember/application";
import { setupTest } from "ember-qunit";
import PrettyText from "pretty-text/pretty-text";
import { module, test } from "qunit";
import { buildQuote } from "discourse/lib/quote";
import DiscourseMarkdownIt from "discourse-markdown-it";
module("Unit | Utility | build-quote", function (hooks) {
setupTest(hooks);
@ -60,7 +60,7 @@ module("Unit | Utility | build-quote", function (hooks) {
test("quoting a quote", function (assert) {
const store = getOwner(this).lookup("service:store");
const post = store.createRecord("post", {
cooked: new PrettyText().cook(
cooked: DiscourseMarkdownIt.minimal().cook(
'[quote="sam, post:1, topic:1, full:true"]\nhello\n[/quote]\n*Test*'
),
username: "eviltrout",

2
app/assets/javascripts/discourse/tests/unit/lib/parse-bbcode-tag-test.js
View File

@ -1,6 +1,6 @@
import { setupTest } from "ember-qunit";
import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
import { module, test } from "qunit";
import { parseBBCodeTag } from "discourse-markdown-it/features/bbcode-block";
module("Unit | Utility | parseBBCodeTag", function (hooks) {
setupTest(hooks);

20
app/assets/javascripts/discourse/tests/unit/lib/pretty-text-test.js
View File

@ -1,14 +1,14 @@
import { setupTest } from "ember-qunit";
import { registerEmoji } from "pretty-text/emoji";
import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
import { extractDataAttribute } from "pretty-text/engines/discourse-markdown-it";
import {
applyCachedInlineOnebox,
deleteCachedInlineOnebox,
} from "pretty-text/inline-oneboxer";
import PrettyText, { buildOptions } from "pretty-text/pretty-text";
import QUnit, { module, test } from "qunit";
import { deepMerge } from "discourse-common/lib/object";
import DiscourseMarkdownIt from "discourse-markdown-it";
import { extractDataAttribute } from "discourse-markdown-it/engine";
const rawOpts = {
siteSettings: {
@ -27,10 +27,12 @@ const rawOpts = {
getURL: (url) => url,
};
const defaultOpts = buildOptions(rawOpts);
function build(options = rawOpts) {
return DiscourseMarkdownIt.withDefaultFeatures().withOptions(options);
}
QUnit.assert.cooked = function (input, expected, message) {
const actual = new PrettyText(defaultOpts).cook(input);
const actual = build().cook(input);
this.pushResult({
result: actual === expected.replace(/\/>/g, ">"),
actual,
@ -41,7 +43,7 @@ QUnit.assert.cooked = function (input, expected, message) {
QUnit.assert.cookedOptions = function (input, opts, expected, message) {
const merged = deepMerge({}, rawOpts, opts);
const actual = new PrettyText(buildOptions(merged)).cook(input);
const actual = build(merged).cook(input);
this.pushResult({
result: actual === expected,
actual,
@ -59,12 +61,12 @@ module("Unit | Utility | pretty-text", function (hooks) {
test("buildOptions", function (assert) {
assert.ok(
buildOptions({ siteSettings: { enable_emoji: true } }).discourse.features
build({ siteSettings: { enable_emoji: true } }).options.discourse.features
.emoji,
"emoji enabled"
);
assert.ok(
!buildOptions({ siteSettings: { enable_emoji: false } }).discourse
!build({ siteSettings: { enable_emoji: false } }).options.discourse
.features.emoji,
"emoji disabled"
);
@ -733,7 +735,7 @@ eviltrout</p>
test("Oneboxing", function (assert) {
function matches(input, regexp) {
return new PrettyText(defaultOpts).cook(input).match(regexp);
return build().cook(input).match(regexp);
}
assert.ok(
@ -1338,7 +1340,7 @@ var bar = 'bar';
});
test("quotes with trailing formatting", function (assert) {
const result = new PrettyText(defaultOpts).cook(
const result = build().cook(
'[quote="EvilTrout, post:123, topic:456, full:true"]\nhello\n[/quote]\n*Test*'
);
assert.strictEqual(

90
app/assets/javascripts/discourse/tests/unit/lib/sanitizer-test.js
View File

@ -1,38 +1,44 @@
import { setupTest } from "ember-qunit";
import AllowLister from "pretty-text/allow-lister";
import PrettyText, { buildOptions } from "pretty-text/pretty-text";
import { hrefAllowed, sanitize } from "pretty-text/sanitizer";
import { module, test } from "qunit";
import DiscourseMarkdownIt from "discourse-markdown-it";
function build(options) {
return DiscourseMarkdownIt.withDefaultFeatures().withOptions(options);
}
module("Unit | Utility | sanitizer", function (hooks) {
setupTest(hooks);
test("sanitize", function (assert) {
const pt = new PrettyText(
buildOptions({
siteSettings: {
allowed_iframes:
"https://www.google.com/maps/embed?|https://www.openstreetmap.org/export/embed.html?",
},
})
);
const engine = build({
siteSettings: {
allowed_iframes:
"https://www.google.com/maps/embed?|https://www.openstreetmap.org/export/embed.html?",
},
});
const cooked = (input, expected, text) =>
assert.strictEqual(pt.cook(input), expected.replace(/\/>/g, ">"), text);
assert.strictEqual(
engine.cook(input),
expected.replace(/\/>/g, ">"),
text
);
assert.strictEqual(
pt.sanitize('<i class="fa-bug fa-spin">bug</i>'),
engine.sanitize('<i class="fa-bug fa-spin">bug</i>'),
"<i>bug</i>"
);
assert.strictEqual(
pt.sanitize("<div><script>alert('hi');</script></div>"),
engine.sanitize("<div><script>alert('hi');</script></div>"),
"<div></div>"
);
assert.strictEqual(
pt.sanitize("<div><p class=\"funky\" wrong='1'>hello</p></div>"),
engine.sanitize("<div><p class=\"funky\" wrong='1'>hello</p></div>"),
"<div><p>hello</p></div>"
);
assert.strictEqual(pt.sanitize("<3 <3"), "&lt;3 &lt;3");
assert.strictEqual(pt.sanitize("<_<"), "&lt;_&lt;");
assert.strictEqual(engine.sanitize("<3 <3"), "&lt;3 &lt;3");
assert.strictEqual(engine.sanitize("<_<"), "&lt;_&lt;");
cooked(
"hello<script>alert(42)</script>",
@ -71,10 +77,16 @@ module("Unit | Utility | sanitizer", function (hooks) {
"it allows iframe to OpenStreetMap"
);
assert.strictEqual(pt.sanitize("<textarea>hullo</textarea>"), "hullo");
assert.strictEqual(pt.sanitize("<button>press me!</button>"), "press me!");
assert.strictEqual(pt.sanitize("<canvas>draw me!</canvas>"), "draw me!");
assert.strictEqual(pt.sanitize("<progress>hello"), "hello");
assert.strictEqual(engine.sanitize("<textarea>hullo</textarea>"), "hullo");
assert.strictEqual(
engine.sanitize("<button>press me!</button>"),
"press me!"
);
assert.strictEqual(
engine.sanitize("<canvas>draw me!</canvas>"),
"draw me!"
);
assert.strictEqual(engine.sanitize("<progress>hello"), "hello");
cooked(
"[the answer](javascript:alert(42))",
@ -148,62 +160,62 @@ module("Unit | Utility | sanitizer", function (hooks) {
});
test("ids on headings", function (assert) {
const pt = new PrettyText(buildOptions({ siteSettings: {} }));
const engine = build({ siteSettings: {} });
assert.strictEqual(
pt.sanitize("<h3>Test Heading</h3>"),
engine.sanitize("<h3>Test Heading</h3>"),
"<h3>Test Heading</h3>"
);
assert.strictEqual(
pt.sanitize(`<h1 id="heading--test">Test Heading</h1>`),
engine.sanitize(`<h1 id="heading--test">Test Heading</h1>`),
`<h1 id="heading--test">Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h2 id="heading--cool">Test Heading</h2>`),
engine.sanitize(`<h2 id="heading--cool">Test Heading</h2>`),
`<h2 id="heading--cool">Test Heading</h2>`
);
assert.strictEqual(
pt.sanitize(`<h3 id="heading--dashed-name">Test Heading</h3>`),
engine.sanitize(`<h3 id="heading--dashed-name">Test Heading</h3>`),
`<h3 id="heading--dashed-name">Test Heading</h3>`
);
assert.strictEqual(
pt.sanitize(`<h4 id="heading--underscored_name">Test Heading</h4>`),
engine.sanitize(`<h4 id="heading--underscored_name">Test Heading</h4>`),
`<h4 id="heading--underscored_name">Test Heading</h4>`
);
assert.strictEqual(
pt.sanitize(`<h5 id="heading--trout">Test Heading</h5>`),
engine.sanitize(`<h5 id="heading--trout">Test Heading</h5>`),
`<h5 id="heading--trout">Test Heading</h5>`
);
assert.strictEqual(
pt.sanitize(`<h6 id="heading--discourse">Test Heading</h6>`),
engine.sanitize(`<h6 id="heading--discourse">Test Heading</h6>`),
`<h6 id="heading--discourse">Test Heading</h6>`
);
});
test("autoplay videos must be muted", function (assert) {
let pt = new PrettyText(buildOptions({ siteSettings: {} }));
let engine = build({ siteSettings: {} });
assert.ok(
pt
engine
.sanitize(
`<p>Hey</p><video autoplay src="http://example.com/music.mp4"/>`
)
.match(/muted/)
);
assert.ok(
pt
engine
.sanitize(
`<p>Hey</p><video autoplay><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
)
.match(/muted/)
);
assert.ok(
pt
engine
.sanitize(
`<p>Hey</p><video autoplay muted><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
)
.match(/muted/)
);
assert.notOk(
pt
engine
.sanitize(
`<p>Hey</p><video><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
)
@ -212,29 +224,29 @@ module("Unit | Utility | sanitizer", function (hooks) {
});
test("poorly formed ids on headings", function (assert) {
let pt = new PrettyText(buildOptions({ siteSettings: {} }));
let engine = build({ siteSettings: {} });
assert.strictEqual(
pt.sanitize(`<h1 id="evil-trout">Test Heading</h1>`),
engine.sanitize(`<h1 id="evil-trout">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h1 id="heading--">Test Heading</h1>`),
engine.sanitize(`<h1 id="heading--">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h1 id="heading--with space">Test Heading</h1>`),
engine.sanitize(`<h1 id="heading--with space">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h1 id="heading--with*char">Test Heading</h1>`),
engine.sanitize(`<h1 id="heading--with*char">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h1 id="heading--">Test Heading</h1>`),
engine.sanitize(`<h1 id="heading--">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
assert.strictEqual(
pt.sanitize(`<h1 id="test-heading--cool">Test Heading</h1>`),
engine.sanitize(`<h1 id="test-heading--cool">Test Heading</h1>`),
`<h1>Test Heading</h1>`
);
});

1
app/assets/javascripts/package.json
View File

@ -12,6 +12,7 @@
"discourse-common",
"discourse-hbr",
"discourse-i18n",
"discourse-markdown-it",
"discourse-plugins",
"discourse-widget-hbs",
"ember-cli-progress-ci",

601
app/assets/javascripts/pretty-text/addon/engines/discourse-markdown-it.js
View File

@ -1,601 +0,0 @@
import AllowLister from "pretty-text/allow-lister";
import guid from "pretty-text/guid";
import { sanitize } from "pretty-text/sanitizer";
import deprecated from "discourse-common/lib/deprecated";
import { cloneJSON } from "discourse-common/lib/object";
export const ATTACHMENT_CSS_CLASS = "attachment";
function deprecate(feature, name) {
return function () {
if (window.console && window.console.log) {
window.console.log(
feature +
": " +
name +
" is deprecated, please use the new markdown it APIs"
);
}
};
}
// We have some custom extensions and extension points for markdown-it, including
// the helper (passed in via setup) that has registerOptions, registerPlugin etc.
// as well as our postProcessText rule to replace text with a regex.
//
// Take a look at https://meta.discourse.org/t/developers-guide-to-markdown-extensions/66023
// for more detailed information.
function createHelper(
featureName,
opts,
optionCallbacks,
pluginCallbacks,
customMarkdownCookFnCallbacks,
getOptions,
allowListed
) {
let helper = {};
helper.markdownIt = true;
helper.allowList = (info) => allowListed.push([featureName, info]);
helper.whiteList = (info) => {
deprecated("`whiteList` has been replaced with `allowList`", {
since: "2.6.0.beta.4",
dropFrom: "2.7.0",
id: "discourse.markdown-it.whitelist",
});
helper.allowList(info);
};
helper.registerInline = deprecate(featureName, "registerInline");
helper.replaceBlock = deprecate(featureName, "replaceBlock");
helper.addPreProcessor = deprecate(featureName, "addPreProcessor");
helper.inlineReplace = deprecate(featureName, "inlineReplace");
helper.postProcessTag = deprecate(featureName, "postProcessTag");
helper.inlineRegexp = deprecate(featureName, "inlineRegexp");
helper.inlineBetween = deprecate(featureName, "inlineBetween");
helper.postProcessText = deprecate(featureName, "postProcessText");
helper.onParseNode = deprecate(featureName, "onParseNode");
helper.registerBlock = deprecate(featureName, "registerBlock");
// hack to allow moving of getOptions
helper.getOptions = () => getOptions.f();
helper.registerOptions = (callback) => {
optionCallbacks.push([featureName, callback]);
};
helper.registerPlugin = (callback) => {
pluginCallbacks.push([featureName, callback]);
};
helper.buildCookFunction = (callback) => {
customMarkdownCookFnCallbacks.push([featureName, callback]);
};
return helper;
}
// TODO we may just use a proper ruler from markdown it... this is a basic proxy
class Ruler {
constructor() {
this.rules = [];
}
getRules() {
return this.rules;
}
getRuleForTag(tag) {
this.ensureCache();
if (this.cache.hasOwnProperty(tag)) {
return this.cache[tag];
}
}
ensureCache() {
if (this.cache) {
return;
}
this.cache = {};
for (let i = this.rules.length - 1; i >= 0; i--) {
let info = this.rules[i];
this.cache[info.rule.tag] = info;
}
}
push(name, rule) {
this.rules.push({ name, rule });
this.cache = null;
}
}
// block bb code ruler for parsing of quotes / code / polls
function setupBlockBBCode(md) {
md.block.bbcode = { ruler: new Ruler() };
}
// inline bbcode ruler for parsing of spoiler tags, discourse-chart etc
function setupInlineBBCode(md) {
md.inline.bbcode = { ruler: new Ruler() };
}
// rule for text replacement via regex, used for @mentions, category hashtags, etc.
function setupTextPostProcessRuler(md) {
const TextPostProcessRuler = requirejs(
"pretty-text/engines/discourse-markdown/text-post-process"
).TextPostProcessRuler;
md.core.textPostProcess = { ruler: new TextPostProcessRuler() };
}
// hoists html_raw tokens out of the render flow and replaces them
// with a GUID. this GUID is then replaced with the final raw HTML
// content in unhoistForCooked
function renderHoisted(tokens, idx, options) {
const content = tokens[idx].content;
if (content && content.length > 0) {
let id = guid();
options.discourse.hoisted[id] = content;
return id;
} else {
return "";
}
}
function setupUrlDecoding(md) {
// this fixed a subtle issue where %20 is decoded as space in
// automatic urls
md.utils.lib.mdurl.decode.defaultChars = ";/?:@&=+$,# ";
}
// html_raw tokens, funnily enough, render raw HTML via renderHoisted and
// unhoistForCooked
function setupHoister(md) {
md.renderer.rules.html_raw = renderHoisted;
}
// videoHTML and audioHTML follow the same HTML syntax
// as oneboxer.rb when dealing with these formats
function videoHTML(token) {
const src = token.attrGet("src");
const origSrc = token.attrGet("data-orig-src");
const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
return `<div class="video-placeholder-container" data-video-src="${src}" ${dataOrigSrcAttr}>
</div>`;
}
function audioHTML(token) {
const src = token.attrGet("src");
const origSrc = token.attrGet("data-orig-src");
const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
return `<audio preload="metadata" controls>
<source src="${src}" ${dataOrigSrcAttr}>
<a href="${src}">${src}</a>
</audio>`;
}
const IMG_SIZE_REGEX =
/^([1-9]+[0-9]*)x([1-9]+[0-9]*)(\s*,\s*(x?)([1-9][0-9]{0,2}?)([%x]?))?$/;
function renderImageOrPlayableMedia(tokens, idx, options, env, slf) {
const token = tokens[idx];
const alt = slf.renderInlineAsText(token.children, options, env);
const split = alt.split("|");
const altSplit = [split[0]];
// markdown-it supports returning HTML instead of continuing to render the current token
// see https://github.com/markdown-it/markdown-it/blob/master/docs/architecture.md#renderer
// handles |video and |audio alt transformations for image tags
if (split[1] === "video") {
if (
options.discourse.previewing &&
options.discourse.limitedSiteSettings.enableDiffhtmlPreview
) {
const src = token.attrGet("src");
const origSrc = token.attrGet("data-orig-src");
const dataOrigSrcAttr =
origSrc !== null ? `data-orig-src="${origSrc}"` : "";
return `<div class="video-container">
<video width="100%" height="100%" preload="metadata" controls>
<source src="${src}" ${dataOrigSrcAttr}>
<a href="${src}">${src}</a>
</video>
</div>`;
} else {
if (options.discourse.previewing) {
return `<div class="onebox-placeholder-container">
<span class="placeholder-icon video"></span>
</div>`;
} else {
return videoHTML(token);
}
}
} else if (split[1] === "audio") {
return audioHTML(token);
}
// parsing ![myimage|500x300]() or ![myimage|75%]() or ![myimage|500x300, 75%]
for (let i = 1, match, data; i < split.length; ++i) {
if ((match = split[i].match(IMG_SIZE_REGEX)) && match[1] && match[2]) {
let width = match[1];
let height = match[2];
// calculate using percentage
if (match[5] && match[6] && match[6] === "%") {
let percent = parseFloat(match[5]) / 100.0;
width = parseInt(width * percent, 10);
height = parseInt(height * percent, 10);
}
// calculate using only given width
if (match[5] && match[6] && match[6] === "x") {
let wr = parseFloat(match[5]) / width;
width = parseInt(match[5], 10);
height = parseInt(height * wr, 10);
}
// calculate using only given height
if (match[5] && match[4] && match[4] === "x" && !match[6]) {
let hr = parseFloat(match[5]) / height;
height = parseInt(match[5], 10);
width = parseInt(width * hr, 10);
}
if (token.attrIndex("width") === -1) {
token.attrs.push(["width", width]);
}
if (token.attrIndex("height") === -1) {
token.attrs.push(["height", height]);
}
if (
options.discourse.previewing &&
match[6] !== "x" &&
match[4] !== "x"
) {
token.attrs.push(["class", "resizable"]);
}
} else if ((data = extractDataAttribute(split[i]))) {
token.attrs.push(data);
} else if (split[i] === "thumbnail") {
token.attrs.push(["data-thumbnail", "true"]);
} else {
altSplit.push(split[i]);
}
}
const altValue = altSplit.join("|").trim();
if (altValue === "") {
token.attrSet("role", "presentation");
} else {
token.attrSet("alt", altValue);
}
return slf.renderToken(tokens, idx, options);
}
// we have taken over the ![]() syntax in markdown to
// be able to render a video or audio URL as well as the
// image using |video and |audio in the text inside []
function setupImageAndPlayableMediaRenderer(md) {
md.renderer.rules.image = renderImageOrPlayableMedia;
}
function renderAttachment(tokens, idx, options, env, slf) {
const linkToken = tokens[idx];
const textToken = tokens[idx + 1];
const split = textToken.content.split("|");
const contentSplit = [];
for (let i = 0, data; i < split.length; ++i) {
if (split[i] === ATTACHMENT_CSS_CLASS) {
linkToken.attrs.unshift(["class", split[i]]);
} else if ((data = extractDataAttribute(split[i]))) {
linkToken.attrs.push(data);
} else {
contentSplit.push(split[i]);
}
}
if (contentSplit.length > 0) {
textToken.content = contentSplit.join("|");
}
return slf.renderToken(tokens, idx, options);
}
function setupAttachments(md) {
md.renderer.rules.link_open = renderAttachment;
}
function buildCustomMarkdownCookFunction(engineOpts, defaultEngineOpts) {
// everything except the engine for opts can just point to the other
// opts references, they do not change and we don't need to worry about
// mutating them. note that this may need to be updated when additional
// opts are added to the pipeline
const newOpts = {};
newOpts.allowListed = defaultEngineOpts.allowListed;
newOpts.pluginCallbacks = defaultEngineOpts.pluginCallbacks;
newOpts.sanitizer = defaultEngineOpts.sanitizer;
newOpts.discourse = {};
const featureConfig = cloneJSON(defaultEngineOpts.discourse.features);
// everything from the discourse part of defaultEngineOpts can be cloned except
// the features, because these can be a limited subset and we don't want to
// change the original object reference
for (const [key, value] of Object.entries(defaultEngineOpts.discourse)) {
if (key !== "features") {
newOpts.discourse[key] = value;
}
}
if (engineOpts.featuresOverride !== undefined) {
overrideMarkdownFeatures(featureConfig, engineOpts.featuresOverride);
}
newOpts.discourse.features = featureConfig;
const markdownitOpts = {
discourse: newOpts.discourse,
html: defaultEngineOpts.engine.options.html,
breaks: defaultEngineOpts.engine.options.breaks,
xhtmlOut: defaultEngineOpts.engine.options.xhtmlOut,
linkify: defaultEngineOpts.engine.options.linkify,
typographer: defaultEngineOpts.engine.options.typographer,
};
newOpts.engine = createMarkdownItEngineWithOpts(
markdownitOpts,
engineOpts.markdownItRules
);
// we have to do this to make sure plugin callbacks, allow list, and helper
// functions are all set up correctly for the new engine
setupMarkdownEngine(newOpts, featureConfig);
// we don't need the whole engine as a consumer, just a cook function
// will do
return function customRenderFn(contentToRender) {
return cook(contentToRender, newOpts);
};
}
function createMarkdownItEngineWithOpts(markdownitOpts, ruleOverrides) {
if (ruleOverrides !== undefined) {
// Preset for "zero", https://github.com/markdown-it/markdown-it/blob/master/lib/presets/zero.js
return globalThis.markdownit("zero", markdownitOpts).enable(ruleOverrides);
}
return globalThis.markdownit(markdownitOpts);
}
function overrideMarkdownFeatures(features, featureOverrides) {
if (featureOverrides !== undefined) {
Object.keys(features).forEach((feature) => {
features[feature] = featureOverrides.includes(feature);
});
}
}
function setupMarkdownEngine(opts, featureConfig) {
const quotation_marks =
opts.discourse.limitedSiteSettings.markdownTypographerQuotationMarks;
if (quotation_marks) {
opts.engine.options.quotes = quotation_marks.split("|");
}
opts.engine.linkify.tlds(
(opts.discourse.limitedSiteSettings.markdownLinkifyTlds || "").split("|")
);
setupUrlDecoding(opts.engine);
setupHoister(opts.engine);
setupImageAndPlayableMediaRenderer(opts.engine);
setupAttachments(opts.engine);
setupBlockBBCode(opts.engine);
setupInlineBBCode(opts.engine);
setupTextPostProcessRuler(opts.engine);
opts.pluginCallbacks.forEach(([feature, callback]) => {
if (featureConfig[feature]) {
if (callback === null || callback === undefined) {
// eslint-disable-next-line no-console
console.log("BAD MARKDOWN CALLBACK FOUND");
// eslint-disable-next-line no-console
console.log(`FEATURE IS: ${feature}`);
}
opts.engine.use(callback);
}
});
// top level markdown it notifier
opts.markdownIt = true;
opts.setup = true;
if (!opts.discourse.sanitizer || !opts.sanitizer) {
const allowLister = new AllowLister(opts.discourse);
opts.allowListed.forEach(([feature, info]) => {
allowLister.allowListFeature(feature, info);
});
opts.sanitizer = opts.discourse.sanitizer = !!opts.discourse.sanitize
? (a) => sanitize(a, allowLister)
: (a) => a;
}
}
function unhoistForCooked(hoisted, cooked) {
const keys = Object.keys(hoisted);
if (keys.length) {
let found = true;
const unhoist = function (key) {
cooked = cooked.replace(new RegExp(key, "g"), function () {
found = true;
return hoisted[key];
});
delete hoisted[key];
};
while (found) {
found = false;
keys.forEach(unhoist);
}
}
return cooked;
}
export function extractDataAttribute(str) {
let sep = str.indexOf("=");
if (sep === -1) {
return null;
}
const key = `data-${str.slice(0, sep)}`.toLowerCase();
if (!/^[A-Za-z]+[\w\-\:\.]*$/.test(key)) {
return null;
}
const value = str.slice(sep + 1);
return [key, value];
}
let Helpers;
export function setup(opts, siteSettings, state) {
if (opts.setup) {
return;
}
// we got to require this late cause bundle is not loaded in pretty-text
Helpers =
Helpers || requirejs("pretty-text/engines/discourse-markdown/helpers");
opts.markdownIt = true;
let optionCallbacks = [];
let pluginCallbacks = [];
let customMarkdownCookFnCallbacks = [];
// ideally I would like to change the top level API a bit, but in the mean time this will do
let getOptions = {
f: () => opts,
};
const check = /discourse-markdown\/|markdown-it\//;
let features = [];
let allowListed = [];
// all of the modules under discourse-markdown or markdown-it
// directories are considered additional markdown "features" which
// may define their own rules
Object.keys(require.entries).forEach((entry) => {
if (check.test(entry)) {
const module = requirejs(entry);
if (module && module.setup) {
const id = entry.split("/").reverse()[0];
let priority = module.priority || 0;
features.unshift({ id, setup: module.setup, priority });
}
}
});
features
.sort((a, b) => a.priority - b.priority)
.forEach((markdownFeature) => {
markdownFeature.setup(
createHelper(
markdownFeature.id,
opts,
optionCallbacks,
pluginCallbacks,
customMarkdownCookFnCallbacks,
getOptions,
allowListed
)
);
});
Object.entries(state.allowListed || {}).forEach((entry) => {
allowListed.push(entry);
});
optionCallbacks.forEach(([, callback]) => {
callback(opts, siteSettings, state);
});
// enable all features by default
features.forEach((feature) => {
if (!opts.features.hasOwnProperty(feature.id)) {
opts.features[feature.id] = true;
}
});
if (opts.featuresOverride !== undefined) {
overrideMarkdownFeatures(opts.features, opts.featuresOverride);
}
let copy = {};
Object.keys(opts).forEach((entry) => {
copy[entry] = opts[entry];
delete opts[entry];
});
copy.helpers = {
textReplace: Helpers.textReplace,
};
opts.discourse = copy;
getOptions.f = () => opts.discourse;
opts.discourse.limitedSiteSettings = {
secureUploads: siteSettings.secure_uploads,
enableDiffhtmlPreview: siteSettings.enable_diffhtml_preview,
traditionalMarkdownLinebreaks: siteSettings.traditional_markdown_linebreaks,
enableMarkdownLinkify: siteSettings.enable_markdown_linkify,
enableMarkdownTypographer: siteSettings.enable_markdown_typographer,
markdownTypographerQuotationMarks:
siteSettings.markdown_typographer_quotation_marks,
markdownLinkifyTlds: siteSettings.markdown_linkify_tlds,
};
const markdownitOpts = {
discourse: opts.discourse,
html: true,
breaks: !opts.discourse.limitedSiteSettings.traditionalMarkdownLinebreaks,
xhtmlOut: false,
linkify: opts.discourse.limitedSiteSettings.enableMarkdownLinkify,
typographer: opts.discourse.limitedSiteSettings.enableMarkdownTypographer,
};
opts.engine = createMarkdownItEngineWithOpts(
markdownitOpts,
opts.discourse.markdownItRules
);
opts.pluginCallbacks = pluginCallbacks;
opts.allowListed = allowListed;
setupMarkdownEngine(opts, opts.discourse.features);
customMarkdownCookFnCallbacks.forEach(([, callback]) => {
callback(opts, (engineOpts, afterBuild) =>
afterBuild(buildCustomMarkdownCookFunction(engineOpts, opts))
);
});
}
export function cook(raw, opts) {
// we still have to hoist html_raw nodes so they bypass the allowlister
// this is the case for oneboxes and also certain plugins that require
// raw HTML rendering within markdown bbcode rules
opts.discourse.hoisted ??= {};
const rendered = opts.engine.render(raw);
let cooked = opts.discourse.sanitizer(rendered).trim();
// opts.discourse.hoisted guid keys will be deleted within here to
// keep the object empty
cooked = unhoistForCooked(opts.discourse.hoisted, cooked);
return cooked;
}

123
app/assets/javascripts/pretty-text/addon/pretty-text.js
View File

@ -1,9 +1,4 @@
import {
cook as cookIt,
setup as setupIt,
} from "pretty-text/engines/discourse-markdown-it";
import deprecated from "discourse-common/lib/deprecated";
import { deepMerge } from "discourse-common/lib/object";
export function registerOption() {
deprecated(
@ -15,121 +10,3 @@ export function registerOption() {
}
);
}
// see also: __optInput in PrettyText#cook and PrettyText#markdown,
// the options are passed here and must be explicitly allowed with
// the const options & state below
export function buildOptions(state) {
const {
siteSettings,
getURL,
lookupAvatar,
lookupPrimaryUserGroup,
getTopicInfo,
topicId,
forceQuoteLink,
userId,
getCurrentUser,
currentUser,
lookupAvatarByPostNumber,
lookupPrimaryUserGroupByPostNumber,
formatUsername,
emojiUnicodeReplacer,
lookupUploadUrls,
previewing,
censoredRegexp,
disableEmojis,
customEmojiTranslation,
watchedWordsReplace,
watchedWordsLink,
emojiDenyList,
featuresOverride,
markdownItRules,
additionalOptions,
hashtagTypesInPriorityOrder,
hashtagIcons,
hashtagLookup,
} = state;
let features = {};
if (state.features) {
features = deepMerge(features, state.features);
}
const options = {
sanitize: true,
getURL,
features,
lookupAvatar,
lookupPrimaryUserGroup,
getTopicInfo,
topicId,
forceQuoteLink,
userId,
getCurrentUser,
currentUser,
lookupAvatarByPostNumber,
lookupPrimaryUserGroupByPostNumber,
formatUsername,
emojiUnicodeReplacer,
lookupUploadUrls,
censoredRegexp,
customEmojiTranslation,
allowedHrefSchemes: siteSettings.allowed_href_schemes
? siteSettings.allowed_href_schemes.split("|")
: null,
allowedIframes: siteSettings.allowed_iframes
? siteSettings.allowed_iframes.split("|")
: [],
markdownIt: true,
previewing,
disableEmojis,
watchedWordsReplace,
watchedWordsLink,
emojiDenyList,
featuresOverride,
markdownItRules,
additionalOptions,
hashtagTypesInPriorityOrder,
hashtagIcons,
hashtagLookup,
};
// note, this will mutate options due to the way the API is designed
// may need a refactor
setupIt(options, siteSettings, state);
return options;
}
export default class {
constructor(opts) {
if (!opts) {
opts = buildOptions({ siteSettings: {} });
}
this.opts = opts;
}
disableSanitizer() {
this.opts.sanitizer = this.opts.discourse.sanitizer = (ident) => ident;
}
cook(raw) {
if (!raw || raw.length === 0) {
return "";
}
let result;
result = cookIt(raw, this.opts);
return result ? result : "";
}
parse(markdown, env = {}) {
return this.opts.engine.parse(markdown, env);
}
sanitize(html) {
return this.opts.sanitizer(html).trim();
}
}

2
app/assets/javascripts/pretty-text/engines/discourse-markdown/helpers.js → app/assets/javascripts/pretty-text/addon/text-replace.js
View File

@ -1,8 +1,6 @@
// since the markdown.it interface is a bit on the verbose side
// we can keep some general patterns here
export default null;
// creates a rule suitable for inline parsing and replacement
//
// example:

36
app/assets/javascripts/pretty-text/index.js
View File

@ -1,44 +1,8 @@
"use strict";
const Funnel = require("broccoli-funnel");
const mergeTrees = require("broccoli-merge-trees");
const path = require("path");
module.exports = {
name: require("./package").name,
// custom method to produce the tree for markdown-it-bundle.js
// called by ember-cli-build.js in discourse core
//
// code in here is only needed by the editor and we do not want them included
// into the main addon/vendor bundle; instead, it'll be included via a script
// tag as needed
treeForMarkdownItBundle() {
return mergeTrees([this._treeForEngines(), this._treeForMarkdownIt()]);
},
// treat the JS code in /engines like any other JS code in the /addon folder
_treeForEngines() {
let enginesTreePath = path.resolve(this.root, "engines");
let enginesTree = this.treeGenerator(enginesTreePath);
// we started at /engines, if we just call treeForAddon, the modules will
// be under pretty-text/*, but we want pretty-text/engines/*
let namespacedTree = new Funnel(enginesTree, {
destDir: "engines",
});
return this.treeForAddon.call(this, namespacedTree);
},
_treeForMarkdownIt() {
let markdownIt = require.resolve("markdown-it/dist/markdown-it.js");
return new Funnel(path.dirname(markdownIt), {
files: ["markdown-it.js"],
});
},
isDevelopingAddon() {
return true;
},

3
app/assets/javascripts/pretty-text/package.json
View File

@ -1,7 +1,7 @@
{
"name": "pretty-text",
"version": "1.0.0",
"description": "Discourse's text rendering pipeline",
"description": "Discourse's text rendering utilities",
"author": "Discourse",
"license": "GPL-2.0-only",
"keywords": [
@ -57,7 +57,6 @@
"ember-source": "~3.28.12",
"ember-source-channel-url": "^3.0.0",
"loader.js": "^4.7.0",
"markdown-it": "^13.0.2",
"webpack": "^5.89.0"
},
"engines": {

1
app/helpers/application_helper.rb
View File

@ -692,7 +692,6 @@ module ApplicationHelper
base_uri: Discourse.base_path,
environment: Rails.env,
letter_avatar_version: LetterAvatar.version,
markdown_it_url: script_asset_path("markdown-it-bundle"),
service_worker_url: "service-worker.js",
default_locale: SiteSetting.default_locale,
asset_version: Discourse.assets_digest,

111
lib/pretty_text.rb
View File

@ -28,50 +28,24 @@ module PrettyText
Rails.root
end
def self.find_file(root, filename)
return filename if File.file?("#{root}#{filename}")
es6_name = "#{filename}.js.es6"
return es6_name if File.file?("#{root}#{es6_name}")
js_name = "#{filename}.js"
return js_name if File.file?("#{root}#{js_name}")
erb_name = "#{filename}.js.es6.erb"
return erb_name if File.file?("#{root}#{erb_name}")
erb_name = "#{filename}.js.erb"
erb_name if File.file?("#{root}#{erb_name}")
def self.apply_es6_file(ctx:, path:, module_name:)
source = File.read(path)
transpiler = DiscourseJsProcessor::Transpiler.new
transpiled = transpiler.perform(source, nil, module_name)
ctx.eval(transpiled, filename: module_name)
end
def self.apply_es6_file(ctx, root_path, part_name)
filename = find_file(root_path, part_name)
if filename
source = File.read("#{root_path}#{filename}")
source = ERB.new(source).result(binding) if filename =~ /\.erb\z/
transpiler = DiscourseJsProcessor::Transpiler.new
transpiled = transpiler.perform(source, "#{Rails.root}/app/assets/javascripts/", part_name)
ctx.eval(transpiled)
else
# Look for vendored stuff
vendor_root = "#{Rails.root}/vendor/assets/javascripts/"
filename = find_file(vendor_root, part_name)
ctx.eval(File.read("#{vendor_root}#{filename}")) if filename
end
end
def self.ctx_load_directory(ctx, path)
root_path = "#{Rails.root}/app/assets/javascripts/"
Dir["#{root_path}#{path}/**/*"].sort.each do |f|
apply_es6_file(ctx, root_path, f.sub(root_path, "").sub(/\.js(.es6)?\z/, ""))
def self.ctx_load_directory(ctx:, base_path:, module_prefix:)
Dir["**/*.js", base: base_path].sort.each do |f|
module_name = "#{module_prefix}#{f.delete_suffix(".js")}"
apply_es6_file(ctx: ctx, path: File.join(base_path, f), module_name: module_name)
end
end
def self.create_es6_context
ctx = MiniRacer::Context.new(timeout: 25_000, ensure_gc_after_idle: 2000)
ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
ctx.eval("window = globalThis; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
ctx.attach("rails.logger.info", proc { |err| Rails.logger.info(err.to_s) })
ctx.attach("rails.logger.warn", proc { |err| Rails.logger.warn(err.to_s) })
@ -91,22 +65,40 @@ module PrettyText
ctx.attach("__helpers.#{method}", PrettyText::Helpers.method(method))
end
root_path = "#{Rails.root}/app/assets/javascripts/"
ctx_load(ctx, "#{root_path}/node_modules/loader.js/dist/loader/loader.js")
ctx_load(ctx, "#{root_path}/handlebars-shim.js")
ctx_load(ctx, "#{root_path}/node_modules/xss/dist/xss.js")
root_path = "#{Rails.root}/app/assets/javascripts"
ctx.load("#{root_path}/node_modules/loader.js/dist/loader/loader.js")
ctx.load("#{root_path}/node_modules/markdown-it/dist/markdown-it.js")
ctx.load("#{root_path}/handlebars-shim.js")
ctx.load("#{root_path}/node_modules/xss/dist/xss.js")
ctx.load("#{Rails.root}/lib/pretty_text/vendor-shims.js")
ctx_load_directory(ctx, "pretty-text/addon")
ctx_load_directory(ctx, "pretty-text/engines/discourse-markdown")
ctx_load(ctx, "#{root_path}/node_modules/markdown-it/dist/markdown-it.js")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/get-url")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/object")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/deprecated")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/escape")
apply_es6_file(ctx, root_path, "discourse-common/addon/lib/avatar-utils")
apply_es6_file(ctx, root_path, "discourse-common/addon/utils/watched-words")
apply_es6_file(ctx, root_path, "discourse/app/lib/to-markdown")
ctx_load_directory(
ctx: ctx,
base_path: "#{root_path}/pretty-text/addon",
module_prefix: "pretty-text/",
)
ctx_load_directory(
ctx: ctx,
base_path: "#{root_path}/discourse-markdown-it/src",
module_prefix: "discourse-markdown-it/",
)
%w[
discourse-common/addon/lib/get-url
discourse-common/addon/lib/object
discourse-common/addon/lib/deprecated
discourse-common/addon/lib/escape
discourse-common/addon/lib/avatar-utils
discourse-common/addon/utils/watched-words
discourse/app/lib/to-markdown
discourse/app/static/markdown-it/features
].each do |f|
apply_es6_file(
ctx: ctx,
path: "#{root_path}/#{f}.js",
module_name: f.sub("/addon/", "/").sub("/app/", "/"),
)
end
ctx.load("#{Rails.root}/lib/pretty_text/shims.js")
ctx.eval("__setUnicode(#{Emoji.unicode_replacements_json})")
@ -116,10 +108,13 @@ module PrettyText
to_load << a if File.file?(a) && a =~ /discourse-markdown/
end
to_load.uniq.each do |f|
if f =~ %r{\A.+assets/javascripts/}
root = Regexp.last_match[0]
apply_es6_file(ctx, root, f.sub(root, "").sub(/\.js(\.es6)?\z/, ""))
end
plugin_name = f[%r{/plugins/([^/]+)/}, 1]
module_name = f[%r{/assets/javascripts/(.+)\.}, 1]
apply_es6_file(
ctx: ctx,
path: f,
module_name: "discourse/plugins/#{plugin_name}/#{module_name}",
)
end
DiscoursePluginRegistry.vendored_core_pretty_text.each { |vpt| ctx.eval(File.read(vpt)) }
@ -227,8 +222,8 @@ module PrettyText
buffer << "__optInput.hashtagTypesInPriorityOrder = [#{hashtag_types_as_js}];\n"
buffer << "__optInput.hashtagIcons = #{HashtagAutocompleteService.data_source_icon_map.to_json};\n"
buffer << "__textOptions = __buildOptions(__optInput);\n"
buffer << ("__pt = new __PrettyText(__textOptions);")
buffer << "__pluginFeatures = __loadPluginFeatures();"
buffer << "__pt = __DiscourseMarkdownIt.withCustomFeatures(__pluginFeatures).withOptions(__optInput);"
# Be careful disabling sanitization. We allow for custom emails
buffer << ("__pt.disableSanitizer();") if opts[:sanitize] == false
@ -666,10 +661,6 @@ module PrettyText
rval
end
def self.ctx_load(ctx, *files)
files.each { |file| ctx.load(app_root + file) }
end
private
USER_TYPE ||= "user"

24
lib/pretty_text/shims.js
View File

@ -1,11 +1,3 @@
__PrettyText = require("pretty-text/pretty-text").default;
__buildOptions = require("pretty-text/pretty-text").buildOptions;
__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
__resetTranslationTree =
require("pretty-text/engines/discourse-markdown/emoji").resetTranslationTree;
I18n = {
t(a, b) {
return __helpers.t(a, b);
@ -28,6 +20,13 @@ define("discourse-common/lib/helpers", ["exports"], function (exports) {
};
});
define("pretty-text/engines/discourse-markdown/bbcode-block", [
"exports",
"discourse-markdown-it/features/bbcode-block",
], function (exports, { parseBBCodeTag }) {
exports.parseBBCodeTag = parseBBCodeTag;
});
__emojiUnicodeReplacer = null;
__setUnicode = function (replacements) {
@ -132,3 +131,12 @@ function __lookupPrimaryUserGroup(username) {
function __getCurrentUser(userId) {
return __helpers.get_current_user(userId);
}
__DiscourseMarkdownIt = require("discourse-markdown-it").default;
__buildOptions = require("discourse-markdown-it/options").default;
__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
__resetTranslationTree =
require("discourse-markdown-it/features/emoji").resetTranslationTree;
__loadPluginFeatures = require("discourse/static/markdown-it/features").default;

4
lib/pretty_text/vendor-shims.js
View File

@ -19,3 +19,7 @@ define("discourse-common/lib/loader-shim", ["exports", "require"], function (
define("xss", ["exports"], function (__exports__) {
__exports__.default = window.filterXSS;
});
define("markdown-it", ["exports"], function (exports) {
exports.default = window.markdownit;
});

21
plugins/discourse-details/test/javascripts/lib/details-cooked-test.js
View File

@ -1,7 +1,7 @@
import PrettyText, { buildOptions } from "pretty-text/pretty-text";
import { module, test } from "qunit";
import { cook } from "discourse/lib/text";
const defaultOpts = buildOptions({
const opts = {
siteSettings: {
enable_emoji: true,
emoji_set: "twitter",
@ -10,24 +10,21 @@ const defaultOpts = buildOptions({
},
censoredWords: "shucks|whiz|whizzer",
getURL: (url) => url,
});
};
module("lib:details-cooked-test", function () {
test("details", function (assert) {
const cooked = (input, expected, text) => {
assert.strictEqual(
new PrettyText(defaultOpts).cook(input),
expected.replace(/\/>/g, ">"),
text
);
test("details", async function (assert) {
const testCooked = async (input, expected, text) => {
const cooked = (await cook(input, opts)).toString();
assert.strictEqual(cooked, expected, text);
};
cooked(
await testCooked(
`<details><summary>Info</summary>coucou</details>`,
`<details><summary>Info</summary>coucou</details>`,
"manual HTML for details"
);
cooked(
await testCooked(
"[details=testing]\ntest\n[/details]",
`<details>
<summary>