FIX: word boundary regex (\b) not working in Unicode languages. (#9163)

app/assets/javascripts/admin/components/site-text-summary.js

@@ -1,5 +1,6 @@
 import Component from "@ember/component";
 import { on } from "discourse-common/utils/decorators";
+import highlightHTML from "discourse/lib/highlight-html";
 
 export default Component.extend({
   classNames: ["site-text"],
@@ -10,11 +11,13 @@ export default Component.extend({
     const term = this._searchTerm();
 
     if (term) {
-      $(
-        this.element.querySelector(".site-text-id, .site-text-value")
-      ).highlight(term, {
-        className: "text-highlight"
-      });
+      highlightHTML(
+        this.element.querySelector(".site-text-id, .site-text-value"),
+        term,
+        {
+          className: "text-highlight"
+        }
+      );
     }
     $(this.element.querySelector(".site-text-value")).ellipsis();
   },

app/assets/javascripts/admin/templates/search-logs-term.hbs

@@ -31,7 +31,7 @@
       <div class='fps-topic'>
         <div class='topic'>
           <a class='search-link' href={{result.url}}>
-            {{topic-status topic=result.topic disableActions=true}}<span class='topic-title'>{{#highlight-text highlight=term}}{{html-safe result.topic.fancyTitle}}{{/highlight-text}}</span>
+            {{topic-status topic=result.topic disableActions=true}}<span class='topic-title'>{{#highlight-search highlight=term}}{{html-safe result.topic.fancyTitle}}{{/highlight-search}}</span>
           </a>
 
           <div class='search-category'>
@@ -54,9 +54,9 @@
           </span>
 
           {{#if result.blurb}}
-            {{#highlight-text highlight=term}}
+            {{#highlight-search highlight=term}}
               {{html-safe result.blurb}}
-            {{/highlight-text}}
+            {{/highlight-search}}
           {{/if}}
         </div>
       </div>

app/assets/javascripts/discourse/components/highlight-search.js

@@ -1,12 +1,12 @@
 import Component from "@ember/component";
-import highlightText from "discourse/lib/highlight-text";
+import highlightSearch from "discourse/lib/highlight-search";
 
 export default Component.extend({
   tagName: "span",
 
   _highlightOnInsert: function() {
     const term = this.highlight;
-    highlightText($(this.element), term);
+    highlightSearch($(this.element), term);
   }
     .observes("highlight")
     .on("didInsertElement")

app/assets/javascripts/discourse/lib/highlight-html.js

@@ -0,0 +1,93 @@
+function highlight(node, pattern, nodeName, className) {
+  if (
+    ![Node.ELEMENT_NODE, Node.TEXT_NODE].includes(node.nodeType) ||
+    ["SCRIPT", "STYLE"].includes(node.tagName) ||
+    (node.tagName === nodeName && node.className === className)
+  ) {
+    return 0;
+  }
+
+  if (node.nodeType === Node.ELEMENT_NODE && node.childNodes) {
+    for (let i = 0; i < node.childNodes.length; i++) {
+      i += highlight(node.childNodes[i], pattern, nodeName, className);
+    }
+    return 0;
+  }
+
+  if (node.nodeType === Node.TEXT_NODE) {
+    const match = node.data.match(pattern);
+
+    if (!match) {
+      return 0;
+    }
+
+    const element = document.createElement(nodeName);
+    element.className = className;
+    element.innerText = match[0];
+    const matchNode = node.splitText(match.index);
+    matchNode.splitText(match[0].length);
+    matchNode.parentNode.replaceChild(element, matchNode);
+    return 1;
+  }
+
+  return 0;
+}
+
+export default function(node, words, opts = {}) {
+  let settings = {
+    nodeName: "span",
+    className: "highlighted",
+    wholeWord: false,
+    matchCase: false
+  };
+
+  Object.assign(settings, opts);
+  words = typeof words === "string" ? [words] : words;
+  words = words
+    .filter(Boolean)
+    .map(word => word.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&"));
+
+  if (!words.length) return node;
+
+  let pattern = `(${words.join("|")})`;
+  let flag;
+
+  if (settings.wholeWord) {
+    const hasUnicode = words.some(word => {
+      return !word.match(new RegExp(`\b${word}\b`));
+    });
+    pattern = hasUnicode
+      ? `(?<=[\\s,.:;"']|^)${pattern}(?=[\\s,.:;"']|$)`
+      : `\b${pattern}\b`;
+  }
+
+  if (settings.matchCase) {
+    flag = "i";
+  }
+
+  highlight(
+    node,
+    new RegExp(pattern, flag),
+    settings.nodeName.toUpperCase(),
+    settings.className
+  );
+
+  return node;
+}
+
+export function unhighlightHTML(opts = {}) {
+  let settings = {
+    nodeName: "span",
+    className: "highlighted"
+  };
+
+  Object.assign(settings, opts);
+
+  document
+    .querySelectorAll(`${settings.nodeName}.${settings.className}`)
+    .forEach(e => {
+      const parentNode = e.parentNode;
+      parentNode.replaceChild(e.firstChild, e);
+      parentNode.normalize();
+    });
+}

app/assets/javascripts/discourse/lib/highlight-search.js

@@ -1,4 +1,5 @@
 import { PHRASE_MATCH_REGEXP_PATTERN } from "discourse/lib/concerns/search-constants";
+import highlightHTML from "discourse/lib/highlight-html";
 
 export const CLASS_NAME = "search-highlight";
 
@@ -11,8 +12,10 @@ export default function($elem, term, opts = {}) {
     );
 
     words = words.map(w => w.replace(/^"(.*)"$/, "$1"));
-    const highlightOpts = { wordsOnly: true };
+    const highlightOpts = { wholeWord: true };
     if (!opts.defaultClassName) highlightOpts.className = CLASS_NAME;
-    $elem.highlight(words, highlightOpts);
+    for (let i = 0; i <= $elem.length; i++) {
+      highlightHTML($elem[0], words, highlightOpts);
+    }
   }
 }

app/assets/javascripts/discourse/templates/full-page-search.hbs

@@ -88,7 +88,7 @@
 
                   <a class='search-link' href={{result.url}} {{action "logClick" result.topic_id}}>
                     {{topic-status topic=result.topic disableActions=true showPrivateMessageIcon=true}}
-                    <span class='topic-title'>{{#highlight-text highlight=q}}{{html-safe result.topic.fancyTitle}}{{/highlight-text}}</span>
+                    <span class='topic-title'>{{#highlight-search highlight=q}}{{html-safe result.topic.fancyTitle}}{{/highlight-search}}</span>
                   </a>
 
                   <div class='search-category'>
@@ -112,9 +112,9 @@
                   </span>
 
                   {{#if result.blurb}}
-                    {{#highlight-text highlight=highlightQuery}}
+                    {{#highlight-search highlight=highlightQuery}}
                       {{html-safe result.blurb}}
-                    {{/highlight-text}}
+                    {{/highlight-search}}
                   {{/if}}
                 </div>
 

app/assets/javascripts/discourse/widgets/post-cooked.js

@@ -2,7 +2,11 @@ import { iconHTML } from "discourse-common/lib/icon-library";
 import { ajax } from "discourse/lib/ajax";
 import { isValidLink } from "discourse/lib/click-track";
 import { number } from "discourse/lib/formatter";
-import highlightText from "discourse/lib/highlight-text";
+import highlightSearch from "discourse/lib/highlight-search";
+import {
+  default as highlightHTML,
+  unhighlightHTML
+} from "discourse/lib/highlight-html";
 
 let _decorators = [];
 
@@ -52,13 +56,13 @@ export default class PostCooked {
 
     if (highlight && highlight.length > 2) {
       if (this._highlighted) {
-        $html.unhighlight();
+        unhighlightHTML($html[0]);
       }
 
-      highlightText($html, highlight, { defaultClassName: true });
+      highlightSearch($html, highlight, { defaultClassName: true });
       this._highlighted = true;
     } else if (this._highlighted) {
-      $html.unhighlight();
+      unhighlightHTML($html[0]);
       this._highlighted = false;
     }
   }
@@ -175,10 +179,8 @@ export default class PostCooked {
           div.html(result.cooked);
           _decorators.forEach(cb => cb(div, this.decoratorHelper));
 
-          div.highlight(originalText, {
-            caseSensitive: true,
-            element: "span",
-            className: "highlighted"
+          highlightHTML(div[0], originalText, {
+            matchCase: true
           });
           $blockQuote.showHtml(div, "fast", finished);
         })

app/assets/javascripts/discourse/widgets/search-menu-results.js

@@ -3,7 +3,7 @@ import { dateNode } from "discourse/helpers/node";
 import RawHtml from "discourse/widgets/raw-html";
 import { createWidget } from "discourse/widgets/widget";
 import { h } from "virtual-dom";
-import highlightText from "discourse/lib/highlight-text";
+import highlightSearch from "discourse/lib/highlight-search";
 import { escapeExpression, formatUsername } from "discourse/lib/utilities";
 import { iconNode } from "discourse-common/lib/icon-library";
 import renderTag from "discourse/lib/render-tag";
@@ -15,7 +15,7 @@ class Highlighted extends RawHtml {
   }
 
   decorate($html) {
-    highlightText($html, this.term);
+    highlightSearch($html, this.term);
   }
 }
 

app/assets/javascripts/vendor.js

@@ -29,5 +29,4 @@
 //= require jquery.autoellipsis-1.0.10
 //= require virtual-dom
 //= require virtual-dom-amd
-//= require highlight.js
 //= require intersection-observer

test/javascripts/acceptance/search-test.js.es6

@@ -94,7 +94,7 @@ QUnit.test("Search with context", async assert => {
 
   const highlighted = [];
 
-  find("#post_7 span.highlight-strong").map((_, span) => {
+  find("#post_7 span.highlighted").map((_, span) => {
     highlighted.push(span.innerText);
   });
 

test/javascripts/lib/highlight-search-test.js.es6

@@ -0,0 +1,48 @@
+import highlightSearch, { CLASS_NAME } from "discourse/lib/highlight-search";
+import { fixture } from "helpers/qunit-helpers";
+
+QUnit.module("lib:highlight-search");
+
+QUnit.test("highlighting text", assert => {
+  fixture().html(
+    `
+    <p>This is some text to highlight</p>
+    `
+  );
+
+  highlightSearch(fixture(), "some text");
+
+  const terms = [];
+
+  fixture(`.${CLASS_NAME}`).each((_, elem) => {
+    terms.push(elem.textContent);
+  });
+
+  assert.equal(
+    terms.join(" "),
+    "some text",
+    "it should highlight the terms correctly"
+  );
+});
+
+QUnit.test("highlighting unicode text", assert => {
+  fixture().html(
+    `
+    <p>This is some தமிழ் and русский text to highlight</p>
+    `
+  );
+
+  highlightSearch(fixture(), "தமிழ் русский");
+
+  const terms = [];
+
+  fixture(`.${CLASS_NAME}`).each((_, elem) => {
+    terms.push(elem.textContent);
+  });
+
+  assert.equal(
+    terms.join(" "),
+    "தமிழ் русский",
+    "it should highlight the terms correctly"
+  );
+});

test/javascripts/lib/highlight-text-test.js.es6

@@ -1,26 +0,0 @@
-import highlightText, { CLASS_NAME } from "discourse/lib/highlight-text";
-import { fixture } from "helpers/qunit-helpers";
-
-QUnit.module("lib:highlight-text");
-
-QUnit.test("highlighting text", assert => {
-  fixture().html(
-    `
-    <p>This is some text to highlight</p>
-    `
-  );
-
-  highlightText(fixture(), "some text");
-
-  const terms = [];
-
-  fixture(`.${CLASS_NAME}`).each((_, elem) => {
-    terms.push(elem.textContent);
-  });
-
-  assert.equal(
-    terms.join(" "),
-    "some text",
-    "it should highlight the terms correctly"
-  );
-});

vendor/assets/javascripts/highlight.js

@@ -1,108 +0,0 @@
-// forked cause we may want to amend the logic a bit
-/*
- * jQuery Highlight plugin
- *
- * Based on highlight v3 by Johann Burkard
- * http://johannburkard.de/blog/programming/javascript/highlight-javascript-text-higlighting-jquery-plugin.html
- *
- * Code a little bit refactored and cleaned (in my humble opinion).
- * Most important changes:
- *  - has an option to highlight only entire words (wordsOnly - false by default),
- *  - has an option to be case sensitive (caseSensitive - false by default)
- *  - highlight element tag and class names can be specified in options
- *
- * Usage:
- *   // wrap every occurrance of text 'lorem' in content
- *   // with <span class='highlight'> (default options)
- *   $('#content').highlight('lorem');
- *
- *   // search for and highlight more terms at once
- *   // so you can save some time on traversing DOM
- *   $('#content').highlight(['lorem', 'ipsum']);
- *   $('#content').highlight('lorem ipsum');
- *
- *   // search only for entire word 'lorem'
- *   $('#content').highlight('lorem', { wordsOnly: true });
- *
- *   // don't ignore case during search of term 'lorem'
- *   $('#content').highlight('lorem', { caseSensitive: true });
- *
- *   // wrap every occurrance of term 'ipsum' in content
- *   // with <em class='important'>
- *   $('#content').highlight('ipsum', { element: 'em', className: 'important' });
- *
- *   // remove default highlight
- *   $('#content').unhighlight();
- *
- *   // remove custom highlight
- *   $('#content').unhighlight({ element: 'em', className: 'important' });
- *
- *
- * Copyright (c) 2009 Bartek Szopka
- *
- * Licensed under MIT license.
- *
- */
-
-jQuery.extend({
-    highlight: function (node, re, nodeName, className) {
-        if (node.nodeType === 3) {
-            var match = node.data.match(re);
-            if (match) {
-                var highlight = document.createElement(nodeName || 'span');
-                highlight.className = className || 'highlight';
-                var wordNode = node.splitText(match.index);
-                wordNode.splitText(match[0].length);
-                var wordClone = wordNode.cloneNode(true);
-                highlight.appendChild(wordClone);
-                wordNode.parentNode.replaceChild(highlight, wordNode);
-                return 1; //skip added node in parent
-            }
-        } else if ((node.nodeType === 1 && node.childNodes) && // only element nodes that have children
-                !/(script|style)/i.test(node.tagName) && // ignore script and style nodes
-                !(node.tagName === nodeName.toUpperCase() && node.className === className)) { // skip if already highlighted
-            for (var i = 0; i < node.childNodes.length; i++) {
-                i += jQuery.highlight(node.childNodes[i], re, nodeName, className);
-            }
-        }
-        return 0;
-    }
-});
-
-jQuery.fn.unhighlight = function (options) {
-    var settings = { className: 'highlight-strong', element: 'span' };
-    jQuery.extend(settings, options);
-
-    return this.find(settings.element + "." + settings.className).each(function () {
-        var parent = this.parentNode;
-        parent.replaceChild(this.firstChild, this);
-        parent.normalize();
-    }).end();
-};
-
-jQuery.fn.highlight = function (words, options) {
-    var settings = { className: 'highlight-strong', element: 'span', caseSensitive: false, wordsOnly: false };
-    jQuery.extend(settings, options);
-
-    if (words.constructor === String) {
-        words = [words];
-    }
-    words = jQuery.grep(words, function(word){
-      return word !== '';
-    });
-    words = jQuery.map(words, function(word) {
-      return word.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
-    });
-    if (words.length === 0) { return this; }
-
-    var flag = settings.caseSensitive ? "" : "i";
-    var pattern = "(" + words.join("|") + ")";
-    if (settings.wordsOnly) {
-        pattern = "\\b" + pattern + "\\b";
-    }
-    var re = new RegExp(pattern, flag);
-
-    return this.each(function () {
-        jQuery.highlight(this, re, settings.element, settings.className);
-    });
-};