Support for transpiling .js files (#9160)

* Remove some `.es6` from comments where it does not matter

* Use a post processor for transpilation

This will allow us to eventually use the directory structure to
transpile rather than the extension.

* FIX: Some errors and clean up in confirm-new-email

It would throw an error if the webauthn element wasn't present.
Also I changed things so that no-module is not explicitly
referenced.

* Remove `no-module`

Instead we allow a magic comment: `// discourse-skip-module` to prevent
the asset pipeline from creating a module.

* DEV: Enable babel transpilation based on directory

If it's in `app/assets/javascripts/dicourse` it will be transpiled
even without the `.es6` extension.

* REFACTOR: Remove Tilt/ES6ModuleTranspiler

.eslintignore

@@ -3,7 +3,7 @@ app/assets/javascripts/main_include_admin.js
 app/assets/javascripts/vendor.js
 app/assets/javascripts/locales/i18n.js
 app/assets/javascripts/ember-addons/
-app/assets/javascripts/discourse/lib/autosize.js.es6
+app/assets/javascripts/discourse/lib/autosize.js
 lib/javascripts/locale/
 lib/javascripts/messageformat.js
 lib/highlight_js/

Gemfile

@@ -120,9 +120,6 @@ gem 'sanitize'
 gem 'sidekiq'
 gem 'mini_scheduler'
 
-# for sidekiq web
-gem 'tilt', require: false
-
 gem 'execjs', require: false
 gem 'mini_racer'
 

Gemfile.lock

@@ -539,7 +539,6 @@ DEPENDENCIES
   stackprof
   test-prof
   thor
-  tilt
   uglifier
   unf
   unicorn

app/assets/javascripts/activate-account.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   setTimeout(function() {
     const $activateButton = $("#activate-account-button");

app/assets/javascripts/auto-redirect.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   const path = document.getElementById("data-auto-redirect").dataset.path;
   setTimeout(function() {

app/assets/javascripts/confirm-new-email/bootstrap.js.es6

@@ -0,0 +1,4 @@
+// discourse-skip-module
+(function() {
+  require("confirm-new-email/confirm-new-email");
+})();

app/assets/javascripts/confirm-new-email/confirm-new-email.js.es6

@@ -1,23 +1,26 @@
 import { getWebauthnCredential } from "discourse/lib/webauthn";
 
-document.getElementById("submit-security-key").onclick = function(e) {
-  e.preventDefault();
-  getWebauthnCredential(
-    document.getElementById("security-key-challenge").value,
-    document
-      .getElementById("security-key-allowed-credential-ids")
-      .value.split(","),
-    credentialData => {
-      document.getElementById("security-key-credential").value = JSON.stringify(
-        credentialData
-      );
+const security = document.getElementById("submit-security-key");
+if (security) {
+  security.onclick = function(e) {
+    e.preventDefault();
+    getWebauthnCredential(
+      document.getElementById("security-key-challenge").value,
+      document
+        .getElementById("security-key-allowed-credential-ids")
+        .value.split(","),
+      credentialData => {
+        document.getElementById(
+          "security-key-credential"
+        ).value = JSON.stringify(credentialData);
 
-      $(e.target)
-        .parents("form")
-        .submit();
-    },
-    errorMessage => {
-      document.getElementById("security-key-error").innerText = errorMessage;
-    }
-  );
-};
+        $(e.target)
+          .parents("form")
+          .submit();
+      },
+      errorMessage => {
+        document.getElementById("security-key-error").innerText = errorMessage;
+      }
+    );
+  };
+}

app/assets/javascripts/confirm-new-email/confirm-new-email.no-module.js.es6

@@ -1 +0,0 @@
-require("confirm-new-email/confirm-new-email").default();

app/assets/javascripts/discourse/controllers/second-factor-add-security-key.js.es6

@@ -6,7 +6,7 @@ import {
   isWebauthnSupported
 } from "discourse/lib/webauthn";
 
-// model for this controller is user.js.es6
+// model for this controller is user
 export default Controller.extend(ModalFunctionality, {
   loading: false,
   errorMessage: null,

app/assets/javascripts/discourse/lib/plugin-api.js.es6

@@ -848,7 +848,7 @@ class PluginApi {
    *
    * Example:
    *
-   * // read /discourse/lib/sharing.js.es6 for options
+   * // read discourse/lib/sharing for options
    * api.addSharingSource(options)
    *
    */

app/assets/javascripts/discourse/lib/plugin-connectors.js.es6

@@ -12,7 +12,7 @@ export function resetExtraClasses() {
 }
 
 // Note: In plugins, define a class by path and it will be wired up automatically
-// eg: discourse/connectors/<OUTLET NAME>/<CONNECTOR NAME>.js.es6
+// eg: discourse/connectors/<OUTLET NAME>/<CONNECTOR NAME>
 export function extraConnectorClass(name, obj) {
   _extraConnectorClasses[name] = obj;
 }

app/assets/javascripts/embed-application.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   const referer = document.getElementById("data-embedded").dataset.referer;
 

app/assets/javascripts/google-tag-manager.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   const gtmDataElement = document.getElementById("data-google-tag-manager");
   const dataLayerJson = JSON.parse(gtmDataElement.dataset.dataLayer);

app/assets/javascripts/google-universal-analytics.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 /* eslint-disable */
 // prettier-ignore
 (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){

app/assets/javascripts/onpopstate-handler.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 window.onpopstate = function(event) {
   // check if Discourse object exists if not take care of back navigation
   if (event.state && !window.hasOwnProperty("Discourse")) {

app/assets/javascripts/preload-application-data.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   var ps = require("preload-store").default;
   var preloadedDataElement = document.getElementById("data-preloaded");
@@ -55,6 +56,7 @@
     Discourse.S3BaseUrl = setupData.s3BaseUrl;
   }
 
+  // eslint-disable-next-line
   Ember.RSVP.configure("onerror", function(e) {
     // Ignore TransitionAborted exceptions that bubble up
     if (e && e.message === "TransitionAborted") {

app/assets/javascripts/wizard-start.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function() {
   var wizard = require("wizard/wizard").default.create();
   wizard.start();

app/views/users_email/show_confirm_new_email.html.erb

@@ -78,5 +78,5 @@
   <%= preload_script "locales/i18n" %>
   <%= preload_script "discourse/lib/webauthn" %>
   <%= preload_script "confirm-new-email/confirm-new-email" %>
-  <%= preload_script "confirm-new-email/confirm-new-email.no-module" %>
+  <%= preload_script "confirm-new-email/bootstrap" %>
 </div>

config/application.rb

@@ -76,7 +76,6 @@ module Discourse
     # confused here if we load the deps without `lib` it thinks
     # discourse.rb is under the discourse folder incorrectly
     require_dependency 'lib/discourse'
-    require_dependency 'lib/es6_module_transpiler/rails'
     require_dependency 'lib/js_locale_helper'
 
     # tiny file needed by site settings
@@ -155,7 +154,7 @@ module Discourse
       locales/i18n.js
       discourse/lib/webauthn.js
       confirm-new-email/confirm-new-email.js
-      confirm-new-email/confirm-new-email.no-module.js
+      confirm-new-email/bootstrap.js
       onpopstate-handler.js
       embed-application.js
     }
@@ -244,6 +243,11 @@ module Discourse
     Sprockets.register_mime_type 'text/x-handlebars', extensions: ['.hbr']
     Sprockets.register_transformer 'text/x-handlebars', 'application/javascript', Ember::Handlebars::Template
 
+    require 'discourse_js_processor'
+
+    Sprockets.register_mime_type 'application/javascript', extensions: ['.js', '.es6', '.js.es6'], charset: :unicode
+    Sprockets.register_postprocessor 'application/javascript', DiscourseJsProcessor
+
     require 'discourse_redis'
     require 'logster/redis_store'
     # Use redis for our cache

config/initializers/010-discourse_iife.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require 'discourse_iife'
-
-Rails.application.config.assets.configure do |env|
-  env.register_preprocessor('application/javascript', DiscourseIIFE)
-
-  unless Rails.env.production?
-    require 'source_url'
-    env.register_postprocessor('application/javascript', SourceURL)
-  end
-end

lib/discourse.rb

@@ -655,7 +655,7 @@ module Discourse
     # in case v8 was initialized we want to make sure it is nil
     PrettyText.reset_context
 
-    Tilt::ES6ModuleTranspilerTemplate.reset_context if defined? Tilt::ES6ModuleTranspilerTemplate
+    DiscourseJsProcessor::Transpiler.reset_context if defined? DiscourseJsProcessor::Transpiler
     JsLocaleHelper.reset_context if defined? JsLocaleHelper
     nil
   end

lib/discourse_iife.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-class DiscourseIIFE
-  def initialize(options = {}, &block)
-  end
-
-  def self.instance
-    @instance ||= new
-  end
-
-  def self.call(input)
-    instance.call(input)
-  end
-
-  # Add a IIFE around our javascript
-  def call(input)
-    path = input[:environment].context_class.new(input).pathname.to_s
-    data = input[:data]
-
-    # Only discourse or admin paths
-    return data unless (path =~ /\/javascripts\/discourse/ || path =~ /\/javascripts\/admin/ || path =~ /\/test\/javascripts/)
-
-    # Ignore the js helpers
-    return data if (path =~ /test\_helper\.js/)
-    return data if (path =~ /javascripts\/helpers\//)
-
-    # Ignore ES6 files
-    return data if (path =~ /\.es6/)
-
-    # Ignore translations
-    return data if (path =~ /\/translations/)
-
-    # We don't add IIFEs to handlebars
-    return data if path =~ /\.handlebars/
-    return data if path =~ /\.shbrs/
-    return data if path =~ /\.hbrs/
-    return data if path =~ /\.hbs/
-    return data if path =~ /\.hbr/
-
-    return data if path =~ /discourse-loader/
-
-    "(function () {\n\nvar $ = window.jQuery;\n// IIFE Wrapped Content Begins:\n\n#{data}\n\n// IIFE Wrapped Content Ends\n\n })(this);"
-  end
-
-end

lib/discourse_js_processor.rb

@@ -1,28 +1,54 @@
 # frozen_string_literal: true
-
 require 'execjs'
 require 'mini_racer'
 
-module Tilt
+class DiscourseJsProcessor
 
-  class ES6ModuleTranspilerTemplate < Tilt::Template
-    self.default_mime_type = 'application/javascript'
+  def self.call(input)
+    root_path = input[:load_path] || ''
+    logical_path = (input[:filename] || '').sub(root_path, '').gsub(/\.(js|es6).*$/, '').sub(/^\//, '')
+    data = input[:data]
 
+    if should_transpile?(input[:filename])
+      data = transpile(data, root_path, logical_path)
+    end
+
+    # add sourceURL until we can do proper source maps
+    unless Rails.env.production?
+      data = "eval(#{data.inspect} + \"\\n//# sourceURL=#{logical_path}\");\n"
+    end
+
+    { data: data }
+  end
+
+  def self.transpile(data, root_path, logical_path)
+    transpiler = Transpiler.new(skip_module: skip_module?(data))
+    transpiler.perform(data, root_path, logical_path)
+  end
+
+  def self.should_transpile?(filename)
+    filename ||= ''
+
+    # es6 is always transpiled
+    return true if filename.end_with?(".es6") || filename.end_with?(".es6.erb")
+
+    # For .js check the path...
+    return false unless filename.end_with?(".js") || filename.end_with?(".js.erb")
+
+    relative_path = filename.sub(Rails.root.to_s, '').sub(/^\/*/, '')
+    relative_path.start_with?("app/assets/javascripts/discourse/")
+  end
+
+  def self.skip_module?(data)
+    !!(data.present? && data =~ /^\/\/ discourse-skip-module$/)
+  end
+
+  class Transpiler
     @mutex = Mutex.new
     @ctx_init = Mutex.new
 
-    def self.call(input)
-      filename = input[:filename]
-      source = input[:data]
-      context = input[:environment].context_class.new(input)
-
-      result = new(filename) { source }.render(context)
-      context.metadata.merge(data: result)
-    end
-
-    def prepare
-      # intentionally left empty
-      # Tilt requires this method to be defined
+    def self.mutex
+      @mutex
     end
 
     def self.create_new_context
@@ -66,33 +92,13 @@ JS
       @ctx
     end
 
-    class JavaScriptError < StandardError
-      attr_accessor :message, :backtrace
-
-      def initialize(message, backtrace)
-        @message = message
-        @backtrace = backtrace
-      end
-
+    def initialize(skip_module: false)
+      @skip_module = skip_module
     end
 
-    def self.protect
-      @mutex.synchronize do
-        yield
-      end
-    end
-
-    def babel_transpile(source)
+    def perform(source, root_path = nil, logical_path = nil)
       klass = self.class
-      klass.protect do
-        klass.v8.eval("console.prefix = 'BABEL: babel-eval: ';")
-        @output = klass.v8.eval(babel_source(source))
-      end
-    end
-
-    def module_transpile(source, root_path, logical_path)
-      klass = self.class
-      klass.protect do
+      klass.mutex.synchronize do
         klass.v8.eval("console.prefix = 'BABEL: babel-eval: ';")
         transpiled = babel_source(
           source,
@@ -103,31 +109,12 @@ JS
       end
     end
 
-    def evaluate(scope, locals, &block)
-      return @output if @output
-
-      klass = self.class
-      klass.protect do
-        klass.v8.eval("console.prefix = 'BABEL: #{scope.logical_path}: ';")
-
-        source = babel_source(
-          data,
-          module_name: module_name(scope.root_path, scope.logical_path),
-          filename: scope.logical_path
-        )
-
-        @output = klass.v8.eval(source)
-      end
-
-      @output
-    end
-
     def babel_source(source, opts = nil)
       opts ||= {}
 
       js_source = ::JSON.generate(source, quirks_mode: true)
 
-      if opts[:module_name] && transpile_into_module?
+      if opts[:module_name] && !@skip_module
         filename = opts[:filename] || 'unknown'
         "Babel.transform(#{js_source}, { moduleId: '#{opts[:module_name]}', filename: '#{filename}', ast: false, presets: ['es2015'], plugins: [['transform-es2015-modules-amd', {noInterop: true}], 'transform-decorators-legacy', exports.WidgetHbsCompiler] }).code"
       else
@@ -135,12 +122,6 @@ JS
       end
     end
 
-    private
-
-    def transpile_into_module?
-      file.nil? || file.exclude?('.no-module')
-    end
-
     def module_name(root_path, logical_path)
       path = nil
 
@@ -153,26 +134,8 @@ JS
         path = "discourse/plugins/#{plugin.name}/#{logical_path.sub(/javascripts\//, '')}" if plugin
       end
 
-      path ||= logical_path
-      if ES6ModuleTranspiler.transform
-        path = ES6ModuleTranspiler.transform.call(path)
-      end
-
-      path
+      path || logical_path
     end
 
-    def compiler_method
-      type = {
-        amd: 'AMD',
-        cjs: 'CJS',
-        globals: 'Globals'
-      }[ES6ModuleTranspiler.compile_to.to_sym]
-
-      "to#{type}"
-    end
-
-    def compiler_options
-      ::JSON.generate(ES6ModuleTranspiler.compiler_options, quirks_mode: true)
-    end
   end
 end

lib/discourse_plugin_registry.rb

@@ -151,7 +151,7 @@ class DiscoursePluginRegistry
     end
   end
 
-  JS_REGEX = /\.js$|\.js\.erb$|\.js\.es6|\.js\.no-module\.es6$/
+  JS_REGEX = /\.js$|\.js\.erb$|\.js\.es6$/
   HANDLEBARS_REGEX = /\.(hb[rs]|js\.handlebars)$/
 
   def self.register_asset(asset, opts = nil, plugin_directory_name = nil)

lib/es6_module_transpiler/rails.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require 'es6_module_transpiler/rails/version'
-require 'es6_module_transpiler/tilt'
-require 'es6_module_transpiler/sprockets'
-
-module ES6ModuleTranspiler
-  def self.compile_to
-    @compile_to || :amd
-  end
-
-  def self.compile_to=(target)
-    @compile_to = target
-  end
-
-  def self.transform=(transform)
-    @transform = transform
-  end
-
-  def self.transform
-    @transform
-  end
-
-  def self.compiler_options
-    @compiler_options ||= {}
-  end
-end

lib/es6_module_transpiler/rails/version.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module ES6ModuleTranspiler
-  module Rails
-    VERSION = '0.4.0'
-  end
-end

lib/es6_module_transpiler/sprockets.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-require 'sprockets'
-
-Sprockets.register_mime_type 'application/ecmascript6', extensions: ['.es6', '.js.es6', '.js.no-module.es6'], charset: :unicode
-Sprockets.register_transformer 'application/ecmascript6', 'application/javascript', Tilt::ES6ModuleTranspilerTemplate

lib/es6_module_transpiler/tilt.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-require 'tilt'
-require 'es6_module_transpiler/tilt/es6_module_transpiler_template'
-
-Tilt.register Tilt::ES6ModuleTranspilerTemplate, 'es6'

lib/freedom_patches/raw_handlebars.rb

@@ -12,8 +12,8 @@ class Barber::Precompiler
     if !@precompiler
 
       source = File.read("#{Rails.root}/app/assets/javascripts/discourse-common/lib/raw-handlebars.js.es6")
-      template = Tilt::ES6ModuleTranspilerTemplate.new {}
-      transpiled = template.babel_transpile(source)
+      transpiler = DiscourseJsProcessor::Transpiler.new(skip_module: true)
+      transpiled = transpiler.perform(source)
 
       # very hacky but lets us use ES6. I'm ashamed of this code -RW
       transpiled = transpiled[0...transpiled.index('export ')]

lib/freedom_patches/zeitwerk.rb

@@ -12,7 +12,6 @@ module ActiveSupport::Dependencies::ZeitwerkIntegration::Inflector
     'postgresql_fallback_adapter' => 'PostgreSQLFallbackHandler',
     'regular' => 'Jobs',
     'scheduled' => 'Jobs',
-    'source_url' => 'SourceURL',
     'topic_query_sql' => 'TopicQuerySQL',
     'version' => 'Discourse',
   }

lib/pretty_text.rb

@@ -29,13 +29,10 @@ module PrettyText
     filename = find_file(root_path, part_name)
     if filename
       source = File.read("#{root_path}#{filename}")
+      source = ERB.new(source).result(binding) if filename =~ /\.erb$/
 
-      if filename =~ /\.erb$/
-        source = ERB.new(source).result(binding)
-      end
-
-      template = Tilt::ES6ModuleTranspilerTemplate.new {}
-      transpiled = template.module_transpile(source, "#{Rails.root}/app/assets/javascripts/", part_name)
+      transpiler = DiscourseJsProcessor::Transpiler.new
+      transpiled = transpiler.perform(source, "#{Rails.root}/app/assets/javascripts/", part_name)
       ctx.eval(transpiled)
     else
       # Look for vendored stuff

lib/source_url.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-class SourceURL < Tilt::Template
-  self.default_mime_type = 'application/javascript'
-
-  def self.call(input)
-    filename = input[:filename]
-    source = input[:data]
-    context = input[:environment].context_class.new(input)
-
-    result = new(filename) { source }.render(context)
-    context.metadata.merge(data: result)
-  end
-
-  def prepare
-  end
-
-  def evaluate(scope, locals, &block)
-    code = +"eval("
-    code << data.inspect
-    code << " + \"\\n//# sourceURL=#{scope.logical_path}\""
-    code << ");\n"
-  end
-end

lib/theme_javascript_compiler.rb

@@ -214,8 +214,8 @@ class ThemeJavascriptCompiler
 
   def append_module(script, name, include_variables: true)
     script = "#{theme_variables}#{script}" if include_variables
-    template = Tilt::ES6ModuleTranspilerTemplate.new {}
-    @content << template.module_transpile(script, "", name)
+    transpiler = DiscourseJsProcessor::Transpiler.new
+    @content << transpiler.perform(script, "", name)
   rescue MiniRacer::RuntimeError => ex
     raise CompileError.new ex.message
   end
@@ -237,7 +237,7 @@ class ThemeJavascriptCompiler
   end
 
   def transpile(es6_source, version)
-    template = Tilt::ES6ModuleTranspilerTemplate.new {}
+    transpiler = DiscourseJsProcessor::Transpiler.new(skip_module: true)
     wrapped = <<~PLUGIN_API_JS
       (function() {
         if ('Discourse' in window && typeof Discourse._registerPluginCode === 'function') {
@@ -254,7 +254,7 @@ class ThemeJavascriptCompiler
       })();
     PLUGIN_API_JS
 
-    template.babel_transpile(wrapped)
+    transpiler.perform(wrapped)
   rescue MiniRacer::RuntimeError => ex
     raise CompileError.new ex.message
   end

plugins/discourse-local-dates/assets/javascripts/discourse-local-dates.js.es6

@@ -1,3 +1,4 @@
+// discourse-skip-module
 (function($) {
   const DATE_TEMPLATE = `
     <span>

plugins/discourse-local-dates/plugin.rb

@@ -6,7 +6,7 @@
 # author: Joffrey Jaffeux
 hide_plugin if self.respond_to?(:hide_plugin)
 
-register_asset 'javascripts/discourse-local-dates.js.no-module.es6'
+register_asset 'javascripts/discourse-local-dates.js.es6'
 register_asset 'stylesheets/common/discourse-local-dates.scss'
 register_asset 'moment.js', :vendored_core_pretty_text
 register_asset 'moment-timezone.js', :vendored_core_pretty_text

spec/lib/discourse_js_processor_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'discourse_js_processor'
+
+describe DiscourseJsProcessor do
+
+  describe 'should_transpile?' do
+    it "returns false for empty strings" do
+      expect(DiscourseJsProcessor.should_transpile?(nil)).to eq(false)
+      expect(DiscourseJsProcessor.should_transpile?('')).to eq(false)
+    end
+
+    it "returns false for a regular js file" do
+      expect(DiscourseJsProcessor.should_transpile?("file.js")).to eq(false)
+    end
+
+    it "returns true for deprecated .es6 files" do
+      expect(DiscourseJsProcessor.should_transpile?("file.es6")).to eq(true)
+      expect(DiscourseJsProcessor.should_transpile?("file.js.es6")).to eq(true)
+      expect(DiscourseJsProcessor.should_transpile?("file.js.es6.erb")).to eq(true)
+    end
+  end
+
+  describe "skip_module?" do
+    it "returns false for empty strings" do
+      expect(DiscourseJsProcessor.skip_module?(nil)).to eq(false)
+      expect(DiscourseJsProcessor.skip_module?('')).to eq(false)
+    end
+
+    it "returns true if the header is present" do
+      expect(DiscourseJsProcessor.skip_module?("// cool comment\n// discourse-skip-module")).to eq(true)
+    end
+
+    it "returns false if the header is not present" do
+      expect(DiscourseJsProcessor.skip_module?("// just some JS\nconsole.log()")).to eq(false)
+    end
+  end
+end