DEV: Use WebPack stats plugin to map entrypoints to chunks (#24239)

Previously, we were parsing webpack JS chunk filenames from the HTML files which ember-cli generates. This worked ok for simple entrypoints, but falls apart once we start using async imports(), which are not included in the HTML.

This commit uses the stats plugin to generate an assets.json file, and updates Rails to parse it instead of the HTML. Caching on the Rails side is also improved to avoid reading from the filesystem multiple times per request in develoment.

Co-authored-by: Godfrey Chan <godfreykfc@gmail.com>

app/assets/javascripts/discourse/app/index.html

@@ -24,14 +24,9 @@
     <!-- bootstrap-content head -->
     {{content-for "head"}}
 
-    <discourse-chunked-script entrypoint="vendor">
-      <script defer src="{{rootURL}}assets/vendor.js"></script>
-    </discourse-chunked-script>
+    <script defer src="{{rootURL}}assets/vendor.js"></script>
 
-    <discourse-chunked-script entrypoint="discourse">
-      <ember-auto-import-scripts defer entrypoint="app"></ember-auto-import-scripts>
-      <script defer src="{{rootURL}}assets/discourse.js"></script>
-    </discourse-chunked-script>
+    <script defer src="{{rootURL}}assets/discourse.js"></script>
 
     <!-- bootstrap-content locale-script -->
   </head>

app/assets/javascripts/discourse/ember-cli-build.js

@@ -13,6 +13,7 @@ const DeprecationSilencer = require("deprecation-silencer");
 const generateWorkboxTree = require("./lib/workbox-tree-builder");
 const { compatBuild } = require("@embroider/compat");
 const { Webpack } = require("@embroider/webpack");
+const { StatsWriterPlugin } = require("webpack-stats-plugin");
 
 process.env.BROCCOLI_ENABLED_MEMOIZE = true;
 
@@ -135,6 +136,13 @@ module.exports = function (defaults) {
         output: {
           publicPath: "auto",
         },
+        entry: {
+          "assets/discourse.js/features/markdown-it.js": {
+            import: "./static/markdown-it",
+            dependOn: "assets/discourse.js",
+            runtime: false,
+          },
+        },
         externals: [
           function ({ request }, callback) {
             if (
@@ -175,6 +183,39 @@ module.exports = function (defaults) {
             },
           ],
         },
+        plugins: [
+          // The server use this output to map each asset to its chunks
+          new StatsWriterPlugin({
+            filename: "assets.json",
+            stats: {
+              all: false,
+              entrypoints: true,
+            },
+            transform({ entrypoints }) {
+              let names = Object.keys(entrypoints);
+              let output = {};
+
+              for (let name of names.sort()) {
+                let assets = entrypoints[name].assets.map(
+                  (asset) => asset.name
+                );
+
+                let parent = names.find((parentName) =>
+                  name.startsWith(parentName + "/")
+                );
+
+                if (parent) {
+                  name = name.slice(parent.length + 1);
+                  output[parent][name] = { assets };
+                } else {
+                  output[name] = { assets };
+                }
+              }
+
+              return JSON.stringify(output, null, 2);
+            },
+          }),
+        ],
       },
     },
   });

app/assets/javascripts/discourse/package.json

@@ -130,6 +130,7 @@
     "util": "^0.12.5",
     "virtual-dom": "^2.1.1",
     "webpack": "^5.89.0",
+    "webpack-stats-plugin": "^1.1.3",
     "wizard": "1.0.0",
     "workbox-cacheable-response": "^7.0.0",
     "workbox-core": "^7.0.0",

app/assets/javascripts/discourse/tests/index.html

@@ -45,19 +45,12 @@
     {{content-for "body"}} {{content-for "test-body"}}
 
     <script src="/testem.js" integrity="" data-embroider-ignore></script>
-    <discourse-chunked-script entrypoint="vendor">
-      <script src="{{rootURL}}assets/vendor.js"></script>
-    </discourse-chunked-script>
 
-    <discourse-chunked-script entrypoint="test-support">
-      <script src="{{rootURL}}assets/test-support.js"></script>
-      <ember-auto-import-scripts entrypoint="tests"></ember-auto-import-scripts>
-    </discourse-chunked-script>
+    <script src="{{rootURL}}assets/vendor.js"></script>
 
-    <discourse-chunked-script entrypoint="discourse-for-tests">
-      <ember-auto-import-scripts entrypoint="app"></ember-auto-import-scripts>
-      <script src="{{rootURL}}assets/discourse.js"></script>
-    </discourse-chunked-script>
+    <script src="{{rootURL}}assets/test-support.js"></script>
+
+    <script src="{{rootURL}}assets/discourse.js"></script>
 
     <script src="{{rootURL}}assets/test-i18n.js" data-embroider-ignore></script>
     <script src="{{rootURL}}assets/test-site-settings.js" data-embroider-ignore></script>

app/assets/javascripts/yarn.lock

@@ -10789,6 +10789,11 @@ webpack-sources@^3.2.3:
   resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde"
   integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==
 
+webpack-stats-plugin@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/webpack-stats-plugin/-/webpack-stats-plugin-1.1.3.tgz#ebcc36c8b468074ad737882e2043c1ce4b55d928"
+  integrity sha512-yUKYyy+e0iF/w31QdfioRKY+h3jDBRpthexBOWGKda99iu2l/wxYsI/XqdlP5IU58/0KB9CsJZgWNAl+/MPkRw==
+
 webpack@^5.89.0:
   version "5.89.0"
   resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.89.0.tgz#56b8bf9a34356e93a6625770006490bf3a7f32dc"

app/views/qunit/theme.html.erb

@@ -8,7 +8,8 @@
     <%- if @has_test_bundle && !@suggested_themes %>
       <%= preload_script "vendor" %>
       <%= preload_script "test-support" %>
-      <%= preload_script "discourse-for-tests" %>
+      <%= preload_script "discourse" %>
+      <%= preload_script "test" %>
       <%= preload_script "locales/#{I18n.locale}" %>
       <%= preload_script "admin" %>
       <%- Discourse.find_plugin_js_assets(include_disabled: true).each do |file| %>

lib/ember_cli.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
-module EmberCli
+class EmberCli < ActiveSupport::CurrentAttributes
+  # Cache which persists for the duration of a request
+  attribute :request_cached_script_chunks
+
   def self.dist_dir
     "#{Rails.root}/app/assets/javascripts/discourse/dist"
   end
@@ -10,30 +13,23 @@ module EmberCli
   end
 
   def self.script_chunks
-    return @chunk_infos if @chunk_infos
+    return @production_chunk_infos if @production_chunk_infos
+    return self.request_cached_script_chunks if self.request_cached_script_chunks
 
-    chunk_infos = {}
+    chunk_infos = JSON.parse(File.read("#{dist_dir}/assets.json"))
 
-    begin
-      test_html = File.read("#{dist_dir}/tests/index.html")
-      chunk_infos.merge! parse_chunks_from_html(test_html)
-    rescue Errno::ENOENT
-      # production build
+    chunk_infos.transform_keys! { |key| key.delete_prefix("assets/").delete_suffix(".js") }
+
+    chunk_infos.transform_values! do |value|
+      value["assets"].map { |chunk| chunk.delete_prefix("assets/").delete_suffix(".js") }
     end
 
-    index_html = File.read("#{dist_dir}/index.html")
-    chunk_infos.merge! parse_chunks_from_html(index_html)
-
-    @chunk_infos = chunk_infos if Rails.env.production?
-    chunk_infos
+    @production_chunk_infos = chunk_infos if Rails.env.production?
+    self.request_cached_script_chunks = chunk_infos
   rescue Errno::ENOENT
     {}
   end
 
-  def self.parse_source_map_path(file)
-    File.read("#{dist_dir}/assets/#{file}")[%r{//# sourceMappingURL=(.*)$}, 1]
-  end
-
   def self.is_ember_cli_asset?(name)
     assets.include?(name) || script_chunks.values.flatten.include?(name.delete_suffix(".js"))
   end
@@ -56,31 +52,13 @@ module EmberCli
       end
   end
 
-  def self.parse_chunks_from_html(html)
-    doc = Nokogiri::HTML5.parse(html)
-
-    chunk_infos = {}
-
-    doc
-      .css("discourse-chunked-script")
-      .each do |discourse_script|
-        entrypoint = discourse_script.attr("entrypoint")
-        chunk_infos[entrypoint] = discourse_script
-          .css("script[src]")
-          .map do |script|
-            script.attr("src").delete_prefix("#{Discourse.base_path}/assets/").delete_suffix(".js")
-          end
-      end
-
-    chunk_infos
-  end
-
   def self.has_tests?
     File.exist?("#{dist_dir}/tests/index.html")
   end
 
   def self.clear_cache!
-    @chunk_infos = nil
+    @prod_chunk_infos = nil
     @assets = nil
+    self.request_cached_script_chunks = nil
   end
 end

spec/lib/ember_cli_spec.rb

@@ -6,39 +6,4 @@ describe EmberCli do
       expect(EmberCli.ember_version).to match(/\A\d+\.\d+/)
     end
   end
-
-  describe ".parse_chunks_from_html" do
-    def generate_html
-      <<~HTML
-        <html>
-          <head>
-            <discourse-chunked-script entrypoint="discourse">
-              <script src="#{Discourse.base_path}/assets/firstchunk.js"></script>
-              <script src="#{Discourse.base_path}/assets/secondchunk.js"></script>
-            </discourse-chunked-script>
-          </head>
-          <body>
-            Hello world
-          </body>
-        </html>
-      HTML
-    end
-
-    it "can parse chunks for a normal site" do
-      chunks = EmberCli.parse_chunks_from_html generate_html
-      expect(chunks["discourse"]).to eq(%w[firstchunk secondchunk])
-    end
-
-    it "can parse chunks for a subfolder site" do
-      set_subfolder "/discuss"
-
-      html = generate_html
-
-      # sanity check that our fixture is working
-      expect(html).to include("/discuss/assets/firstchunk.js")
-
-      chunks = EmberCli.parse_chunks_from_html html
-      expect(chunks["discourse"]).to eq(%w[firstchunk secondchunk])
-    end
-  end
 end