split lowlight languages out of the worker bundle

package.json

@@ -20,7 +20,8 @@
   },
   "resolutions": {
     "babel-core": "7.0.0-bridge.0",
-    "gitdiff-parser": "https://github.com/scm-manager/gitdiff-parser#ed3fe7de73dbb0a06c3e6adbbdf22dbae6e66351"
+    "gitdiff-parser": "https://github.com/scm-manager/gitdiff-parser#ed3fe7de73dbb0a06c3e6adbbdf22dbae6e66351",
+    "lowlight": "1.13.1"
   },
   "babel": {
     "presets": [

scm-ui/jest-preset/jest-preset.js

@@ -25,8 +25,7 @@ module.exports = {
   transformIgnorePatterns: ["node_modules/(?!(@scm-manager)/)"],
   moduleNameMapper: {
     "\\.(png|svg|jpg|gif|woff2?|eot|ttf)$": path.join(mockDirectory, "fileMock.js"),
-    "\\.(css|scss|sass)$": path.join(mockDirectory, "styleMock.js"),
-    "\\.worker$": path.join(mockDirectory, "workerMock.js")
+    "\\.(css|scss|sass)$": path.join(mockDirectory, "styleMock.js")
   },
   setupFiles: [path.resolve(__dirname, "src", "setup.js")],
   collectCoverage: true,

scm-ui/jest-preset/src/setup.js

@@ -1,2 +1,4 @@
 import registerRequireContextHook from "babel-plugin-require-context-hook/register";
+import Worker from "./__mocks__/workerMock";
 registerRequireContextHook();
+window.Worker = Worker;

scm-ui/ui-components/src/repos/Tokenize.worker.ts

@@ -1,8 +1,11 @@
-/* eslint-disable no-restricted-globals */
 // @ts-ignore we have no types for react-diff-view
 import { tokenize } from "react-diff-view";
 import refractor from "./refractorAdapter";
 
+// the WorkerGlobalScope is assigned to self
+// see https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/self
+declare const self: Worker;
+
 self.addEventListener("message", ({ data: { id, payload } }) => {
   const { hunks, language } = payload;
   const options = {
@@ -10,20 +13,27 @@ self.addEventListener("message", ({ data: { id, payload } }) => {
     language: language,
     refractor
   };
-  try {
-    const tokens = tokenize(hunks, options);
-    const payload = {
-      success: true,
-      tokens: tokens
-    };
-    // @ts-ignore seems to use wrong typing
-    self.postMessage({ id, payload });
-  } catch (ex) {
-    const payload = {
-      success: false,
-      reason: ex.message
-    };
-    // @ts-ignore seems to use wrong typing
-    self.postMessage({ id, payload });
+
+  const doTokenization = (worker: Worker) => {
+    try {
+      const tokens = tokenize(hunks, options);
+      const payload = {
+        success: true,
+        tokens: tokens
+      };
+      worker.postMessage({ id, payload });
+    } catch (ex) {
+      const payload = {
+        success: false,
+        reason: ex.message
+      };
+      worker.postMessage({ id, payload });
+    }
+  };
+
+  const createTokenizer = (worker: Worker) => () => doTokenization(worker);
+
+  if (options.highlight) {
+    refractor.loadLanguage(language, createTokenizer(self));
   }
 });

scm-ui/ui-components/src/repos/TokenizedDiffView.tsx

@@ -2,8 +2,6 @@ import React, { FC } from "react";
 import styled from "styled-components";
 // @ts-ignore we have no typings for react-diff-view
 import { Diff, useTokenizeWorker } from "react-diff-view";
-// @ts-ignore we use webpack worker-loader to load the web worker
-import TokenizeWorker from "./Tokenize.worker";
 import { File } from "./DiffTypes";
 
 // styling for the diff tokens
@@ -38,7 +36,7 @@ const DiffView = styled(Diff)`
 `;
 
 // WebWorker which creates tokens for syntax highlighting
-const tokenize = new TokenizeWorker();
+const tokenize = new Worker("./Tokenize.worker.ts", { name: "tokenizer", type: "module" });
 
 type Props = {
   file: File;

scm-ui/ui-components/src/repos/refractorAdapter.ts

@@ -1,11 +1,33 @@
-import lowlight from "lowlight";
+import lowlight from "lowlight/lib/core";
 
 // adapter to let lowlight look like refractor
 // this is required because react-diff-view does only support refractor,
 // but we want same highlighting as in the source code browser.
 
+const isLanguageRegistered = (lang: string) => {
+  // @ts-ignore listLanguages seems unknown to type
+  const registeredLanguages = lowlight.listLanguages();
+  return !!registeredLanguages[lang];
+};
+
+const loadLanguage = (lang: string, callback: () => void) => {
+  if (isLanguageRegistered(lang)) {
+    callback();
+  } else {
+    import(
+      /* webpackChunkName: "tokenizer-lowlight-[request]" */
+      `highlight.js/lib/languages/${lang}`
+    ).then(loadedLanguage => {
+      lowlight.registerLanguage(lang, loadedLanguage.default);
+      callback();
+    });
+  }
+};
+
 const refractorAdapter = {
   ...lowlight,
+  isLanguageRegistered,
+  loadLanguage,
   highlight: (value: string, language: string) => {
     return lowlight.highlight(language, value).value;
   }

scm-ui/ui-components/src/storyshots.test.ts

@@ -9,7 +9,6 @@ const createNodeMock = (element: any) => {
       querySelector: (selector: string) => {}
     };
   }
-  return null;
 };
 
 initStoryshots({
@@ -17,6 +16,7 @@ initStoryshots({
   // fix snapshot tests with react-diff-view which uses a ref on tr
   // @see https://github.com/storybookjs/storybook/pull/1090
   test: snapshotWithOptions({
+    // @ts-ignore types seems not to match
     createNodeMock
   })
 });

scm-ui/ui-scripts/package.json

@@ -12,7 +12,6 @@
   "dependencies": {
     "@pmmmwh/react-refresh-webpack-plugin": "^0.1.3",
     "babel-loader": "^8.0.6",
-    "cache-loader": "^4.1.0",
     "css-loader": "^3.2.0",
     "file-loader": "^4.2.0",
     "mini-css-extract-plugin": "^0.8.0",
@@ -23,11 +22,10 @@
     "sass-loader": "^8.0.0",
     "script-loader": "^0.7.2",
     "style-loader": "^1.0.0",
-    "thread-loader": "^2.1.3",
-    "worker-loader": "^2.0.0",
     "webpack": "^4.41.5",
     "webpack-cli": "^3.3.10",
-    "webpack-dev-server": "^3.10.1"
+    "webpack-dev-server": "^3.10.1",
+    "worker-plugin": "^3.2.0"
   },
   "eslintConfig": {
     "extends": "@scm-manager/eslint-config",
@@ -37,5 +35,6 @@
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "devDependencies": {}
 }

scm-ui/ui-scripts/src/webpack.config.js

@@ -3,14 +3,28 @@ const createIndexMiddleware = require("./middleware/IndexMiddleware");
 const createContextPathMiddleware = require("./middleware/ContextPathMiddleware");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
-const ReactRefreshWebpackPlugin = require("@pmmmwh/react-refresh-webpack-plugin");
+const WorkerPlugin = require("worker-plugin");
 
 const isDevelopment = process.env.NODE_ENV === "development";
 const root = path.resolve(process.cwd(), "scm-ui");
 
+const babelPlugins = [];
+const webpackPlugins = [new WorkerPlugin()];
+
+let mode = "production";
+
+if (isDevelopment) {
+  mode = "development";
+  babelPlugins.push(require.resolve("react-refresh/babel"));
+  const ReactRefreshWebpackPlugin = require("@pmmmwh/react-refresh-webpack-plugin");
+  webpackPlugins.push(new ReactRefreshWebpackPlugin());
+}
+
+console.log(`build ${mode} bundles`);
+
 module.exports = [
   {
-    mode: isDevelopment ? "development" : "production",
+    mode,
     context: root,
     entry: {
       webapp: [path.resolve(__dirname, "webpack-public-path.js"), "./ui-webapp/src/index.tsx"]
@@ -30,26 +44,16 @@ module.exports = [
             systemjs: false
           }
         },
-        {
-          test: /\.worker\.(j|t)s$/,
-          use: { loader: "worker-loader" }
-        },
         {
           test: /\.(js|ts|jsx|tsx)$/i,
           exclude: /node_modules/,
           use: [
-            {
-              loader: "cache-loader"
-            },
-            {
-              loader: "thread-loader"
-            },
             {
               loader: "babel-loader",
               options: {
                 cacheDirectory: true,
                 presets: ["@scm-manager/babel-preset"],
-                plugins: [isDevelopment && require.resolve("react-refresh/babel")].filter(Boolean)
+                plugins: babelPlugins
               }
             }
           ]
@@ -76,7 +80,8 @@ module.exports = [
     },
     output: {
       path: path.join(root, "target", "assets"),
-      filename: "[name].bundle.js"
+      filename: "[name].bundle.js",
+      chunkFilename: "[name].bundle.js"
     },
     devServer: {
       contentBase: path.join(root, "ui-webapp", "public"),
@@ -98,6 +103,7 @@ module.exports = [
     },
     optimization: {
       runtimeChunk: "single",
+      namedChunks: true,
       splitChunks: {
         chunks: "all",
         cacheGroups: {
@@ -114,7 +120,7 @@ module.exports = [
         }
       }
     },
-    plugins: [isDevelopment && new ReactRefreshWebpackPlugin()].filter(Boolean)
+    plugins: webpackPlugins
   },
   {
     context: root,