split lowlight languages out of the worker bundle

scm-ui/ui-components/src/repos/Tokenize.worker.ts

@@ -1,8 +1,11 @@
-/* eslint-disable no-restricted-globals */
 // @ts-ignore we have no types for react-diff-view
 import { tokenize } from "react-diff-view";
 import refractor from "./refractorAdapter";
 
+// the WorkerGlobalScope is assigned to self
+// see https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/self
+declare const self: Worker;
+
 self.addEventListener("message", ({ data: { id, payload } }) => {
   const { hunks, language } = payload;
   const options = {
@@ -10,20 +13,27 @@ self.addEventListener("message", ({ data: { id, payload } }) => {
     language: language,
     refractor
   };
-  try {
-    const tokens = tokenize(hunks, options);
-    const payload = {
-      success: true,
-      tokens: tokens
-    };
-    // @ts-ignore seems to use wrong typing
-    self.postMessage({ id, payload });
-  } catch (ex) {
-    const payload = {
-      success: false,
-      reason: ex.message
-    };
-    // @ts-ignore seems to use wrong typing
-    self.postMessage({ id, payload });
+
+  const doTokenization = (worker: Worker) => {
+    try {
+      const tokens = tokenize(hunks, options);
+      const payload = {
+        success: true,
+        tokens: tokens
+      };
+      worker.postMessage({ id, payload });
+    } catch (ex) {
+      const payload = {
+        success: false,
+        reason: ex.message
+      };
+      worker.postMessage({ id, payload });
+    }
+  };
+
+  const createTokenizer = (worker: Worker) => () => doTokenization(worker);
+
+  if (options.highlight) {
+    refractor.loadLanguage(language, createTokenizer(self));
   }
 });

scm-ui/ui-components/src/repos/TokenizedDiffView.tsx

@@ -2,8 +2,6 @@ import React, { FC } from "react";
 import styled from "styled-components";
 // @ts-ignore we have no typings for react-diff-view
 import { Diff, useTokenizeWorker } from "react-diff-view";
-// @ts-ignore we use webpack worker-loader to load the web worker
-import TokenizeWorker from "./Tokenize.worker";
 import { File } from "./DiffTypes";
 
 // styling for the diff tokens
@@ -38,7 +36,7 @@ const DiffView = styled(Diff)`
 `;
 
 // WebWorker which creates tokens for syntax highlighting
-const tokenize = new TokenizeWorker();
+const tokenize = new Worker("./Tokenize.worker.ts", { name: "tokenizer", type: "module" });
 
 type Props = {
   file: File;

scm-ui/ui-components/src/repos/refractorAdapter.ts

@@ -1,11 +1,33 @@
-import lowlight from "lowlight";
+import lowlight from "lowlight/lib/core";
 
 // adapter to let lowlight look like refractor
 // this is required because react-diff-view does only support refractor,
 // but we want same highlighting as in the source code browser.
 
+const isLanguageRegistered = (lang: string) => {
+  // @ts-ignore listLanguages seems unknown to type
+  const registeredLanguages = lowlight.listLanguages();
+  return !!registeredLanguages[lang];
+};
+
+const loadLanguage = (lang: string, callback: () => void) => {
+  if (isLanguageRegistered(lang)) {
+    callback();
+  } else {
+    import(
+      /* webpackChunkName: "tokenizer-lowlight-[request]" */
+      `highlight.js/lib/languages/${lang}`
+    ).then(loadedLanguage => {
+      lowlight.registerLanguage(lang, loadedLanguage.default);
+      callback();
+    });
+  }
+};
+
 const refractorAdapter = {
   ...lowlight,
+  isLanguageRegistered,
+  loadLanguage,
   highlight: (value: string, language: string) => {
     return lowlight.highlight(language, value).value;
   }

scm-ui/ui-components/src/storyshots.test.ts

@@ -9,7 +9,6 @@ const createNodeMock = (element: any) => {
       querySelector: (selector: string) => {}
     };
   }
-  return null;
 };
 
 initStoryshots({
@@ -17,6 +16,7 @@ initStoryshots({
   // fix snapshot tests with react-diff-view which uses a ref on tr
   // @see https://github.com/storybookjs/storybook/pull/1090
   test: snapshotWithOptions({
+    // @ts-ignore types seems not to match
     createNodeMock
   })
 });