Use more accurate language detection for syntax highlighting (#1891)

Updated spotter to version 4 in order to get prism syntax mode for detected coding languages.
Expose syntax modes of coding languages as headers on content endpoint and as fields on diff dto.
Remove leading line break on search result fragments.
Use mark instead of span or strong for highlighted search results.
Add option to use syntax highlighting in TextHitField component.

Co-authored-by: Matthias Thieroff <matthias.thieroff@cloudogu.com>

gradle/changelog/search_highlighter.yaml

@@ -1,2 +1,4 @@
 - type: changed
   description: Keep whole lines for code highlighting in search ([#1871](https://github.com/scm-manager/scm-manager/pull/1871))
+- type: changed
+  description: Use more accurate language detection for syntax highlighting ([#1891](https://github.com/scm-manager/scm-manager/pull/1891))

gradle/dependencies.gradle

@@ -127,7 +127,7 @@ ext {
     webResources: 'com.github.sdorra:web-resources:1.1.1',
 
     // content type detection
-    spotter: 'com.github.sdorra:spotter-core:3.0.1',
+    spotter: 'com.cloudogu.spotter:spotter-core:4.0.0',
     tika: 'org.apache.tika:tika-core:1.25',
 
     // restart on unix

scm-core/src/main/java/sonia/scm/io/ContentType.java

@@ -24,6 +24,8 @@
 
 package sonia.scm.io;
 
+import java.util.Collections;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -68,4 +70,14 @@ public interface ContentType {
    * @return programming language or empty
    */
   Optional<String> getLanguage();
+
+  /**
+   * Returns a map of syntax modes such as codemirror, ace or prism.
+   *
+   * @return map of syntax modes
+   * @since 2.28.0
+   */
+  default Map<String, String> getSyntaxModes() {
+    return Collections.emptyMap();
+  }
 }

scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java

@@ -24,6 +24,9 @@
 
 package sonia.scm.io;
 
+import java.util.Collections;
+import java.util.Map;
+
 /**
  * ContentTypeResolver is able to detect the {@link ContentType} of files based on their path and (optinally) a few starting bytes. These files do not have to be real files on the file system, but can be hypothetical constructs ("What content type is most probable for a file named like this").
  *
@@ -35,7 +38,6 @@ public interface ContentTypeResolver {
    * Detects the {@link ContentType} of the given path, by only using path based strategies.
    *
    * @param path path of the file
-   *
    * @return {@link ContentType} of path
    */
   ContentType resolve(String path);
@@ -43,10 +45,19 @@ public interface ContentTypeResolver {
   /**
    * Detects the {@link ContentType} of the given path, by using path and content based strategies.
    *
-   * @param path path of the file
+   * @param path          path of the file
    * @param contentPrefix first few bytes of the content
-   *
    * @return {@link ContentType} of path and content prefix
    */
   ContentType resolve(String path, byte[] contentPrefix);
+
+  /**
+   * Returns a map of syntax highlighting modes such as ace, codemirror or prism by language.
+   * @param language name of the coding language
+   * @return map of syntax highlighting modes
+   * @since 2.28.0
+   */
+  default Map<String, String> findSyntaxModesByLanguage(String language) {
+    return Collections.emptyMap();
+  }
 }

scm-ui/ui-api/src/contentType.ts

@@ -28,6 +28,9 @@ import { ApiResultWithFetching } from "./base";
 export type ContentType = {
   type: string;
   language?: string;
+  aceMode?: string;
+  codemirrorMode?: string;
+  prismMode?: string;
 };
 
 function getContentType(url: string): Promise<ContentType> {
@@ -35,6 +38,9 @@ function getContentType(url: string): Promise<ContentType> {
     return {
       type: response.headers.get("Content-Type") || "application/octet-stream",
       language: response.headers.get("X-Programming-Language") || undefined,
+      aceMode: response.headers.get("X-Syntax-Mode-Ace") || undefined,
+      codemirrorMode: response.headers.get("X-Syntax-Mode-Codemirror") || undefined,
+      prismMode: response.headers.get("X-Syntax-Mode-Prism") || undefined,
     };
   });
 }

scm-ui/ui-components/src/SplitAndReplace.tsx

@@ -33,9 +33,10 @@ export type Replacement = {
 type Props = {
   text: string;
   replacements: Replacement[];
+  textWrapper?: (s: string) => ReactNode;
 };
 
-const textWrapper = (s: string) => {
+const defaultTextWrapper = (s: string) => {
   const first = s.startsWith(" ") ? <>&nbsp;</> : "";
   const last = s.endsWith(" ") ? <>&nbsp;</> : "";
   return (
@@ -47,7 +48,7 @@ const textWrapper = (s: string) => {
   );
 };
 
-const SplitAndReplace: FC<Props> = ({ text, replacements }) => {
+const SplitAndReplace: FC<Props> = ({ text, replacements, textWrapper = defaultTextWrapper }) => {
   const parts = textSplitAndReplace<ReactNode>(text, replacements, textWrapper);
   if (parts.length === 0) {
     return <>{parts[0]}</>;

scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx

@@ -48,7 +48,7 @@ storiesOf("SyntaxHighlighter", module)
   ))
   .add("Go", () => (
     <Spacing>
-      <SyntaxHighlighter language="golang" value={GoHttpServer} />
+      <SyntaxHighlighter language="go" value={GoHttpServer} />
     </Spacing>
   ))
   .add("Javascript", () => (

scm-ui/ui-components/src/__resources__/ContentSearchHit.ts

@@ -0,0 +1,67 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+import { Hit } from "@scm-manager/ui-types";
+
+export const javaHit: Hit = {
+  score: 2.5,
+  fields: {
+    content: {
+      highlighted: true,
+      fragments: [
+        "import org.slf4j.LoggerFactory;\n\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * Jwt implementation of {@link <|[[--AccessTokenBuilder--]]|>}.\n * \n * @author Sebastian Sdorra\n * @since 2.0.0\n */\npublic final class <|[[--JwtAccessTokenBuilder--]]|> implements <|[[--AccessTokenBuilder--]]|> {\n\n  /**\n   * the logger for <|[[--JwtAccessTokenBuilder--]]|>\n   */\n  private static final Logger LOG = LoggerFactory.getLogger(<|[[--JwtAccessTokenBuilder.class--]]|>);\n  \n  private final KeyGenerator keyGenerator; \n  private final SecureKeyResolver keyResolver; \n  \n  private String subject;\n  private String issuer;\n",
+        "  private final Map<String,Object> custom = Maps.newHashMap();\n  \n  <|[[--JwtAccessTokenBuilder--]]|>(KeyGenerator keyGenerator, SecureKeyResolver keyResolver)  {\n    this.keyGenerator = keyGenerator;\n    this.keyResolver = keyResolver;\n  }\n\n  @Override\n  public <|[[--JwtAccessTokenBuilder--]]|> subject(String subject) {\n",
+        '  public <|[[--JwtAccessTokenBuilder--]]|> custom(String key, Object value) {\n    Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "null or empty value not allowed");\n    Preconditions.checkArgument(value != null, "null or empty value not allowed");\n'
+      ]
+    }
+  },
+  _links: {}
+};
+
+export const bashHit: Hit = {
+  score: 2.5,
+  fields: {
+    content: {
+      highlighted: true,
+      fragments: [
+        '# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n#\n\n<|[[--getent--]]|> group scm >/dev/null || groupadd -r scm\n<|[[--getent--]]|> passwd scm >/dev/null || \\\n    useradd -r -g scm -M \\\n    -s /sbin/nologin -d /var/lib/scm \\\n    -c "user for the scm-server process" scm\nexit 0\n\n'
+      ]
+    }
+  },
+  _links: {}
+};
+
+export const markdownHit: Hit = {
+  score: 2.5,
+  fields: {
+    content: {
+      highlighted: true,
+      fragments: [
+        "---\ntitle: SCM-Manager v2 Test <|[[--Cases--]]|>\n---\n\nDescribes the expected behaviour for SCMM v2 REST Resources using manual tests.\n\nThe following states general test <|[[--cases--]]|> per HTTP Method and en expected return code as well as exemplary curl calls.\nResource-specifics are stated \n\n## Test <|[[--Cases--]]|>\n\n### GET\n\n- Collection Resource (e.g. `/users`)\n    - Without parameters -> 200\n    - Parameters\n        - `?pageSize=1` -> Only one embedded element, pageTotal reflects the correct number of pages, `last` link points to last page.\n        - `?pageSize=1&page=1` -> `next` link points to page 0 ; `prev` link points to page 2\n        - `?sortBy=admin` -> Sorted by `admin` field of embedded objects\n        - `?sortBy=admin&desc=true` -> Invert sorting\n- Individual Resource (e.g. `/users/scmadmin`)\n    - Exists  -> 200\n",
+        "\n### DELETE\n\n- existing -> 204\n- not existing -> 204\n- without permission -> 401\n\n## Exemplary calls & Resource specific test <|[[--cases--]]|>\n\nIn order to extend those tests to other Resources, have a look at the rest docs. Note that the Content Type is specific to each resource as well.\n"
+      ]
+    }
+  },
+  _links: {}
+};

scm-ui/ui-components/src/languages.test.ts

@@ -35,7 +35,7 @@ describe("syntax highlighter", () => {
     expect(java).toBe("java");
   });
 
-  it("should return text if language is undefied", () => {
+  it("should return text if language is undefined", () => {
     const lang = determineLanguage();
     expect(lang).toBe("text");
   });
@@ -45,8 +45,4 @@ describe("syntax highlighter", () => {
     expect(lang).toBe("text");
   });
 
-  it("should use alias go for golang", () => {
-    const go = determineLanguage("golang");
-    expect(go).toBe("go");
-  });
 });

scm-ui/ui-components/src/languages.ts

@@ -22,20 +22,11 @@
  * SOFTWARE.
  */
 
-// this aliases are only to map from spotter detection to prismjs
-const languageAliases: { [key: string]: string } = {
-  golang: "go",
-};
-
 export const defaultLanguage = "text";
 
 export const determineLanguage = (language?: string) => {
   if (!language) {
     return defaultLanguage;
   }
-  const lang = language.toLowerCase();
+  return language.toLowerCase();
-  if (languageAliases[lang]) {
-    return languageAliases[lang];
-  }
-  return lang;
 };

scm-ui/ui-components/src/repos/TokenizedDiffView.tsx

@@ -26,35 +26,46 @@ import styled from "styled-components";
 // @ts-ignore we have no typings for react-diff-view
 import { Diff, useTokenizeWorker } from "react-diff-view";
 import { FileDiff } from "@scm-manager/ui-types";
-import { determineLanguage } from "../languages";
 
 // @ts-ignore no types for css modules
 import theme from "../syntax-highlighting.module.css";
+import { determineLanguage } from "../languages";
 
 const DiffView = styled(Diff)`
   /* align line numbers */
+
   & .diff-gutter {
     text-align: right;
   }
+
   /* column sizing */
+
   > colgroup .diff-gutter-col {
     width: 3.25rem;
   }
+
   /* prevent following content from moving down */
+
   > .diff-gutter:empty:hover::after {
     font-size: 0.7rem;
   }
+
   /* smaller font size for code and
   ensure same monospace font throughout whole scmm */
+
   & .diff-line {
     font-size: 0.75rem;
     font-family: "Courier New", Monaco, Menlo, "Ubuntu Mono", "source-code-pro", monospace;
   }
+
   /* comment padding for sidebyside view */
+
   &.split .diff-widget-content .is-indented-line {
     padding-left: 3.25rem;
   }
+
   /* comment padding for combined view */
+
   &.unified .diff-widget-content .is-indented-line {
     padding-left: 6.5rem;
   }
@@ -71,10 +82,17 @@ type Props = {
   className?: string;
 };
 
+const findSyntaxHighlightingLanguage = (file: FileDiff) => {
+  if (file.syntaxModes) {
+    return file.syntaxModes["prism"] || file.syntaxModes["codemirror"] || file.syntaxModes["ace"] || file.language;
+  }
+  return file.language;
+};
+
 const TokenizedDiffView: FC<Props> = ({ file, viewType, className, children }) => {
   const { tokens } = useTokenizeWorker(tokenize, {
     hunks: file.hunks,
-    language: determineLanguage(file.language),
+    language: determineLanguage(findSyntaxHighlightingLanguage(file))
   });
 
   return (

scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx

@@ -67,7 +67,7 @@ const commitImplementMain = {
 };
 
 const source: AnnotatedSource = {
-  language: "golang",
+  language: "go",
   lines: [
     {
       lineNumber: 1,

scm-ui/ui-components/src/repos/refractorAdapter.ts

@@ -46,10 +46,15 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
       import(
         /* webpackChunkName: "tokenizer-refractor-[request]" */
         `refractor/lang/${lang}`
-      ).then((loadedLanguage) => {
+      )
-        refractor.register(loadedLanguage.default);
+        .then(loadedLanguage => {
-        callback();
+          refractor.register(loadedLanguage.default);
-      });
+          callback();
+        })
+        .catch(e => {
+          // eslint-disable-next-line no-console
+          console.log(`failed to load refractor language ${lang}: ${e}`);
+        });
     }
   };
 
@@ -58,7 +63,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
   const runHook = (name: string, env: RunHookEnv) => {
     originalRunHook.apply(name, env);
     if (env.classes) {
-      env.classes = env.classes.map((className) => theme[className] || className);
+      env.classes = env.classes.map(className => theme[className] || className);
     }
   };
   // @ts-ignore hooks are not in the type definition
@@ -67,7 +72,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
   return {
     isLanguageRegistered,
     loadLanguage,
-    ...refractor,
+    ...refractor
   };
 };
 

scm-ui/ui-components/src/search/HighlightedFragment.tsx

@@ -42,7 +42,7 @@ const HighlightedFragment: FC<Props> = ({ value }) => {
       if (start > 0) {
         result.push(content.substring(0, start));
       }
-      result.push(<strong>{content.substring(start + PRE_TAG.length, end)}</strong>);
+      result.push(<mark>{content.substring(start + PRE_TAG.length, end)}</mark>);
       content = content.substring(end + POST_TAG.length);
     } else {
       result.push(content);

scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx

@@ -0,0 +1,130 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+import React, { FC, ReactNode, useEffect, useMemo, useState } from "react";
+import createAdapter from "../repos/refractorAdapter";
+// @ts-ignore no types for css modules
+import theme from "../syntax-highlighting.module.css";
+import SplitAndReplace, { Replacement } from "../SplitAndReplace";
+import { AST, RefractorNode } from "refractor";
+import { determineLanguage } from "../languages";
+
+const PRE_TAG = "<|[[--";
+const POST_TAG = "--]]|>";
+const PRE_TAG_REGEX = /<\|\[\[--/g;
+const POST_TAG_REGEX = /--]]\|>/g;
+
+const adapter = createAdapter(theme);
+
+function createReplacement(textToReplace: string): Replacement {
+  return {
+    textToReplace,
+    replacement: <mark>{textToReplace}</mark>,
+    replaceAll: true
+  };
+}
+
+function mapWithDepth(depth: number, replacements: Replacement[]) {
+  return function mapChildrenWithDepth(child: RefractorNode, i: number) {
+    return mapChild(child, i, depth, replacements);
+  };
+}
+
+function isAstElement(node: RefractorNode): node is AST.Element {
+  return (node as AST.Element).tagName !== undefined;
+}
+
+function mapChild(child: RefractorNode, i: number, depth: number, replacements: Replacement[]): ReactNode {
+  if (isAstElement(child)) {
+    const className =
+      child.properties && Array.isArray(child.properties.className)
+        ? child.properties.className.join(" ")
+        : child.properties.className;
+
+    return React.createElement(
+      child.tagName,
+      Object.assign({ key: `fract-${depth}-${i}` }, child.properties, { className }),
+      child.children && child.children.map(mapWithDepth(depth + 1, replacements))
+    );
+  }
+
+  return (
+    <SplitAndReplace key={`fract-${depth}-${i}`} text={child.value} replacements={replacements} textWrapper={s => s} />
+  );
+}
+
+type Props = {
+  value: string;
+  language: string;
+};
+
+const stripAndReplace = (value: string) => {
+  const strippedValue = value.replace(PRE_TAG_REGEX, "").replace(POST_TAG_REGEX, "");
+  let content = value;
+
+  const result: string[] = [];
+  while (content.length > 0) {
+    const start = content.indexOf(PRE_TAG);
+    const end = content.indexOf(POST_TAG);
+    if (start >= 0 && end > 0) {
+      const item = content.substring(start + PRE_TAG.length, end);
+      if (!result.includes(item)) {
+        result.push(item);
+      }
+      content = content.substring(end + POST_TAG.length);
+    } else {
+      break;
+    }
+  }
+
+  result.sort((a, b) => b.length - a.length);
+
+  return {
+    strippedValue,
+    replacements: result.map(createReplacement)
+  };
+};
+
+const SyntaxHighlightedFragment: FC<Props> = ({ value, language }) => {
+  const [isLoading, setIsLoading] = useState(true);
+  const determinedLanguage = determineLanguage(language);
+  const { strippedValue, replacements } = useMemo(() => stripAndReplace(value), [value]);
+
+  useEffect(() => {
+    adapter.loadLanguage(determinedLanguage, () => {
+      setIsLoading(false);
+    });
+  }, [determinedLanguage]);
+
+  if (isLoading) {
+    return <SplitAndReplace text={strippedValue} replacements={replacements} textWrapper={s => s} />;
+  }
+
+  const refractorNodes = adapter.highlight(strippedValue, determinedLanguage);
+  const highlightedFragment = refractorNodes.map(mapWithDepth(0, replacements));
+
+  return <>{highlightedFragment}</>;
+};
+
+export default SyntaxHighlightedFragment;

scm-ui/ui-components/src/search/TextHitField.stories.tsx

@@ -0,0 +1,54 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+import React from "react";
+import { storiesOf } from "@storybook/react";
+import { bashHit, javaHit, markdownHit } from "../__resources__/ContentSearchHit";
+import TextHitField from "./TextHitField";
+
+storiesOf("TextHitField", module)
+  .add("Default", () => (
+    <pre>
+      <TextHitField hit={javaHit} field={"content"} />
+    </pre>
+  ))
+  .add("Java SyntaxHighlighting", () => (
+    <pre>
+      <TextHitField hit={javaHit} field={"content"} syntaxHighlightingLanguage="java" />
+    </pre>
+  ))
+  .add("Bash SyntaxHighlighting", () => (
+    <pre>
+      <TextHitField hit={bashHit} field={"content"} syntaxHighlightingLanguage="bash" />
+    </pre>
+  ))
+  .add("Markdown SyntaxHighlighting", () => (
+    <pre>
+      <TextHitField hit={markdownHit} field={"content"} syntaxHighlightingLanguage="markdown" />
+    </pre>
+  ))
+  .add("Unknown SyntaxHighlighting", () => (
+    <pre>
+      <TextHitField hit={bashHit} field={"content"} syntaxHighlightingLanguage="__unknown__" />
+    </pre>
+  ));

scm-ui/ui-components/src/search/TextHitField.tsx

@@ -26,35 +26,51 @@ import React, { FC } from "react";
 import { HighlightedHitField, Hit } from "@scm-manager/ui-types";
 import HighlightedFragment from "./HighlightedFragment";
 import { isHighlightedHitField } from "./fields";
+import SyntaxHighlightedFragment from "./SyntaxHighlightedFragment";
+
+type HighlightedTextFieldProps = {
+  field: HighlightedHitField;
+  syntaxHighlightingLanguage?: string;
+};
+
+const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field, syntaxHighlightingLanguage }) => {
+  const separator = syntaxHighlightingLanguage ? "...\n" : " ... ";
+  return (
+    <>
+      {field.fragments.map((fragment, i) => (
+        <React.Fragment key={fragment}>
+          {separator}
+          {syntaxHighlightingLanguage ? (
+            <SyntaxHighlightedFragment value={fragment} language={syntaxHighlightingLanguage} />
+          ) : (
+            <HighlightedFragment value={fragment} />
+          )}
+          {i + 1 >= field.fragments.length ? separator : null}
+        </React.Fragment>
+      ))}
+    </>
+  );
+};
 
 type Props = {
   hit: Hit;
   field: string;
   truncateValueAt?: number;
+  syntaxHighlightingLanguage?: string;
 };
 
-type HighlightedTextFieldProps = {
+const TextHitField: FC<Props> = ({
-  field: HighlightedHitField;
+  hit,
-};
+  field: fieldName,
-
+  children,
-const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field }) => (
+  syntaxHighlightingLanguage,
-  <>
+  truncateValueAt = 0
-    {field.fragments.map((fr, i) => (
+}) => {
-      <React.Fragment key={fr}>
-        {" ... "}
-        <HighlightedFragment value={fr} />
-        {i + 1 >= field.fragments.length ? " ... " : null}
-      </React.Fragment>
-    ))}
-  </>
-);
-
-const TextHitField: FC<Props> = ({ hit, field: fieldName, children, truncateValueAt = 0 }) => {
   const field = hit.fields[fieldName];
   if (!field) {
     return <>{children}</>;
   } else if (isHighlightedHitField(field)) {
-    return <HighlightedTextField field={field} />;
+    return <HighlightedTextField field={field} syntaxHighlightingLanguage={syntaxHighlightingLanguage} />;
   } else {
     let value = field.value;
     if (value === "") {

scm-ui/ui-types/src/Diff.ts

@@ -43,6 +43,7 @@ export type FileDiff = {
   oldRevision?: string;
   type: FileChangeType;
   language?: string;
+  syntaxModes?: { [mode: string]: string };
   // TODO does this property exists?
   isBinary?: boolean;
   _links?: Links;

scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx

@@ -79,7 +79,7 @@ const SwitchableMarkdownViewer: FC<Props> = ({ file, basePath }) => {
       {renderMarkdown ? (
         <MarkdownViewer content={content || ""} basePath={basePath} permalink={permalink} />
       ) : (
-        <SyntaxHighlighter language="MARKDOWN" value={content || ""} permalink={permalink} />
+        <SyntaxHighlighter language="markdown" value={content || ""} permalink={permalink} />
       )}
     </div>
   );

scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx

@@ -26,6 +26,7 @@ import React, { FC } from "react";
 import { File, Link, Repository } from "@scm-manager/ui-types";
 import { Annotate, ErrorNotification, Loading } from "@scm-manager/ui-components";
 import { useAnnotations, useContentType } from "@scm-manager/ui-api";
+import { determineSyntaxHighlightingLanguage } from "../utils/files";
 
 type Props = {
   file: File;
@@ -52,7 +53,8 @@ const AnnotateView: FC<Props> = ({ file, repository, revision }) => {
     return <Loading />;
   }
 
-  return <Annotate source={{ ...annotation, language: contentType.language }} repository={repository} />;
+  const language = determineSyntaxHighlightingLanguage(contentType);
+  return <Annotate source={{ ...annotation, language }} repository={repository} />;
 };
 
 export default AnnotateView;

scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx

@@ -31,7 +31,8 @@ import { File, Link, Repository } from "@scm-manager/ui-types";
 import { ErrorNotification, Loading, PdfViewer } from "@scm-manager/ui-components";
 import SwitchableMarkdownViewer from "../components/content/SwitchableMarkdownViewer";
 import styled from "styled-components";
-import { useContentType } from "@scm-manager/ui-api";
+import { ContentType, useContentType } from "@scm-manager/ui-api";
+import {determineSyntaxHighlightingLanguage} from "../utils/files";
 
 const NoSpacingSyntaxHighlighterContainer = styled.div`
   & pre {
@@ -46,6 +47,8 @@ type Props = {
   revision: string;
 };
 
+
+
 const SourcesView: FC<Props> = ({ file, repository, revision }) => {
   const { data: contentTypeData, error, isLoading } = useContentType((file._links.self as Link).href);
 
@@ -59,7 +62,8 @@ const SourcesView: FC<Props> = ({ file, repository, revision }) => {
 
   let sources;
 
-  const { type: contentType, language } = contentTypeData;
+  const language = determineSyntaxHighlightingLanguage(contentTypeData);
+  const contentType = contentTypeData.type;
   const basePath = `/repo/${repository.namespace}/${repository.name}/code/sources/${revision}/`;
   if (contentType.startsWith("image/")) {
     sources = <ImageViewer file={file} />;

scm-ui/ui-webapp/src/repos/sources/utils/files.ts

@@ -22,6 +22,7 @@
  * SOFTWARE.
  */
 import { File } from "@scm-manager/ui-types";
+import { ContentType } from "@scm-manager/ui-api";
 
 export const isRootPath = (path: string) => {
   return path === "" || path === "/";
@@ -40,3 +41,7 @@ export const isEmptyDirectory = (file: File) => {
   }
   return (file._embedded?.children?.length || 0) === 0;
 };
+
+export const determineSyntaxHighlightingLanguage = (data: ContentType) => {
+  return data.prismMode || data.codemirrorMode || data.aceMode || data.language;
+};

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java

@@ -54,6 +54,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.Locale;
 
 public class ContentResource {
 
@@ -211,6 +212,11 @@ public class ContentResource {
     contentType.getLanguage().ifPresent(
       language -> responseBuilder.header(ProgrammingLanguages.HEADER, language)
     );
+
+    contentType.getSyntaxModes().forEach((mode, lang) -> {
+      String modeName = mode.substring(0, 1).toUpperCase(Locale.ENGLISH) + mode.substring(1);
+      responseBuilder.header(ProgrammingLanguages.HEADER_SYNTAX_MODE_PREFIX + modeName, lang);
+    });
   }
 
   private byte[] getHead(String revision, String path, RepositoryService repositoryService) throws IOException {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java

@@ -32,6 +32,7 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 @EqualsAndHashCode(callSuper = false)
@@ -63,6 +64,7 @@ public class DiffResultDto extends HalRepresentation {
     private String oldMode;
     private String type;
     private String language;
+    private Map<String, String> syntaxModes;
     private List<HunkDto> hunks;
 
   }

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java

@@ -26,6 +26,7 @@ package sonia.scm.api.v2.resources;
 
 import com.google.inject.Inject;
 import de.otto.edison.hal.Links;
+import sonia.scm.io.ContentType;
 import sonia.scm.io.ContentTypeResolver;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.DiffFile;
@@ -155,8 +156,10 @@ class DiffResultToDiffResultDtoMapper {
     dto.setOldPath(oldPath);
     dto.setOldRevision(file.getOldRevision());
 
-    Optional<String> language = contentTypeResolver.resolve(path).getLanguage();
+    ContentType contentType = contentTypeResolver.resolve(path);
+    Optional<String> language = contentType.getLanguage();
     language.ifPresent(dto::setLanguage);
+    dto.setSyntaxModes(contentType.getSyntaxModes());
 
     List<DiffResultDto.HunkDto> hunks = new ArrayList<>();
     for (Hunk hunk : file) {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java

@@ -28,6 +28,8 @@ final class ProgrammingLanguages {
 
   static final String HEADER = "X-Programming-Language";
 
+  static final String HEADER_SYNTAX_MODE_PREFIX = "X-Syntax-Mode-";
+
   private ProgrammingLanguages() {
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java

@@ -24,15 +24,18 @@
 
 package sonia.scm.io;
 
+import com.cloudogu.spotter.Language;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Collections;
+import java.util.Map;
 import java.util.Optional;
 
 public class DefaultContentType implements ContentType {
 
-  private static final String DEFAULT_LANG_MODE = "text";
+  private final com.cloudogu.spotter.ContentType contentType;
 
-  private final com.github.sdorra.spotter.ContentType contentType;
+  DefaultContentType(com.cloudogu.spotter.ContentType contentType) {
-
-  DefaultContentType(com.github.sdorra.spotter.ContentType contentType) {
     this.contentType = contentType;
   }
 
@@ -58,9 +61,23 @@ public class DefaultContentType implements ContentType {
 
   @Override
   public Optional<String> getLanguage() {
-    return contentType.getLanguage().map(language -> {
+    return contentType.getLanguage().map(Language::getName);
-      Optional<String> aceMode = language.getAceMode();
+  }
-      return aceMode.orElseGet(() -> language.getCodemirrorMode().orElse(DEFAULT_LANG_MODE));
+
-    });
+  @Override
+  public Map<String, String> getSyntaxModes() {
+    Optional<Language> language = contentType.getLanguage();
+    if (language.isPresent()) {
+      return syntaxMode(language.get());
+    }
+    return Collections.emptyMap();
+  }
+
+  static Map<String, String> syntaxMode(Language language) {
+    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+    language.getAceMode().ifPresent(mode -> builder.put("ace", mode));
+    language.getCodemirrorMode().ifPresent(mode -> builder.put("codemirror", mode));
+    language.getPrismMode().ifPresent(mode -> builder.put("prism", mode));
+    return builder.build();
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java

@@ -24,17 +24,32 @@
 
 package sonia.scm.io;
 
-import com.github.sdorra.spotter.ContentTypeDetector;
+import com.cloudogu.spotter.ContentTypeDetector;
-import com.github.sdorra.spotter.Language;
+import com.cloudogu.spotter.Language;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
 
 public final class DefaultContentTypeResolver implements ContentTypeResolver {
 
+  private static final Language[] BOOST = new Language[]{
+    // GCC Machine Description uses .md as extension, but markdown is much more likely
+    Language.MARKDOWN,
+    // XML uses .rs as extension, but rust is much more likely
+    Language.RUST,
+    // XML is also returned by content type boost strategy, but rust is really much more likely
+    Language.RUST,
+  };
+
   private static final ContentTypeDetector PATH_BASED = ContentTypeDetector.builder()
-    .defaultPathBased().boost(Language.MARKDOWN)
+    .defaultPathBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   private static final ContentTypeDetector PATH_AND_CONTENT_BASED = ContentTypeDetector.builder()
-    .defaultPathAndContentBased().boost(Language.MARKDOWN)
+    .defaultPathAndContentBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   @Override
@@ -46,4 +61,13 @@ public final class DefaultContentTypeResolver implements ContentTypeResolver {
   public DefaultContentType resolve(String path, byte[] contentPrefix) {
     return new DefaultContentType(PATH_AND_CONTENT_BASED.detect(path, contentPrefix));
   }
+
+  @Override
+  public Map<String, String> findSyntaxModesByLanguage(String language) {
+    Optional<Language> byName = Language.getByName(language);
+    if (byName.isPresent()) {
+      return DefaultContentType.syntaxMode(byName.get());
+    }
+    return Collections.emptyMap();
+  }
 }

scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java

@@ -90,11 +90,21 @@ public final class LuceneHighlighter {
     int index = content.indexOf(raw);
 
     int start = content.lastIndexOf('\n', index);
-    if (start < 0) {
-      start = 0;
-    }
 
-    String snippet = content.substring(start, index) + fragment;
+    String snippet;
+    if (start == index) {
+      // fragment starts with a linebreak
+      snippet = fragment.substring(1);
+    } else {
+      if (start < 0) {
+        // no leading linebreak
+        start = 0;
+      } else if (start < content.length()) {
+        // skip linebreak
+        start++;
+      }
+      snippet = content.substring(start, index) + fragment;
+    }
 
     int end = content.indexOf('\n', index + raw.length());
     if (end < 0) {

scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java

@@ -141,7 +141,7 @@ public class ContentResourceTest {
     Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
     assertEquals(200, response.getStatus());
 
-    assertEquals("golang", response.getHeaderString("X-Programming-Language"));
+    assertEquals("Go", response.getHeaderString("X-Programming-Language"));
     assertEquals("text/x-go", response.getHeaderString("Content-Type"));
   }
 
@@ -152,10 +152,22 @@ public class ContentResourceTest {
     Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "Dockerfile", null, null);
     assertEquals(200, response.getStatus());
 
-    assertEquals("dockerfile", response.getHeaderString("X-Programming-Language"));
+    assertEquals("Dockerfile", response.getHeaderString("X-Programming-Language"));
     assertEquals("text/plain", response.getHeaderString("Content-Type"));
   }
 
+  @Test
+  public void shouldRecognizeSyntaxModes() throws Exception {
+    mockContentFromResource("SomeGoCode.go");
+
+    Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
+    assertEquals(200, response.getStatus());
+
+    assertEquals("golang", response.getHeaderString("X-Syntax-Mode-Ace"));
+    assertEquals("go", response.getHeaderString("X-Syntax-Mode-Codemirror"));
+    assertEquals("go", response.getHeaderString("X-Syntax-Mode-Prism"));
+  }
+
   @Test
   public void shouldHandleRandomByteFile() throws Exception {
     mockContentFromResource("JustBytes");
@@ -190,6 +202,7 @@ public class ContentResourceTest {
     assertEquals("application/octet-stream", response.getHeaderString("Content-Type"));
   }
 
+  @SuppressWarnings("UnstableApiUsage")
   private void mockContentFromResource(String fileName) throws Exception {
     URL url = Resources.getResource(fileName);
     mockContent(fileName, Resources.toByteArray(url));

scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java

@@ -60,11 +60,16 @@ class DiffResultToDiffResultDtoMapperTest {
     DiffResultDto dto = mapper.mapForRevision(REPOSITORY, createResult(), "123");
 
     List<DiffResultDto.FileDto> files = dto.getFiles();
-    assertAddedFile(files.get(0), "A.java", "abc", "java");
+    assertAddedFile(files.get(0), "A.java", "abc", "Java");
-    assertModifiedFile(files.get(1), "B.ts", "abc", "def", "typescript");
+    assertModifiedFile(files.get(1), "B.ts", "abc", "def", "TypeScript");
-    assertDeletedFile(files.get(2), "C.go", "ghi", "golang");
+    DiffResultDto.FileDto cGo = files.get(2);
-    assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "typescript");
+    assertDeletedFile(cGo, "C.go", "ghi", "Go");
-    assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "typescript");
+    assertThat(cGo.getSyntaxModes())
+      .containsEntry("ace", "golang")
+      .containsEntry("codemirror", "go")
+      .containsEntry("prism", "go");
+    assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "TypeScript");
+    assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "TypeScript");
 
     DiffResultDto.HunkDto hunk = files.get(1).getHunks().get(0);
     assertHunk(hunk, "@@ -3,4 1,2 @@", 1, 2, 3, 4);

scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java

@@ -24,15 +24,15 @@
 
 package sonia.scm.io;
 
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Map;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
 
 class DefaultContentTypeResolverTest {
 
@@ -84,42 +84,45 @@ class DefaultContentTypeResolverTest {
         "% Which does not start with markdown"
       );
       ContentType contentType = contentTypeResolver.resolve("somedoc.md", content.getBytes(StandardCharsets.UTF_8));
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveMarkdownWithoutContent() {
       ContentType contentType = contentTypeResolver.resolve("somedoc.md");
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveMarkdownEvenWithDotsInFilename() {
       ContentType contentType = contentTypeResolver.resolve("somedoc.1.1.md");
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveDockerfile() {
       ContentType contentType = contentTypeResolver.resolve("Dockerfile");
-      Assertions.assertThat(contentType.getLanguage()).contains("dockerfile");
+      assertThat(contentType.getLanguage()).contains("Dockerfile");
     }
 
+  }
+
+  @Nested
+  class GetSyntaxModesTests {
 
     @Test
-    void shouldReturnAceModeIfPresent() {
+    void shouldReturnEmptyMapOfModesWithoutLanguage() {
-      assertThat(contentTypeResolver.resolve("app.go").getLanguage()).contains("golang"); // codemirror is just go
+      Map<String, String> syntaxModes = contentTypeResolver.resolve("app.exe").getSyntaxModes();
-      assertThat(contentTypeResolver.resolve("App.java").getLanguage()).contains("java"); // codemirror is clike
+      assertThat(syntaxModes).isEmpty();
     }
 
     @Test
-    void shouldReturnCodemirrorIfAceModeIsMissing() {
+    void shouldReturnMapOfModes() {
-      assertThat(contentTypeResolver.resolve("index.ecr").getLanguage()).contains("htmlmixed");
+      Map<String, String> syntaxModes = contentTypeResolver.resolve("app.rs").getSyntaxModes();
-    }
+      assertThat(syntaxModes)
-
+        .containsEntry("ace", "rust")
-    @Test
+        .containsEntry("codemirror", "rust")
-    void shouldReturnTextIfNoModeIsPresent() {
+        .containsEntry("prism", "rust");
-      assertThat(contentTypeResolver.resolve("index.hxml").getLanguage()).contains("text");
     }
 
   }

scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java

@@ -47,8 +47,6 @@ import static org.mockito.Mockito.when;
 @ExtendWith(MockitoExtension.class)
 class LuceneHighlighterTest {
 
-
-
   @Test
   void shouldHighlightText() throws InvalidTokenOffsetsException, IOException {
     StandardAnalyzer analyzer = new StandardAnalyzer();
@@ -80,6 +78,15 @@ class LuceneHighlighterTest {
     );
   }
 
+  @Test
+  void shouldNotStartHighlightedFragmentWithLineBreak() throws IOException, InvalidTokenOffsetsException {
+    String[] snippets = highlightCode("GameOfLife.java", "die");
+
+    assertThat(snippets).hasSize(1).allSatisfy(
+      snippet -> assertThat(snippet).doesNotStartWith("\n")
+    );
+  }
+
   @Test
   void shouldHighlightCodeInTsx() throws IOException, InvalidTokenOffsetsException {
     String[] snippets = highlightCode("Button.tsx", "inherit");