Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-11 16:05:44 +01:00
Code Issues Packages Projects Releases Activity

Use more accurate language detection for syntax highlighting (#1891)

Browse Source 
Updated spotter to version 4 in order to get prism syntax mode for detected coding languages.
Expose syntax modes of coding languages as headers on content endpoint and as fields on diff dto.
Remove leading line break on search result fragments.
Use mark instead of span or strong for highlighted search results.
Add option to use syntax highlighting in TextHitField component.

Co-authored-by: Matthias Thieroff <matthias.thieroff@cloudogu.com>
This commit is contained in:
Sebastian Sdorra
2021-12-13 17:03:08 +01:00
committed by GitHub
parent 6eba01161f
commit e2d63cc2a1
34 changed files with 809 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/changelog/search_highlighter.yaml
View File

@@ -1,2 +1,4 @@
- type: changed
description: Keep whole lines for code highlighting in search ([#1871](https://github.com/scm-manager/scm-manager/pull/1871))
- type: changed
description: Use more accurate language detection for syntax highlighting ([#1891](https://github.com/scm-manager/scm-manager/pull/1891))

2
gradle/dependencies.gradle
View File

@@ -127,7 +127,7 @@ ext {
webResources: 'com.github.sdorra:web-resources:1.1.1',
// content type detection
spotter: 'com.github.sdorra:spotter-core:3.0.1',
spotter: 'com.cloudogu.spotter:spotter-core:4.0.0',
tika: 'org.apache.tika:tika-core:1.25',
// restart on unix

12
scm-core/src/main/java/sonia/scm/io/ContentType.java
View File

@@ -24,6 +24,8 @@
package sonia.scm.io;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
/**
@@ -68,4 +70,14 @@ public interface ContentType {
* @return programming language or empty
*/
Optional<String> getLanguage();
/**
* Returns a map of syntax modes such as codemirror, ace or prism.
*
* @return map of syntax modes
* @since 2.28.0
*/
default Map<String, String> getSyntaxModes() {
return Collections.emptyMap();
}
}

15
scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java
View File

@@ -24,6 +24,9 @@
package sonia.scm.io;
import java.util.Collections;
import java.util.Map;
/**
* ContentTypeResolver is able to detect the {@link ContentType} of files based on their path and (optinally) a few starting bytes. These files do not have to be real files on the file system, but can be hypothetical constructs ("What content type is most probable for a file named like this").
*
@@ -35,7 +38,6 @@ public interface ContentTypeResolver {
* Detects the {@link ContentType} of the given path, by only using path based strategies.
*
* @param path path of the file
*
* @return {@link ContentType} of path
*/
ContentType resolve(String path);
@@ -45,8 +47,17 @@ public interface ContentTypeResolver {
*
* @param path path of the file
* @param contentPrefix first few bytes of the content
*
* @return {@link ContentType} of path and content prefix
*/
ContentType resolve(String path, byte[] contentPrefix);
/**
* Returns a map of syntax highlighting modes such as ace, codemirror or prism by language.
* @param language name of the coding language
* @return map of syntax highlighting modes
* @since 2.28.0
*/
default Map<String, String> findSyntaxModesByLanguage(String language) {
return Collections.emptyMap();
}
}

6
scm-ui/ui-api/src/contentType.ts
View File

@@ -28,6 +28,9 @@ import { ApiResultWithFetching } from "./base";
export type ContentType = {
type: string;
language?: string;
aceMode?: string;
codemirrorMode?: string;
prismMode?: string;
};
function getContentType(url: string): Promise<ContentType> {
@@ -35,6 +38,9 @@ function getContentType(url: string): Promise<ContentType> {
return {
type: response.headers.get("Content-Type") || "application/octet-stream",
language: response.headers.get("X-Programming-Language") || undefined,
aceMode: response.headers.get("X-Syntax-Mode-Ace") || undefined,
codemirrorMode: response.headers.get("X-Syntax-Mode-Codemirror") || undefined,
prismMode: response.headers.get("X-Syntax-Mode-Prism") || undefined,
};
});
}

5
scm-ui/ui-components/src/SplitAndReplace.tsx
View File

@@ -33,9 +33,10 @@ export type Replacement = {
type Props = {
text: string;
replacements: Replacement[];
textWrapper?: (s: string) => ReactNode;
};
const textWrapper = (s: string) => {
const defaultTextWrapper = (s: string) => {
const first = s.startsWith(" ") ? <>&nbsp;</> : "";
const last = s.endsWith(" ") ? <>&nbsp;</> : "";
return (
@@ -47,7 +48,7 @@ const textWrapper = (s: string) => {
);
};
const SplitAndReplace: FC<Props> = ({ text, replacements }) => {
const SplitAndReplace: FC<Props> = ({ text, replacements, textWrapper = defaultTextWrapper }) => {
const parts = textSplitAndReplace<ReactNode>(text, replacements, textWrapper);
if (parts.length === 0) {
return <>{parts[0]}</>;

2
scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx
View File

@@ -48,7 +48,7 @@ storiesOf("SyntaxHighlighter", module)
))
.add("Go", () => (
<Spacing>
<SyntaxHighlighter language="golang" value={GoHttpServer} />
<SyntaxHighlighter language="go" value={GoHttpServer} />
</Spacing>
))
.add("Javascript", () => (

67
scm-ui/ui-components/src/__resources__/ContentSearchHit.ts Normal file
View File

@@ -0,0 +1,67 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import { Hit } from "@scm-manager/ui-types";
export const javaHit: Hit = {
score: 2.5,
fields: {
content: {
highlighted: true,
fragments: [
"import org.slf4j.LoggerFactory;\n\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * Jwt implementation of {@link <|[[--AccessTokenBuilder--]]|>}.\n * \n * @author Sebastian Sdorra\n * @since 2.0.0\n */\npublic final class <|[[--JwtAccessTokenBuilder--]]|> implements <|[[--AccessTokenBuilder--]]|> {\n\n /**\n * the logger for <|[[--JwtAccessTokenBuilder--]]|>\n */\n private static final Logger LOG = LoggerFactory.getLogger(<|[[--JwtAccessTokenBuilder.class--]]|>);\n \n private final KeyGenerator keyGenerator; \n private final SecureKeyResolver keyResolver; \n \n private String subject;\n private String issuer;\n",
" private final Map<String,Object> custom = Maps.newHashMap();\n \n <|[[--JwtAccessTokenBuilder--]]|>(KeyGenerator keyGenerator, SecureKeyResolver keyResolver) {\n this.keyGenerator = keyGenerator;\n this.keyResolver = keyResolver;\n }\n\n @Override\n public <|[[--JwtAccessTokenBuilder--]]|> subject(String subject) {\n",
' public <|[[--JwtAccessTokenBuilder--]]|> custom(String key, Object value) {\n Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "null or empty value not allowed");\n Preconditions.checkArgument(value != null, "null or empty value not allowed");\n'
]
}
},
_links: {}
};
export const bashHit: Hit = {
score: 2.5,
fields: {
content: {
highlighted: true,
fragments: [
'# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n#\n\n<|[[--getent--]]|> group scm >/dev/null || groupadd -r scm\n<|[[--getent--]]|> passwd scm >/dev/null || \\\n useradd -r -g scm -M \\\n -s /sbin/nologin -d /var/lib/scm \\\n -c "user for the scm-server process" scm\nexit 0\n\n'
]
}
},
_links: {}
};
export const markdownHit: Hit = {
score: 2.5,
fields: {
content: {
highlighted: true,
fragments: [
"---\ntitle: SCM-Manager v2 Test <|[[--Cases--]]|>\n---\n\nDescribes the expected behaviour for SCMM v2 REST Resources using manual tests.\n\nThe following states general test <|[[--cases--]]|> per HTTP Method and en expected return code as well as exemplary curl calls.\nResource-specifics are stated \n\n## Test <|[[--Cases--]]|>\n\n### GET\n\n- Collection Resource (e.g. `/users`)\n - Without parameters -> 200\n - Parameters\n - `?pageSize=1` -> Only one embedded element, pageTotal reflects the correct number of pages, `last` link points to last page.\n - `?pageSize=1&page=1` -> `next` link points to page 0 ; `prev` link points to page 2\n - `?sortBy=admin` -> Sorted by `admin` field of embedded objects\n - `?sortBy=admin&desc=true` -> Invert sorting\n- Individual Resource (e.g. `/users/scmadmin`)\n - Exists -> 200\n",
"\n### DELETE\n\n- existing -> 204\n- not existing -> 204\n- without permission -> 401\n\n## Exemplary calls & Resource specific test <|[[--cases--]]|>\n\nIn order to extend those tests to other Resources, have a look at the rest docs. Note that the Content Type is specific to each resource as well.\n"
]
}
},
_links: {}
};

998
scm-ui/ui-components/src/__snapshots__/storyshots.test.ts.snap
View File

File diff suppressed because it is too large Load Diff

6
scm-ui/ui-components/src/languages.test.ts
View File

@@ -35,7 +35,7 @@ describe("syntax highlighter", () => {
expect(java).toBe("java");
});
it("should return text if language is undefied", () => {
it("should return text if language is undefined", () => {
const lang = determineLanguage();
expect(lang).toBe("text");
});
@@ -45,8 +45,4 @@ describe("syntax highlighter", () => {
expect(lang).toBe("text");
});
it("should use alias go for golang", () => {
const go = determineLanguage("golang");
expect(go).toBe("go");
});
});

11
scm-ui/ui-components/src/languages.ts
View File

@@ -22,20 +22,11 @@
* SOFTWARE.
*/
// this aliases are only to map from spotter detection to prismjs
const languageAliases: { [key: string]: string } = {
golang: "go",
};
export const defaultLanguage = "text";
export const determineLanguage = (language?: string) => {
if (!language) {
return defaultLanguage;
}
const lang = language.toLowerCase();
if (languageAliases[lang]) {
return languageAliases[lang];
}
return lang;
return language.toLowerCase();
};

22
scm-ui/ui-components/src/repos/TokenizedDiffView.tsx
View File

@@ -26,35 +26,46 @@ import styled from "styled-components";
// @ts-ignore we have no typings for react-diff-view
import { Diff, useTokenizeWorker } from "react-diff-view";
import { FileDiff } from "@scm-manager/ui-types";
import { determineLanguage } from "../languages";
// @ts-ignore no types for css modules
import theme from "../syntax-highlighting.module.css";
import { determineLanguage } from "../languages";
const DiffView = styled(Diff)`
/* align line numbers */
& .diff-gutter {
text-align: right;
}
/* column sizing */
> colgroup .diff-gutter-col {
width: 3.25rem;
}
/* prevent following content from moving down */
> .diff-gutter:empty:hover::after {
font-size: 0.7rem;
}
/* smaller font size for code and
ensure same monospace font throughout whole scmm */
& .diff-line {
font-size: 0.75rem;
font-family: "Courier New", Monaco, Menlo, "Ubuntu Mono", "source-code-pro", monospace;
}
/* comment padding for sidebyside view */
&.split .diff-widget-content .is-indented-line {
padding-left: 3.25rem;
}
/* comment padding for combined view */
&.unified .diff-widget-content .is-indented-line {
padding-left: 6.5rem;
}
@@ -71,10 +82,17 @@ type Props = {
className?: string;
};
const findSyntaxHighlightingLanguage = (file: FileDiff) => {
if (file.syntaxModes) {
return file.syntaxModes["prism"] || file.syntaxModes["codemirror"] || file.syntaxModes["ace"] || file.language;
}
return file.language;
};
const TokenizedDiffView: FC<Props> = ({ file, viewType, className, children }) => {
const { tokens } = useTokenizeWorker(tokenize, {
hunks: file.hunks,
language: determineLanguage(file.language),
language: determineLanguage(findSyntaxHighlightingLanguage(file))
});
return (

2
scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx
View File

@@ -67,7 +67,7 @@ const commitImplementMain = {
};
const source: AnnotatedSource = {
language: "golang",
language: "go",
lines: [
{
lineNumber: 1,

11
scm-ui/ui-components/src/repos/refractorAdapter.ts
View File

@@ -46,9 +46,14 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
import(
/* webpackChunkName: "tokenizer-refractor-[request]" */
`refractor/lang/${lang}`
).then((loadedLanguage) => {
)
.then(loadedLanguage => {
refractor.register(loadedLanguage.default);
callback();
})
.catch(e => {
// eslint-disable-next-line no-console
console.log(`failed to load refractor language ${lang}: ${e}`);
});
}
};
@@ -58,7 +63,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
const runHook = (name: string, env: RunHookEnv) => {
originalRunHook.apply(name, env);
if (env.classes) {
env.classes = env.classes.map((className) => theme[className] || className);
env.classes = env.classes.map(className => theme[className] || className);
}
};
// @ts-ignore hooks are not in the type definition
@@ -67,7 +72,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => {
return {
isLanguageRegistered,
loadLanguage,
...refractor,
...refractor
};
};

2
scm-ui/ui-components/src/search/HighlightedFragment.tsx
View File

@@ -42,7 +42,7 @@ const HighlightedFragment: FC<Props> = ({ value }) => {
if (start > 0) {
result.push(content.substring(0, start));
}
result.push(<strong>{content.substring(start + PRE_TAG.length, end)}</strong>);
result.push(<mark>{content.substring(start + PRE_TAG.length, end)}</mark>);
content = content.substring(end + POST_TAG.length);
} else {
result.push(content);

130
scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx Normal file
View File

@@ -0,0 +1,130 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import React, { FC, ReactNode, useEffect, useMemo, useState } from "react";
import createAdapter from "../repos/refractorAdapter";
// @ts-ignore no types for css modules
import theme from "../syntax-highlighting.module.css";
import SplitAndReplace, { Replacement } from "../SplitAndReplace";
import { AST, RefractorNode } from "refractor";
import { determineLanguage } from "../languages";
const PRE_TAG = "<|[[--";
const POST_TAG = "--]]|>";
const PRE_TAG_REGEX = /<\|\[\[--/g;
const POST_TAG_REGEX = /--]]\|>/g;
const adapter = createAdapter(theme);
function createReplacement(textToReplace: string): Replacement {
return {
textToReplace,
replacement: <mark>{textToReplace}</mark>,
replaceAll: true
};
}
function mapWithDepth(depth: number, replacements: Replacement[]) {
return function mapChildrenWithDepth(child: RefractorNode, i: number) {
return mapChild(child, i, depth, replacements);
};
}
function isAstElement(node: RefractorNode): node is AST.Element {
return (node as AST.Element).tagName !== undefined;
}
function mapChild(child: RefractorNode, i: number, depth: number, replacements: Replacement[]): ReactNode {
if (isAstElement(child)) {
const className =
child.properties && Array.isArray(child.properties.className)
? child.properties.className.join(" ")
: child.properties.className;
return React.createElement(
child.tagName,
Object.assign({ key: `fract-${depth}-${i}` }, child.properties, { className }),
child.children && child.children.map(mapWithDepth(depth + 1, replacements))
);
}
return (
<SplitAndReplace key={`fract-${depth}-${i}`} text={child.value} replacements={replacements} textWrapper={s => s} />
);
}
type Props = {
value: string;
language: string;
};
const stripAndReplace = (value: string) => {
const strippedValue = value.replace(PRE_TAG_REGEX, "").replace(POST_TAG_REGEX, "");
let content = value;
const result: string[] = [];
while (content.length > 0) {
const start = content.indexOf(PRE_TAG);
const end = content.indexOf(POST_TAG);
if (start >= 0 && end > 0) {
const item = content.substring(start + PRE_TAG.length, end);
if (!result.includes(item)) {
result.push(item);
}
content = content.substring(end + POST_TAG.length);
} else {
break;
}
}
result.sort((a, b) => b.length - a.length);
return {
strippedValue,
replacements: result.map(createReplacement)
};
};
const SyntaxHighlightedFragment: FC<Props> = ({ value, language }) => {
const [isLoading, setIsLoading] = useState(true);
const determinedLanguage = determineLanguage(language);
const { strippedValue, replacements } = useMemo(() => stripAndReplace(value), [value]);
useEffect(() => {
adapter.loadLanguage(determinedLanguage, () => {
setIsLoading(false);
});
}, [determinedLanguage]);
if (isLoading) {
return <SplitAndReplace text={strippedValue} replacements={replacements} textWrapper={s => s} />;
}
const refractorNodes = adapter.highlight(strippedValue, determinedLanguage);
const highlightedFragment = refractorNodes.map(mapWithDepth(0, replacements));
return <>{highlightedFragment}</>;
};
export default SyntaxHighlightedFragment;

54
scm-ui/ui-components/src/search/TextHitField.stories.tsx Normal file
View File

@@ -0,0 +1,54 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import React from "react";
import { storiesOf } from "@storybook/react";
import { bashHit, javaHit, markdownHit } from "../__resources__/ContentSearchHit";
import TextHitField from "./TextHitField";
storiesOf("TextHitField", module)
.add("Default", () => (
<pre>
<TextHitField hit={javaHit} field={"content"} />
</pre>
))
.add("Java SyntaxHighlighting", () => (
<pre>
<TextHitField hit={javaHit} field={"content"} syntaxHighlightingLanguage="java" />
</pre>
))
.add("Bash SyntaxHighlighting", () => (
<pre>
<TextHitField hit={bashHit} field={"content"} syntaxHighlightingLanguage="bash" />
</pre>
))
.add("Markdown SyntaxHighlighting", () => (
<pre>
<TextHitField hit={markdownHit} field={"content"} syntaxHighlightingLanguage="markdown" />
</pre>
))
.add("Unknown SyntaxHighlighting", () => (
<pre>
<TextHitField hit={bashHit} field={"content"} syntaxHighlightingLanguage="__unknown__" />
</pre>
));

52
scm-ui/ui-components/src/search/TextHitField.tsx
View File

@@ -26,35 +26,51 @@ import React, { FC } from "react";
import { HighlightedHitField, Hit } from "@scm-manager/ui-types";
import HighlightedFragment from "./HighlightedFragment";
import { isHighlightedHitField } from "./fields";
import SyntaxHighlightedFragment from "./SyntaxHighlightedFragment";
type HighlightedTextFieldProps = {
field: HighlightedHitField;
syntaxHighlightingLanguage?: string;
};
const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field, syntaxHighlightingLanguage }) => {
const separator = syntaxHighlightingLanguage ? "...\n" : " ... ";
return (
<>
{field.fragments.map((fragment, i) => (
<React.Fragment key={fragment}>
{separator}
{syntaxHighlightingLanguage ? (
<SyntaxHighlightedFragment value={fragment} language={syntaxHighlightingLanguage} />
) : (
<HighlightedFragment value={fragment} />
)}
{i + 1 >= field.fragments.length ? separator : null}
</React.Fragment>
))}
</>
);
};
type Props = {
hit: Hit;
field: string;
truncateValueAt?: number;
syntaxHighlightingLanguage?: string;
};
type HighlightedTextFieldProps = {
field: HighlightedHitField;
};
const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field }) => (
<>
{field.fragments.map((fr, i) => (
<React.Fragment key={fr}>
{" ... "}
<HighlightedFragment value={fr} />
{i + 1 >= field.fragments.length ? " ... " : null}
</React.Fragment>
))}
</>
);
const TextHitField: FC<Props> = ({ hit, field: fieldName, children, truncateValueAt = 0 }) => {
const TextHitField: FC<Props> = ({
hit,
field: fieldName,
children,
syntaxHighlightingLanguage,
truncateValueAt = 0
}) => {
const field = hit.fields[fieldName];
if (!field) {
return <>{children}</>;
} else if (isHighlightedHitField(field)) {
return <HighlightedTextField field={field} />;
return <HighlightedTextField field={field} syntaxHighlightingLanguage={syntaxHighlightingLanguage} />;
} else {
let value = field.value;
if (value === "") {

1
scm-ui/ui-types/src/Diff.ts
View File

@@ -43,6 +43,7 @@ export type FileDiff = {
oldRevision?: string;
type: FileChangeType;
language?: string;
syntaxModes?: { [mode: string]: string };
// TODO does this property exists?
isBinary?: boolean;
_links?: Links;

2
scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx
View File

@@ -79,7 +79,7 @@ const SwitchableMarkdownViewer: FC<Props> = ({ file, basePath }) => {
{renderMarkdown ? (
<MarkdownViewer content={content || ""} basePath={basePath} permalink={permalink} />
) : (
<SyntaxHighlighter language="MARKDOWN" value={content || ""} permalink={permalink} />
<SyntaxHighlighter language="markdown" value={content || ""} permalink={permalink} />
)}
</div>
);

4
scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx
View File

@@ -26,6 +26,7 @@ import React, { FC } from "react";
import { File, Link, Repository } from "@scm-manager/ui-types";
import { Annotate, ErrorNotification, Loading } from "@scm-manager/ui-components";
import { useAnnotations, useContentType } from "@scm-manager/ui-api";
import { determineSyntaxHighlightingLanguage } from "../utils/files";
type Props = {
file: File;
@@ -52,7 +53,8 @@ const AnnotateView: FC<Props> = ({ file, repository, revision }) => {
return <Loading />;
}
return <Annotate source={{ ...annotation, language: contentType.language }} repository={repository} />;
const language = determineSyntaxHighlightingLanguage(contentType);
return <Annotate source={{ ...annotation, language }} repository={repository} />;
};
export default AnnotateView;

8
scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx
View File

@@ -31,7 +31,8 @@ import { File, Link, Repository } from "@scm-manager/ui-types";
import { ErrorNotification, Loading, PdfViewer } from "@scm-manager/ui-components";
import SwitchableMarkdownViewer from "../components/content/SwitchableMarkdownViewer";
import styled from "styled-components";
import { useContentType } from "@scm-manager/ui-api";
import { ContentType, useContentType } from "@scm-manager/ui-api";
import {determineSyntaxHighlightingLanguage} from "../utils/files";
const NoSpacingSyntaxHighlighterContainer = styled.div`
& pre {
@@ -46,6 +47,8 @@ type Props = {
revision: string;
};
const SourcesView: FC<Props> = ({ file, repository, revision }) => {
const { data: contentTypeData, error, isLoading } = useContentType((file._links.self as Link).href);
@@ -59,7 +62,8 @@ const SourcesView: FC<Props> = ({ file, repository, revision }) => {
let sources;
const { type: contentType, language } = contentTypeData;
const language = determineSyntaxHighlightingLanguage(contentTypeData);
const contentType = contentTypeData.type;
const basePath = `/repo/${repository.namespace}/${repository.name}/code/sources/${revision}/`;
if (contentType.startsWith("image/")) {
sources = <ImageViewer file={file} />;

5
scm-ui/ui-webapp/src/repos/sources/utils/files.ts
View File

@@ -22,6 +22,7 @@
* SOFTWARE.
*/
import { File } from "@scm-manager/ui-types";
import { ContentType } from "@scm-manager/ui-api";
export const isRootPath = (path: string) => {
return path === "" || path === "/";
@@ -40,3 +41,7 @@ export const isEmptyDirectory = (file: File) => {
}
return (file._embedded?.children?.length || 0) === 0;
};
export const determineSyntaxHighlightingLanguage = (data: ContentType) => {
return data.prismMode || data.codemirrorMode || data.aceMode || data.language;
};

6
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java
View File

@@ -54,6 +54,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Locale;
public class ContentResource {
@@ -211,6 +212,11 @@ public class ContentResource {
contentType.getLanguage().ifPresent(
language -> responseBuilder.header(ProgrammingLanguages.HEADER, language)
);
contentType.getSyntaxModes().forEach((mode, lang) -> {
String modeName = mode.substring(0, 1).toUpperCase(Locale.ENGLISH) + mode.substring(1);
responseBuilder.header(ProgrammingLanguages.HEADER_SYNTAX_MODE_PREFIX + modeName, lang);
});
}
private byte[] getHead(String revision, String path, RepositoryService repositoryService) throws IOException {

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java
View File

@@ -32,6 +32,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;
import java.util.Map;
@Data
@EqualsAndHashCode(callSuper = false)
@@ -63,6 +64,7 @@ public class DiffResultDto extends HalRepresentation {
private String oldMode;
private String type;
private String language;
private Map<String, String> syntaxModes;
private List<HunkDto> hunks;
}

5
scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java
View File

@@ -26,6 +26,7 @@ package sonia.scm.api.v2.resources;
import com.google.inject.Inject;
import de.otto.edison.hal.Links;
import sonia.scm.io.ContentType;
import sonia.scm.io.ContentTypeResolver;
import sonia.scm.repository.Repository;
import sonia.scm.repository.api.DiffFile;
@@ -155,8 +156,10 @@ class DiffResultToDiffResultDtoMapper {
dto.setOldPath(oldPath);
dto.setOldRevision(file.getOldRevision());
Optional<String> language = contentTypeResolver.resolve(path).getLanguage();
ContentType contentType = contentTypeResolver.resolve(path);
Optional<String> language = contentType.getLanguage();
language.ifPresent(dto::setLanguage);
dto.setSyntaxModes(contentType.getSyntaxModes());
List<DiffResultDto.HunkDto> hunks = new ArrayList<>();
for (Hunk hunk : file) {

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java
View File

@@ -28,6 +28,8 @@ final class ProgrammingLanguages {
static final String HEADER = "X-Programming-Language";
static final String HEADER_SYNTAX_MODE_PREFIX = "X-Syntax-Mode-";
private ProgrammingLanguages() {
}
}

33
scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java
View File

@@ -24,15 +24,18 @@
package sonia.scm.io;
import com.cloudogu.spotter.Language;
import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
public class DefaultContentType implements ContentType {
private static final String DEFAULT_LANG_MODE = "text";
private final com.cloudogu.spotter.ContentType contentType;
private final com.github.sdorra.spotter.ContentType contentType;
DefaultContentType(com.github.sdorra.spotter.ContentType contentType) {
DefaultContentType(com.cloudogu.spotter.ContentType contentType) {
this.contentType = contentType;
}
@@ -58,9 +61,23 @@ public class DefaultContentType implements ContentType {
@Override
public Optional<String> getLanguage() {
return contentType.getLanguage().map(language -> {
Optional<String> aceMode = language.getAceMode();
return aceMode.orElseGet(() -> language.getCodemirrorMode().orElse(DEFAULT_LANG_MODE));
});
return contentType.getLanguage().map(Language::getName);
}
@Override
public Map<String, String> getSyntaxModes() {
Optional<Language> language = contentType.getLanguage();
if (language.isPresent()) {
return syntaxMode(language.get());
}
return Collections.emptyMap();
}
static Map<String, String> syntaxMode(Language language) {
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
language.getAceMode().ifPresent(mode -> builder.put("ace", mode));
language.getCodemirrorMode().ifPresent(mode -> builder.put("codemirror", mode));
language.getPrismMode().ifPresent(mode -> builder.put("prism", mode));
return builder.build();
}
}

32
scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java
View File

@@ -24,17 +24,32 @@
package sonia.scm.io;
import com.github.sdorra.spotter.ContentTypeDetector;
import com.github.sdorra.spotter.Language;
import com.cloudogu.spotter.ContentTypeDetector;
import com.cloudogu.spotter.Language;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
public final class DefaultContentTypeResolver implements ContentTypeResolver {
private static final Language[] BOOST = new Language[]{
// GCC Machine Description uses .md as extension, but markdown is much more likely
Language.MARKDOWN,
// XML uses .rs as extension, but rust is much more likely
Language.RUST,
// XML is also returned by content type boost strategy, but rust is really much more likely
Language.RUST,
};
private static final ContentTypeDetector PATH_BASED = ContentTypeDetector.builder()
.defaultPathBased().boost(Language.MARKDOWN)
.defaultPathBased()
.boost(BOOST)
.bestEffortMatch();
private static final ContentTypeDetector PATH_AND_CONTENT_BASED = ContentTypeDetector.builder()
.defaultPathAndContentBased().boost(Language.MARKDOWN)
.defaultPathAndContentBased()
.boost(BOOST)
.bestEffortMatch();
@Override
@@ -46,4 +61,13 @@ public final class DefaultContentTypeResolver implements ContentTypeResolver {
public DefaultContentType resolve(String path, byte[] contentPrefix) {
return new DefaultContentType(PATH_AND_CONTENT_BASED.detect(path, contentPrefix));
}
@Override
public Map<String, String> findSyntaxModesByLanguage(String language) {
Optional<Language> byName = Language.getByName(language);
if (byName.isPresent()) {
return DefaultContentType.syntaxMode(byName.get());
}
return Collections.emptyMap();
}
}

18
scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java
View File

@@ -90,11 +90,21 @@ public final class LuceneHighlighter {
int index = content.indexOf(raw);
int start = content.lastIndexOf('\n', index);
if (start < 0) {
start = 0;
}
String snippet = content.substring(start, index) + fragment;
String snippet;
if (start == index) {
// fragment starts with a linebreak
snippet = fragment.substring(1);
} else {
if (start < 0) {
// no leading linebreak
start = 0;
} else if (start < content.length()) {
// skip linebreak
start++;
}
snippet = content.substring(start, index) + fragment;
}
int end = content.indexOf('\n', index + raw.length());
if (end < 0) {

17
scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java
View File

@@ -141,7 +141,7 @@ public class ContentResourceTest {
Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
assertEquals(200, response.getStatus());
assertEquals("golang", response.getHeaderString("X-Programming-Language"));
assertEquals("Go", response.getHeaderString("X-Programming-Language"));
assertEquals("text/x-go", response.getHeaderString("Content-Type"));
}
@@ -152,10 +152,22 @@ public class ContentResourceTest {
Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "Dockerfile", null, null);
assertEquals(200, response.getStatus());
assertEquals("dockerfile", response.getHeaderString("X-Programming-Language"));
assertEquals("Dockerfile", response.getHeaderString("X-Programming-Language"));
assertEquals("text/plain", response.getHeaderString("Content-Type"));
}
@Test
public void shouldRecognizeSyntaxModes() throws Exception {
mockContentFromResource("SomeGoCode.go");
Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
assertEquals(200, response.getStatus());
assertEquals("golang", response.getHeaderString("X-Syntax-Mode-Ace"));
assertEquals("go", response.getHeaderString("X-Syntax-Mode-Codemirror"));
assertEquals("go", response.getHeaderString("X-Syntax-Mode-Prism"));
}
@Test
public void shouldHandleRandomByteFile() throws Exception {
mockContentFromResource("JustBytes");
@@ -190,6 +202,7 @@ public class ContentResourceTest {
assertEquals("application/octet-stream", response.getHeaderString("Content-Type"));
}
@SuppressWarnings("UnstableApiUsage")
private void mockContentFromResource(String fileName) throws Exception {
URL url = Resources.getResource(fileName);
mockContent(fileName, Resources.toByteArray(url));

15
scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java
View File

@@ -60,11 +60,16 @@ class DiffResultToDiffResultDtoMapperTest {
DiffResultDto dto = mapper.mapForRevision(REPOSITORY, createResult(), "123");
List<DiffResultDto.FileDto> files = dto.getFiles();
assertAddedFile(files.get(0), "A.java", "abc", "java");
assertModifiedFile(files.get(1), "B.ts", "abc", "def", "typescript");
assertDeletedFile(files.get(2), "C.go", "ghi", "golang");
assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "typescript");
assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "typescript");
assertAddedFile(files.get(0), "A.java", "abc", "Java");
assertModifiedFile(files.get(1), "B.ts", "abc", "def", "TypeScript");
DiffResultDto.FileDto cGo = files.get(2);
assertDeletedFile(cGo, "C.go", "ghi", "Go");
assertThat(cGo.getSyntaxModes())
.containsEntry("ace", "golang")
.containsEntry("codemirror", "go")
.containsEntry("prism", "go");
assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "TypeScript");
assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "TypeScript");
DiffResultDto.HunkDto hunk = files.get(1).getHunks().get(0);
assertHunk(hunk, "@@ -3,4 1,2 @@", 1, 2, 3, 4);

35
scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java
View File

@@ -24,15 +24,15 @@
package sonia.scm.io;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.assertj.core.api.Assertions.assertThat;
class DefaultContentTypeResolverTest {
@@ -84,42 +84,45 @@ class DefaultContentTypeResolverTest {
"% Which does not start with markdown"
);
ContentType contentType = contentTypeResolver.resolve("somedoc.md", content.getBytes(StandardCharsets.UTF_8));
Assertions.assertThat(contentType.getLanguage()).contains("markdown");
assertThat(contentType.getLanguage()).contains("Markdown");
}
@Test
void shouldResolveMarkdownWithoutContent() {
ContentType contentType = contentTypeResolver.resolve("somedoc.md");
Assertions.assertThat(contentType.getLanguage()).contains("markdown");
assertThat(contentType.getLanguage()).contains("Markdown");
}
@Test
void shouldResolveMarkdownEvenWithDotsInFilename() {
ContentType contentType = contentTypeResolver.resolve("somedoc.1.1.md");
Assertions.assertThat(contentType.getLanguage()).contains("markdown");
assertThat(contentType.getLanguage()).contains("Markdown");
}
@Test
void shouldResolveDockerfile() {
ContentType contentType = contentTypeResolver.resolve("Dockerfile");
Assertions.assertThat(contentType.getLanguage()).contains("dockerfile");
assertThat(contentType.getLanguage()).contains("Dockerfile");
}
}
@Nested
class GetSyntaxModesTests {
@Test
void shouldReturnAceModeIfPresent() {
assertThat(contentTypeResolver.resolve("app.go").getLanguage()).contains("golang"); // codemirror is just go
assertThat(contentTypeResolver.resolve("App.java").getLanguage()).contains("java"); // codemirror is clike
void shouldReturnEmptyMapOfModesWithoutLanguage() {
Map<String, String> syntaxModes = contentTypeResolver.resolve("app.exe").getSyntaxModes();
assertThat(syntaxModes).isEmpty();
}
@Test
void shouldReturnCodemirrorIfAceModeIsMissing() {
assertThat(contentTypeResolver.resolve("index.ecr").getLanguage()).contains("htmlmixed");
}
@Test
void shouldReturnTextIfNoModeIsPresent() {
assertThat(contentTypeResolver.resolve("index.hxml").getLanguage()).contains("text");
void shouldReturnMapOfModes() {
Map<String, String> syntaxModes = contentTypeResolver.resolve("app.rs").getSyntaxModes();
assertThat(syntaxModes)
.containsEntry("ace", "rust")
.containsEntry("codemirror", "rust")
.containsEntry("prism", "rust");
}
}

11
scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java
View File

@@ -47,8 +47,6 @@ import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class LuceneHighlighterTest {
@Test
void shouldHighlightText() throws InvalidTokenOffsetsException, IOException {
StandardAnalyzer analyzer = new StandardAnalyzer();
@@ -80,6 +78,15 @@ class LuceneHighlighterTest {
);
}
@Test
void shouldNotStartHighlightedFragmentWithLineBreak() throws IOException, InvalidTokenOffsetsException {
String[] snippets = highlightCode("GameOfLife.java", "die");
assertThat(snippets).hasSize(1).allSatisfy(
snippet -> assertThat(snippet).doesNotStartWith("\n")
);
}
@Test
void shouldHighlightCodeInTsx() throws IOException, InvalidTokenOffsetsException {
String[] snippets = highlightCode("Button.tsx", "inherit");