Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-16 18:26:16 +01:00
Code Issues Packages Projects Releases Activity

Refactor Search API and allow analyzer per field (#1755)

Browse Source 
The Search api is now simpler, because it provides useful defaults. Only if you want to deviate from the defaults, you can set these values. This is mostly reached by using the builder pattern. Furthermore it is now possible to configure an analyzer per field. The default analyzer is still the one which is derived from the index options, but it is possible to configure a new indexer with the analyzer attribute of the indexed annotation. The attribute allows the configuration for code, identifiers and path. The current implementation uses the same analyzer code, identifiers and path. The new implemented splits tokens on more delimiters as the default analyzer e.g.: dots, underscores etc.

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>
This commit is contained in:
Sebastian Sdorra
2021-08-05 08:21:46 +02:00
committed by GitHub
parent 4fa5ad1f0d
commit 21a6943980
47 changed files with 1069 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gradle/changelog/analyzer_per_field.yaml Normal file
View File

@@ -0,0 +1,4 @@
- type: Added
description: Support for different types of analyzer per field ([#1755](https://github.com/scm-manager/scm-manager/pull/1755))
- type: Changed
description: Improve Search API ([#1755](https://github.com/scm-manager/scm-manager/pull/1755))

36
scm-annotations/src/main/java/sonia/scm/search/Indexed.java
View File

@@ -80,6 +80,42 @@ public @interface Indexed {
*/
boolean highlighted() default false;
/**
* Describes how the field is analyzed and tokenized.
*
* @return type of analyzer
* @since 2.23.0
*/
Analyzer analyzer() default Analyzer.DEFAULT;
/**
* Describes how fields are analyzed and tokenized.
*
* @since 2.23.0
*/
enum Analyzer {
/**
* Uses the analyzer which was used to open the index.
*/
DEFAULT,
/**
* Uses an analyzer which is specialized for identifiers like repository names.
*/
IDENTIFIER,
/**
* Uses an analyzer which is specialized for paths.
*/
PATH,
/**
* Uses an analyzer which is specialized for source code.
*/
CODE
}
/**
* Describes how the field is indexed.
*/

4
scm-core/src/main/java/sonia/scm/repository/Repository.java
View File

@@ -70,10 +70,10 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per
private String id;
@Indexed(defaultQuery = true, boost = 1.25f)
@Indexed(defaultQuery = true, boost = 1.25f, analyzer = Indexed.Analyzer.IDENTIFIER)
private String namespace;
@Indexed(defaultQuery = true, boost = 1.5f)
@Indexed(defaultQuery = true, boost = 1.5f, analyzer = Indexed.Analyzer.IDENTIFIER)
private String name;
@Indexed(type = Indexed.Type.SEARCHABLE)
private String type;

104
scm-core/src/main/java/sonia/scm/search/Index.java
View File

@@ -28,11 +28,11 @@ import com.google.common.annotations.Beta;
/**
* Can be used to index objects for full text searches.
*
* @param <T> type of indexed objects
* @since 2.21.0
*/
@Beta
public interface Index extends AutoCloseable {
public interface Index<T> extends AutoCloseable {
/**
* Store the given object in the index.
@@ -44,39 +44,87 @@ public interface Index extends AutoCloseable {
*
* @see Indexed
*/
void store(Id id, String permission, Object object);
void store(Id id, String permission, T object);
/**
* Delete the object with the given id and type from index.
*
* @param id id of object
* @param type type of object
* Delete provides an api to delete objects from the index
* @return delete api
* @since 2.23.0
*/
void delete(Id id, Class<?> type);
/**
* Delete all objects which are related the given repository from index.
*
* @param repositoryId id of repository
*/
void deleteByRepository(String repositoryId);
/**
* Delete all objects with the given type from index.
* @param type type of objects
*/
void deleteByType(Class<?> type);
/**
* Delete all objects with the given type from index.
* This method is mostly if the index type has changed and the old type (in form of class) is no longer available.
* @param typeName type name of objects
*/
void deleteByTypeName(String typeName);
Deleter delete();
/**
* Close index and commit changes.
*/
@Override
void close();
/**
* Deleter provides an api to delete object from index.
*
* @since 2.23.0
*/
interface Deleter {
/**
* Returns an api which allows deletion of objects from the type of this index.
* @return type restricted delete api
*/
ByTypeDeleter byType();
/**
* Returns an api which allows deletion of objects of every type.
* @return unrestricted delete api for all types.
*/
AllTypesDeleter allTypes();
}
/**
* Delete api for the type of the index. This means, that only entries for this
* type will be deleted.
*
* @since 2.23.0
*/
interface ByTypeDeleter {
/**
* Delete the object with the given id and type from index.
* @param id id of object
*/
void byId(Id id);
/**
* Delete all objects of the given type from index.
*/
void all();
/**
* Delete all objects which are related the given type and repository from index.
*
* @param repositoryId id of repository
*/
void byRepository(String repositoryId);
}
/**
* Delete api for the overall index regarding all types.
*
* @since 2.23.0
*/
interface AllTypesDeleter {
/**
* Delete all objects which are related to the given repository from index regardless of their type.
* @param repositoryId repository id
*/
void byRepository(String repositoryId);
/**
* Delete all objects with the given type from index.
* This method is mostly useful if the index type has changed and the old type (in form of a class)
* is no longer available.
* @param typeName type name of objects
*/
void byTypeName(String typeName);
}
}

3
scm-core/src/main/java/sonia/scm/search/IndexLog.java
View File

@@ -36,7 +36,8 @@ import java.time.Instant;
/**
* A marker keeping track of when and with which model version an object type was last indexed.
* @since 2.21
*
* @since 2.21.0
*/
@Beta
@Data

45
scm-core/src/main/java/sonia/scm/search/IndexLogStore.java
View File

@@ -39,23 +39,42 @@ import java.util.Optional;
public interface IndexLogStore {
/**
* Log index and version of a type which is now indexed.
*
* Returns an index log store for the given index.
* @param index name of index
* @param type type which was indexed
* @param version model version
* @return index log store for given index
* @since 2.23.0
*/
void log(String index, Class<?> type, int version);
ForIndex forIndex(String index);
/**
* Returns version and date of the indexed type or an empty object,
* if the object was not indexed at all.
*
* @param index name if index
* @param type type of object
*
* @return log entry or empty
* Returns the index log store for the default index.
* @return index log store for default index
* @since 2.23.0
*/
Optional<IndexLog> get(String index, Class<?> type);
ForIndex defaultIndex();
/**
* Index log store for a specific index.
* @since 2.23.0
*/
interface ForIndex {
/**
* Log index and version of a type which is now indexed.
*
* @param type type which was indexed
* @param version model version
*/
void log(Class<?> type, int version);
/**
* Returns version and date of the indexed type or an empty object,
* if the object was not indexed at all.
*
* @param type type of object
*
* @return log entry or empty
*/
Optional<IndexLog> get(Class<?> type);
}
}

7
scm-core/src/main/java/sonia/scm/search/Indexer.java
View File

@@ -45,13 +45,6 @@ public interface Indexer<T> {
*/
Class<T> getType();
/**
* Returns name of index.
*
* @return name of index
*/
String getIndex();
/**
* Returns version of index type.
* The version should be increased if the type changes and should be re indexed.

60
scm-core/src/main/java/sonia/scm/search/QueryBuilder.java
View File

@@ -37,22 +37,22 @@ import static sonia.scm.NotFoundException.notFound;
/**
* Build and execute queries against an index.
*
* @param <T> type of indexed objects
* @since 2.21.0
*/
@Beta
public abstract class QueryBuilder {
public abstract class QueryBuilder<T> {
private String repositoryId;
private int start = 0;
private int limit = 10;
/**
* Return only results which are related to the given repository.
* @param repository repository
* @return {@code this}
*/
public QueryBuilder repository(Repository repository) {
public QueryBuilder<T> repository(Repository repository) {
return repository(repository.getId());
}
@@ -61,7 +61,7 @@ public abstract class QueryBuilder {
* @param repositoryId id of the repository
* @return {@code this}
*/
public QueryBuilder repository(String repositoryId) {
public QueryBuilder<T> repository(String repositoryId) {
this.repositoryId = repositoryId;
return this;
}
@@ -72,7 +72,7 @@ public abstract class QueryBuilder {
* @param start start of result
* @return {@code this}
*/
public QueryBuilder start(int start) {
public QueryBuilder<T> start(int start) {
this.start = start;
return this;
}
@@ -82,7 +82,7 @@ public abstract class QueryBuilder {
* @param limit limit of hits
* @return {@code this}
*/
public QueryBuilder limit(int limit) {
public QueryBuilder<T> limit(int limit) {
this.limit = limit;
return this;
}
@@ -90,61 +90,26 @@ public abstract class QueryBuilder {
/**
* Executes the query and returns the matches.
*
* @param type type of objects which are searched
* @param queryString searched query
* @return result of query
*/
public QueryResult execute(Class<?> type, String queryString){
return execute(new QueryParams(type, repositoryId, queryString, start, limit));
public QueryResult execute(String queryString){
return execute(new QueryParams(repositoryId, queryString, start, limit));
}
/**
* Executes the query and returns the total count of hits.
*
* @param type type of objects which are searched
* @param queryString searched query
*
* @return total count of hits
* @since 2.22.0
*/
public QueryCountResult count(Class<?> type, String queryString) {
return count(new QueryParams(type, repositoryId, queryString, start, limit));
public QueryCountResult count(String queryString) {
return count(new QueryParams(repositoryId, queryString, start, limit));
}
/**
* Executes the query and returns the matches.
*
* @param typeName type name of objects which are searched
* @param queryString searched query
* @return result of query
*
* @throws NotFoundException if type could not be found
*/
public QueryResult execute(String typeName, String queryString){
return execute(resolveTypeByName(typeName), queryString);
}
/**
* Executes the query and returns the total count of hits.
*
* @param typeName type name of objects which are searched
* @param queryString searched query
*
* @return total count of hits
* @since 2.22.0
*/
public QueryCountResult count(String typeName, String queryString) {
return count(resolveTypeByName(typeName), queryString);
}
/**
* Resolves the type by its name. Returns optional with class of type or an empty optional.
*
* @param typeName name of type
* @return optional with class of type or empty
*/
protected abstract Optional<Class<?>> resolveByName(String typeName);
/**
* Executes the query and returns the matches.
@@ -163,16 +128,11 @@ public abstract class QueryBuilder {
return execute(queryParams);
}
private Class<?> resolveTypeByName(String typeName) {
return resolveByName(typeName).orElseThrow(() -> notFound(entity("type", typeName)));
}
/**
* The searched query and all parameters, which belong to the query.
*/
@Value
static class QueryParams {
Class<?> type;
String repositoryId;
String queryString;
int start;

77
scm-core/src/main/java/sonia/scm/search/SearchEngine.java
View File

@@ -30,10 +30,7 @@ import java.util.Collection;
/**
* The {@link SearchEngine} is the main entry point for indexing and searching.
* Note that this is kind of a low level api for indexing.
* For non expert indexing the {@link IndexQueue} should be used.
*
* @see IndexQueue
* @since 2.21.0
*/
@Beta
@@ -47,46 +44,58 @@ public interface SearchEngine {
Collection<SearchableType> getSearchableTypes();
/**
* Returns the index with the given name and the given options.
* The index is created if it does not exist.
* Warning: Be careful, because an index can't be opened multiple times in parallel.
* If you are not sure how you should index your objects, use the {@link IndexQueue}.
* Returns a type specific api which can be used to index objects of that specific type.
*
* @param name name of the index
* @param options index options
* @return existing index or a new one if none exists
* @param type type of object
* @param <T> type of object
* @return type specific index and search api
* @since 2.23.0
*/
Index getOrCreate(String name, IndexOptions options);
<T> ForType<T> forType(Class<T> type);
/**
* Same as {@link #getOrCreate(String, IndexOptions)} with default options.
*
* @param name name of the index
* @return existing index or a new one if none exists
* @see IndexOptions#defaults()
* Returns an api which can be used to index and search object of that type.
* @param name name of type
* @return search and index api
* @since 2.23.0
*/
default Index getOrCreate(String name) {
return getOrCreate(name, IndexOptions.defaults());
}
ForType<Object> forType(String name);
/**
* Search the index.
* Returns a {@link QueryBuilder} which allows to query the index.
* Search and index api.
*
* @param name name of the index
* @param options options for searching the index
* @return query builder
* @param <T> type of searchable objects
* @since 2.23.0
*/
QueryBuilder search(String name, IndexOptions options);
interface ForType<T> {
/**
* Same as {@link #search(String, IndexOptions)} with default options.
*
* @param name name of the index
* @return query builder
* @see IndexOptions#defaults()
*/
default QueryBuilder search(String name) {
return search(name, IndexOptions.defaults());
/**
* Specify options for the index.
* If not used the default options will be used.
* @param options index options
* @return {@code this}
* @see IndexOptions#defaults()
*/
ForType<T> withOptions(IndexOptions options);
/**
* Name of the index which should be used.
* If not specified the default index will be used.
* @param name name of index
* @return {@code this}
*/
ForType<T> withIndex(String name);
/**
* Returns an index object which provides method to update the search index.
* @return index object
*/
Index<T> getOrCreate();
/**
* Returns a query builder object which can be used to search the index.
* @return query builder
*/
QueryBuilder<T> search();
}
}

25
scm-ui/ui-components/src/search/Hit.tsx
View File

@@ -24,28 +24,37 @@
import React, { FC } from "react";
import { Hit as HitType } from "@scm-manager/ui-types";
import classNames from "classnames";
export type HitProps = {
hit: HitType;
};
type Props = {
className?: string;
};
type SearchResultType = FC & {
Title: FC;
Left: FC;
Content: FC;
Right: FC;
Title: FC<Props>;
Left: FC<Props>;
Content: FC<Props>;
Right: FC<Props>;
};
const Hit: SearchResultType = ({ children }) => {
return <article className="media p-1">{children}</article>;
};
Hit.Title = ({ children }) => <h3 className="has-text-weight-bold is-ellipsis-overflow">{children}</h3>;
Hit.Title = ({ className, children }) => (
<h3 className={classNames("has-text-weight-bold is-ellipsis-overflow", className)}>{children}</h3>
);
Hit.Left = ({ children }) => <div className="media-left">{children}</div>;
Hit.Left = ({ className, children }) => <div className={classNames("media-left", className)}>{children}</div>;
Hit.Right = ({ children }) => <div className="media-right is-size-7 has-text-right">{children}</div>;
Hit.Right = ({ className, children }) => (
<div className={classNames("media-right is-size-7 has-text-right", className)}>{children}</div>
);
Hit.Content = ({ children }) => <div className="media-content">{children}</div>;
Hit.Content = ({ className, children }) => <div className={classNames("media-content", className)}> {children}</div>;
export default Hit;

9
scm-ui/ui-components/src/search/TextHitField.tsx
View File

@@ -30,6 +30,7 @@ import { isHighlightedHitField } from "./fields";
type Props = {
hit: Hit;
field: string;
truncateValueAt?: number;
};
type HighlightedTextFieldProps = {
@@ -48,14 +49,18 @@ const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field }) => (
</>
);
const TextHitField: FC<Props> = ({ hit, field: fieldName }) => {
const TextHitField: FC<Props> = ({ hit, field: fieldName, truncateValueAt = 0 }) => {
const field = hit.fields[fieldName];
if (!field) {
return null;
} else if (isHighlightedHitField(field)) {
return <HighlightedTextField field={field} />;
} else {
return <>{field.value}</>;
let value = field.value;
if (typeof value === "string" && truncateValueAt > 0 && value.length > truncateValueAt) {
value = value.substring(0, truncateValueAt) + "...";
}
return <>{value}</>;
}
};

11
scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java
View File

@@ -31,7 +31,6 @@ import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
@@ -112,17 +111,19 @@ public class SearchResource {
}
private QueryResultDto search(SearchParameters params) {
QueryResult result = engine.search(IndexNames.DEFAULT)
QueryResult result = engine.forType(params.getType())
.search()
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
.execute(params.getType(), params.getQuery());
.execute(params.getQuery());
return mapper.map(params, result);
}
private QueryResultDto count(SearchParameters params) {
QueryCountResult result = engine.search(IndexNames.DEFAULT)
.count(params.getType(), params.getQuery());
QueryCountResult result = engine.forType(params.getType())
.search()
.count(params.getQuery());
return mapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList()));
}

26
scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java
View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -41,18 +40,16 @@ import javax.inject.Singleton;
@Singleton
public class GroupIndexer implements Indexer<Group> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting
static final int VERSION = 1;
private final GroupManager groupManager;
private final IndexQueue indexQueue;
private final SearchEngine searchEngine;
@Inject
public GroupIndexer(GroupManager groupManager, IndexQueue indexQueue) {
public GroupIndexer(GroupManager groupManager, SearchEngine searchEngine) {
this.groupManager = groupManager;
this.indexQueue = indexQueue;
this.searchEngine = searchEngine;
}
@Override
@@ -60,11 +57,6 @@ public class GroupIndexer implements Indexer<Group> {
return Group.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Override
public int getVersion() {
return VERSION;
@@ -77,15 +69,15 @@ public class GroupIndexer implements Indexer<Group> {
@Override
public Updater<Group> open() {
return new GroupIndexUpdater(groupManager, indexQueue.getQueuedIndex(INDEX));
return new GroupIndexUpdater(groupManager, searchEngine.forType(Group.class).getOrCreate());
}
public static class GroupIndexUpdater implements Updater<Group> {
private final GroupManager groupManager;
private final Index index;
private final Index<Group> index;
private GroupIndexUpdater(GroupManager groupManager, Index index) {
private GroupIndexUpdater(GroupManager groupManager, Index<Group> index) {
this.groupManager = groupManager;
this.index = index;
}
@@ -97,12 +89,12 @@ public class GroupIndexer implements Indexer<Group> {
@Override
public void delete(Group group) {
index.delete(Id.of(group), Group.class);
index.delete().byType().byId(Id.of(group));
}
@Override
public void reIndexAll() {
index.deleteByType(Group.class);
index.delete().byType().all();
for (Group group : groupManager.getAll()) {
store(group);
}

3
scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java
View File

@@ -101,9 +101,7 @@ import sonia.scm.repository.xml.XmlRepositoryRoleDAO;
import sonia.scm.schedule.CronScheduler;
import sonia.scm.schedule.Scheduler;
import sonia.scm.search.DefaultIndexLogStore;
import sonia.scm.search.DefaultIndexQueue;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.LuceneSearchEngine;
import sonia.scm.search.SearchEngine;
import sonia.scm.security.AccessTokenCookieIssuer;
@@ -289,7 +287,6 @@ class ScmServletModule extends ServletModule {
bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class);
// bind search stuff
bind(IndexQueue.class, DefaultIndexQueue.class);
bind(SearchEngine.class, LuceneSearchEngine.class);
bind(IndexLogStore.class, DefaultIndexLogStore.class);

31
scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java
View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -42,18 +41,15 @@ import javax.inject.Singleton;
public class RepositoryIndexer implements Indexer<Repository> {
@VisibleForTesting
static final int VERSION = 2;
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
static final int VERSION = 3;
private final RepositoryManager repositoryManager;
private final IndexQueue indexQueue;
private final SearchEngine searchEngine;
@Inject
public RepositoryIndexer(RepositoryManager repositoryManager, IndexQueue indexQueue) {
public RepositoryIndexer(RepositoryManager repositoryManager, SearchEngine searchEngine) {
this.repositoryManager = repositoryManager;
this.indexQueue = indexQueue;
this.searchEngine = searchEngine;
}
@Override
@@ -66,11 +62,6 @@ public class RepositoryIndexer implements Indexer<Repository> {
return Repository.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Subscribe(async = false)
public void handleEvent(RepositoryEvent event) {
new HandlerEventIndexSyncer<>(this).handleEvent(event);
@@ -78,15 +69,15 @@ public class RepositoryIndexer implements Indexer<Repository> {
@Override
public Updater<Repository> open() {
return new RepositoryIndexUpdater(repositoryManager, indexQueue.getQueuedIndex(INDEX));
return new RepositoryIndexUpdater(repositoryManager, searchEngine.forType(getType()).getOrCreate());
}
public static class RepositoryIndexUpdater implements Updater<Repository> {
private final RepositoryManager repositoryManager;
private final Index index;
private final Index<Repository> index;
public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index index) {
public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index<Repository> index) {
this.repositoryManager = repositoryManager;
this.index = index;
}
@@ -98,14 +89,14 @@ public class RepositoryIndexer implements Indexer<Repository> {
@Override
public void delete(Repository repository) {
index.deleteByRepository(repository.getId());
index.delete().allTypes().byRepository(repository.getId());
}
@Override
public void reIndexAll() {
// v1 used the whole classname as type
index.deleteByTypeName(Repository.class.getName());
index.deleteByType(Repository.class);
index.delete().allTypes().byTypeName(Repository.class.getName());
index.delete().byType().all();
for (Repository repository : repositoryManager.getAll()) {
store(repository);
}

20
scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java
View File

@@ -28,9 +28,12 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import javax.annotation.Nonnull;
import java.util.HashMap;
import java.util.Map;
public class AnalyzerFactory {
@@ -59,4 +62,21 @@ public class AnalyzerFactory {
}
}
public Analyzer create(LuceneSearchableType type, IndexOptions options) {
Analyzer defaultAnalyzer = create(options);
Map<String, Analyzer> analyzerMap = new HashMap<>();
for (SearchableField field : type.getFields()) {
addFieldAnalyzer(analyzerMap, field);
}
return new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap);
}
private void addFieldAnalyzer(Map<String, Analyzer> analyzerMap, SearchableField field) {
if (field.getAnalyzer() != Indexed.Analyzer.DEFAULT) {
analyzerMap.put(field.getName(), new NonNaturalLanguageAnalyzer());
}
}
}

38
scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java
View File

@@ -42,18 +42,36 @@ public class DefaultIndexLogStore implements IndexLogStore {
}
@Override
public void log(String index,Class<?> type, int version) {
String id = id(index, type);
dataStore.put(id, new IndexLog(version));
}
private String id(String index, Class<?> type) {
return index + "_" + type.getName();
public ForIndex forIndex(String index) {
return new DefaultForIndex(index);
}
@Override
public Optional<IndexLog> get(String index, Class<?> type) {
String id = id(index, type);
return dataStore.getOptional(id);
public ForIndex defaultIndex() {
// constant
return new DefaultForIndex("default");
}
class DefaultForIndex implements ForIndex {
private final String index;
private DefaultForIndex(String index) {
this.index = index;
}
private String id(Class<?> type) {
return index + "_" + type.getName();
}
@Override
public void log(Class<?> type, int version) {
dataStore.put(id(type), new IndexLog(version));
}
@Override
public Optional<IndexLog> get(Class<?> type) {
return dataStore.getOptional(id(type));
}
}
}

4
scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java
View File

@@ -62,7 +62,7 @@ public class IndexBootstrapListener implements ServletContextListener {
}
private void bootstrap(Indexer indexer) {
Optional<IndexLog> indexLog = indexLogStore.get(indexer.getIndex(), indexer.getType());
Optional<IndexLog> indexLog = indexLogStore.defaultIndex().get(indexer.getType());
if (indexLog.isPresent()) {
int version = indexLog.get().getVersion();
if (version < indexer.getVersion()) {
@@ -82,7 +82,7 @@ public class IndexBootstrapListener implements ServletContextListener {
}
});
indexLogStore.log(indexer.getIndex(), indexer.getType(), indexer.getVersion());
indexLogStore.defaultIndex().log(indexer.getType(), indexer.getVersion());
}
@Override

6
scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java
View File

@@ -52,10 +52,10 @@ public class IndexOpener {
return DirectoryReader.open(directory(name));
}
public IndexWriter openForWrite(String name, IndexOptions options) throws IOException {
IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options));
public IndexWriter openForWrite(IndexParams indexParams) throws IOException {
IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions()));
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(name), config);
return new IndexWriter(directory(indexParams.getIndex()), config);
}
private Directory directory(String name) throws IOException {

19
scm-core/src/main/java/sonia/scm/search/IndexNames.java → scm-webapp/src/main/java/sonia/scm/search/IndexParams.java
View File

@@ -24,20 +24,13 @@
package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.Value;
/**
* Names of predefined indexes.
* @since 2.21.0
*/
@Beta
public final class IndexNames {
@Value
public class IndexParams {
/**
* The default index.
*/
public static final String DEFAULT = "_default";
String index;
LuceneSearchableType searchableType;
IndexOptions options;
private IndexNames() {
}
}

19
scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java → scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java
View File

@@ -35,29 +35,28 @@ import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
@Singleton
public class DefaultIndexQueue implements IndexQueue, Closeable {
public class IndexQueue implements Closeable {
private final ExecutorService executor = Executors.newSingleThreadExecutor();
private final AtomicLong size = new AtomicLong(0);
private final SearchEngine searchEngine;
private final LuceneIndexFactory indexFactory;
@Inject
public DefaultIndexQueue(SearchEngine searchEngine) {
this.searchEngine = searchEngine;
public IndexQueue(LuceneIndexFactory indexFactory) {
this.indexFactory = indexFactory;
}
@Override
public Index getQueuedIndex(String name, IndexOptions indexOptions) {
return new QueuedIndex(this, name, indexOptions);
public <T> Index<T> getQueuedIndex(IndexParams indexParams) {
return new QueuedIndex<>(this, indexParams);
}
public SearchEngine getSearchEngine() {
return searchEngine;
public LuceneIndexFactory getIndexFactory() {
return indexFactory;
}
void enqueue(IndexQueueTaskWrapper task) {
<T> void enqueue(IndexQueueTaskWrapper<T> task) {
size.incrementAndGet();
executor.execute(() -> {
task.run();

4
scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java
View File

@@ -25,8 +25,8 @@
package sonia.scm.search;
@FunctionalInterface
public interface IndexQueueTask {
public interface IndexQueueTask<T> {
void updateIndex(Index index);
void updateIndex(Index<T> index);
}

22
scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java
View File

@@ -27,30 +27,28 @@ package sonia.scm.search;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class IndexQueueTaskWrapper implements Runnable {
public final class IndexQueueTaskWrapper<T> implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class);
private final SearchEngine searchEngine;
private final String indexName;
private final IndexOptions options;
private final Iterable<IndexQueueTask> tasks;
private final LuceneIndexFactory indexFactory;
private final IndexParams indexParams;
private final Iterable<IndexQueueTask<T>> tasks;
IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable<IndexQueueTask> tasks) {
this.searchEngine = searchEngine;
this.indexName = indexName;
this.options = options;
IndexQueueTaskWrapper(LuceneIndexFactory indexFactory, IndexParams indexParams, Iterable<IndexQueueTask<T>> tasks) {
this.indexFactory = indexFactory;
this.indexParams = indexParams;
this.tasks = tasks;
}
@Override
public void run() {
try (Index index = searchEngine.getOrCreate(this.indexName, options)) {
for (IndexQueueTask task : tasks) {
try (Index<T> index = indexFactory.create(indexParams)) {
for (IndexQueueTask<T> task : tasks) {
task.updateIndex(index);
}
} catch (Exception e) {
LOG.warn("failure during execution of index task for index {}", indexName, e);
LOG.warn("failure during execution of index task for index {}", indexParams.getIndex(), e);
}
}
}

120
scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java
View File

@@ -30,6 +30,9 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import java.io.IOException;
@@ -39,26 +42,25 @@ import static sonia.scm.search.FieldNames.REPOSITORY;
import static sonia.scm.search.FieldNames.TYPE;
import static sonia.scm.search.FieldNames.UID;
public class LuceneIndex implements Index {
class LuceneIndex<T> implements Index<T> {
private final SearchableTypeResolver resolver;
private final LuceneSearchableType searchableType;
private final IndexWriter writer;
LuceneIndex(SearchableTypeResolver resolver, IndexWriter writer) {
this.resolver = resolver;
LuceneIndex(LuceneSearchableType searchableType, IndexWriter writer) {
this.searchableType = searchableType;
this.writer = writer;
}
@Override
public void store(Id id, String permission, Object object) {
LuceneSearchableType type = resolver.resolve(object);
String uid = createUid(id, type);
Document document = type.getTypeConverter().convert(object);
String uid = createUid(id, searchableType);
Document document = searchableType.getTypeConverter().convert(object);
try {
field(document, UID, uid);
field(document, ID, id.getValue());
id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository));
field(document, TYPE, type.getName());
field(document, TYPE, searchableType.getName());
if (!Strings.isNullOrEmpty(permission)) {
field(document, PERMISSION, permission);
}
@@ -77,37 +79,8 @@ public class LuceneIndex implements Index {
}
@Override
public void delete(Id id, Class<?> type) {
LuceneSearchableType searchableType = resolver.resolve(type);
try {
writer.deleteDocuments(new Term(UID, createUid(id, searchableType)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void deleteByRepository(String repository) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repository));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex);
}
}
@Override
public void deleteByType(Class<?> type) {
LuceneSearchableType searchableType = resolver.resolve(type);
deleteByTypeName(searchableType.getName());
}
@Override
public void deleteByTypeName(String typeName) {
try {
writer.deleteDocuments(new Term(TYPE, typeName));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + typeName + " from index", ex);
}
public Deleter delete() {
return new LuceneDeleter();
}
@Override
@@ -118,4 +91,73 @@ public class LuceneIndex implements Index {
throw new SearchEngineException("failed to close index writer", e);
}
}
private class LuceneDeleter implements Deleter {
@Override
public ByTypeDeleter byType() {
return new LuceneByTypeDeleter();
}
@Override
public AllTypesDeleter allTypes() {
return new LuceneAllTypesDelete();
}
}
@SuppressWarnings("java:S1192")
private class LuceneByTypeDeleter implements ByTypeDeleter {
@Override
public void byId(Id id) {
try {
writer.deleteDocuments(new Term(UID, createUid(id, searchableType)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void all() {
try {
writer.deleteDocuments(new Term(TYPE, searchableType.getName()));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by type " + searchableType.getName() + " from index", ex);
}
}
@Override
public void byRepository(String repositoryId) {
try {
BooleanQuery query = new BooleanQuery.Builder()
.add(new TermQuery(new Term(TYPE, searchableType.getName())), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(REPOSITORY, repositoryId)), BooleanClause.Occur.MUST)
.build();
writer.deleteDocuments(query);
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repositoryId + " from index", ex);
}
}
}
private class LuceneAllTypesDelete implements AllTypesDeleter {
@Override
public void byRepository(String repositoryId) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repositoryId));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete all documents by repository " + repositoryId + " from index", ex);
}
}
@Override
public void byTypeName(String typeName) {
try {
writer.deleteDocuments(new Term(TYPE, typeName));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by type " + typeName + " from index", ex);
}
}
}
}

10
scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java
View File

@@ -29,20 +29,18 @@ import java.io.IOException;
public class LuceneIndexFactory {
private final SearchableTypeResolver typeResolver;
private final IndexOpener indexOpener;
@Inject
public LuceneIndexFactory(SearchableTypeResolver typeResolver, IndexOpener indexOpener) {
this.typeResolver = typeResolver;
public LuceneIndexFactory(IndexOpener indexOpener) {
this.indexOpener = indexOpener;
}
public LuceneIndex create(String name, IndexOptions options) {
public <T> LuceneIndex<T> create(IndexParams indexParams) {
try {
return new LuceneIndex(typeResolver, indexOpener.openForWrite(name, options));
return new LuceneIndex<>(indexParams.getSearchableType(), indexOpener.openForWrite(indexParams));
} catch (IOException ex) {
throw new SearchEngineException("failed to open index " + name, ex);
throw new SearchEngineException("failed to open index " + indexParams.getIndex(), ex);
}
}
}

30
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java
View File

@@ -43,62 +43,50 @@ import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.shiro.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Locale;
import java.util.Optional;
public class LuceneQueryBuilder extends QueryBuilder {
public class LuceneQueryBuilder<T> extends QueryBuilder<T> {
private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class);
private final IndexOpener opener;
private final SearchableTypeResolver resolver;
private final LuceneSearchableType searchableType;
private final String indexName;
private final Analyzer analyzer;
LuceneQueryBuilder(IndexOpener opener, SearchableTypeResolver resolver, String indexName, Analyzer analyzer) {
LuceneQueryBuilder(IndexOpener opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) {
this.opener = opener;
this.resolver = resolver;
this.indexName = indexName;
this.searchableType = searchableType;
this.analyzer = analyzer;
}
@Override
protected Optional<Class<?>> resolveByName(String typeName) {
return resolver.resolveClassByName(typeName);
}
@Override
protected QueryCountResult count(QueryParams queryParams) {
TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
return search(
queryParams, totalHitCountCollector,
(searcher, type, query) -> new QueryCountResult(type.getType(), totalHitCountCollector.getTotalHits())
(searcher, query) -> new QueryCountResult(searchableType.getType(), totalHitCountCollector.getTotalHits())
);
}
@Override
protected QueryResult execute(QueryParams queryParams) {
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
return search(queryParams, topScoreCollector, (searcher, searchableType, query) -> {
return search(queryParams, topScoreCollector, (searcher, query) -> {
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
});
}
private <T> T search(QueryParams queryParams, Collector collector, ResultBuilder<T> resultBuilder) {
private <R> R search(QueryParams queryParams, Collector collector, ResultBuilder<R> resultBuilder) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString());
LuceneSearchableType searchableType = resolver.resolve(queryParams.getType());
searchableType.getPermission().ifPresent(
permission -> SecurityUtils.getSubject().checkPermission(permission)
);
Query parsedQuery = createQuery(searchableType, queryParams, queryString);
Query query = Queries.filter(parsedQuery, searchableType, queryParams);
if (LOG.isDebugEnabled()) {
@@ -109,7 +97,7 @@ public class LuceneQueryBuilder extends QueryBuilder {
searcher.search(query, new PermissionAwareCollector(reader, collector));
return resultBuilder.create(searcher, searchableType, parsedQuery);
return resultBuilder.create(searcher, parsedQuery);
} catch (IOException e) {
throw new SearchEngineException("failed to search index", e);
} catch (InvalidTokenOffsetsException e) {
@@ -196,6 +184,6 @@ public class LuceneQueryBuilder extends QueryBuilder {
@FunctionalInterface
private interface ResultBuilder<T> {
T create(IndexSearcher searcher, LuceneSearchableType searchableType, Query query) throws IOException, InvalidTokenOffsetsException;
T create(IndexSearcher searcher, Query query) throws IOException, InvalidTokenOffsetsException;
}
}

13
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java
View File

@@ -29,18 +29,21 @@ import javax.inject.Inject;
public class LuceneQueryBuilderFactory {
private final IndexOpener indexOpener;
private final SearchableTypeResolver searchableTypeResolver;
private final AnalyzerFactory analyzerFactory;
@Inject
public LuceneQueryBuilderFactory(IndexOpener indexOpener, SearchableTypeResolver searchableTypeResolver, AnalyzerFactory analyzerFactory) {
public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) {
this.indexOpener = indexOpener;
this.searchableTypeResolver = searchableTypeResolver;
this.analyzerFactory = analyzerFactory;
}
public LuceneQueryBuilder create(String name, IndexOptions options) {
return new LuceneQueryBuilder(indexOpener, searchableTypeResolver, name, analyzerFactory.create(options));
public <T> LuceneQueryBuilder<T> create(IndexParams indexParams) {
return new LuceneQueryBuilder<>(
indexOpener,
indexParams.getIndex(),
indexParams.getSearchableType(),
analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())
);
}
}

58
scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java
View File

@@ -34,13 +34,13 @@ import java.util.stream.Collectors;
public class LuceneSearchEngine implements SearchEngine {
private final SearchableTypeResolver resolver;
private final LuceneIndexFactory indexFactory;
private final IndexQueue indexQueue;
private final LuceneQueryBuilderFactory queryBuilderFactory;
@Inject
public LuceneSearchEngine(SearchableTypeResolver resolver, LuceneIndexFactory indexFactory, LuceneQueryBuilderFactory queryBuilderFactory) {
public LuceneSearchEngine(SearchableTypeResolver resolver, IndexQueue indexQueue, LuceneQueryBuilderFactory queryBuilderFactory) {
this.resolver = resolver;
this.indexFactory = indexFactory;
this.indexQueue = indexQueue;
this.queryBuilderFactory = queryBuilderFactory;
}
@@ -54,13 +54,57 @@ public class LuceneSearchEngine implements SearchEngine {
}
@Override
public Index getOrCreate(String name, IndexOptions options) {
return indexFactory.create(name, options);
public <T> ForType<T> forType(Class<T> type) {
return forType(resolver.resolve(type));
}
@Override
public QueryBuilder search(String name, IndexOptions options) {
return queryBuilderFactory.create(name, options);
public ForType<Object> forType(String typeName) {
return forType(resolver.resolveByName(typeName));
}
private <T> ForType<T> forType(LuceneSearchableType searchableType) {
return new LuceneForType<>(searchableType);
}
class LuceneForType<T> implements ForType<T> {
private final LuceneSearchableType searchableType;
private IndexOptions options = IndexOptions.defaults();
private String index = "default";
private LuceneForType(LuceneSearchableType searchableType) {
this.searchableType = searchableType;
}
@Override
public ForType<T> withOptions(IndexOptions options) {
this.options = options;
return this;
}
@Override
public ForType<T> withIndex(String index) {
this.index = index;
return this;
}
private IndexParams params() {
return new IndexParams(index, searchableType, options);
}
@Override
public Index<T> getOrCreate() {
return indexQueue.getQueuedIndex(params());
}
@Override
public QueryBuilder<T> search() {
searchableType.getPermission().ifPresent(
permission -> SecurityUtils.getSubject().checkPermission(permission)
);
return queryBuilderFactory.create(params());
}
}
}

52
scm-webapp/src/main/java/sonia/scm/search/NonNaturalLanguageAnalyzer.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_WORD_PARTS;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.PRESERVE_ORIGINAL;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_NUMERICS;
public class NonNaturalLanguageAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
StandardTokenizer tokenizer = new StandardTokenizer();
TokenStream stream = new WordDelimiterGraphFilter(
tokenizer,
GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | PRESERVE_ORIGINAL,
null // list of words which are protected from being delimited
);
stream = new LowerCaseFilter(stream);
return new TokenStreamComponents(tokenizer, stream);
}
}

86
scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java
View File

@@ -27,50 +27,76 @@ package sonia.scm.search;
import java.util.ArrayList;
import java.util.List;
public class QueuedIndex implements Index {
class QueuedIndex<T> implements Index<T> {
private final DefaultIndexQueue queue;
private final String indexName;
private final IndexOptions indexOptions;
private final IndexQueue queue;
private final IndexParams indexParams;
private final List<IndexQueueTask<T>> tasks = new ArrayList<>();
private final List<IndexQueueTask> tasks = new ArrayList<>();
QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) {
QueuedIndex(IndexQueue queue, IndexParams indexParams) {
this.queue = queue;
this.indexName = indexName;
this.indexOptions = indexOptions;
}
this.indexParams = indexParams;
}
@Override
public void store(Id id, String permission, Object object) {
public void store(Id id, String permission, T object) {
tasks.add(index -> index.store(id, permission, object));
}
@Override
public void delete(Id id, Class<?> type) {
tasks.add(index -> index.delete(id, type));
}
@Override
public void deleteByRepository(String repository) {
tasks.add(index -> index.deleteByRepository(repository));
}
@Override
public void deleteByType(Class<?> type) {
tasks.add(index -> index.deleteByType(type));
}
@Override
public void deleteByTypeName(String typeName) {
tasks.add(index -> index.deleteByTypeName(typeName));
public Deleter delete() {
return new QueueDeleter();
}
@Override
public void close() {
IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper(
queue.getSearchEngine(), indexName, indexOptions, tasks
IndexQueueTaskWrapper<T> wrappedTask = new IndexQueueTaskWrapper<>(
queue.getIndexFactory(), indexParams, tasks
);
queue.enqueue(wrappedTask);
}
private class QueueDeleter implements Deleter {
@Override
public ByTypeDeleter byType() {
return new QueueByTypeDeleter();
}
@Override
public AllTypesDeleter allTypes() {
return new QueueAllTypesDeleter();
}
}
private class QueueByTypeDeleter implements ByTypeDeleter {
@Override
public void byId(Id id) {
tasks.add(index -> index.delete().byType().byId(id));
}
@Override
public void all() {
tasks.add(index -> index.delete().byType().all());
}
@Override
public void byRepository(String repositoryId) {
tasks.add(index -> index.delete().byType().byRepository(repositoryId));
}
}
private class QueueAllTypesDeleter implements AllTypesDeleter {
@Override
public void byRepository(String repositoryId) {
tasks.add(index -> index.delete().allTypes().byRepository(repositoryId));
}
@Override
public void byTypeName(String typeName) {
tasks.add(index -> index.delete().allTypes().byTypeName(typeName));
}
}
}

2
scm-webapp/src/main/java/sonia/scm/search/SearchableField.java
View File

@@ -41,6 +41,7 @@ class SearchableField {
private final boolean defaultQuery;
private final boolean highlighted;
private final PointsConfig pointsConfig;
private final Indexed.Analyzer analyzer;
SearchableField(Field field, Indexed indexed) {
this.name = name(field, indexed);
@@ -50,6 +51,7 @@ class SearchableField {
this.defaultQuery = indexed.defaultQuery();
this.highlighted = indexed.highlighted();
this.pointsConfig = IndexableFields.pointConfig(field);
this.analyzer = indexed.analyzer();
}
Object value(Document document) {

9
scm-webapp/src/main/java/sonia/scm/search/SearchableTypeResolver.java
View File

@@ -35,7 +35,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -94,8 +93,12 @@ class SearchableTypeResolver {
return searchableType;
}
public Optional<Class<?>> resolveClassByName(String typeName) {
return Optional.ofNullable(nameToClass.get(typeName));
public LuceneSearchableType resolveByName(String typeName) {
Class<?> type = nameToClass.get(typeName);
if (type == null) {
throw notFound(entity("type", typeName));
}
return resolve(type);
}
}

26
scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java
View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -41,18 +40,16 @@ import javax.inject.Singleton;
@Singleton
public class UserIndexer implements Indexer<User> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting
static final int VERSION = 1;
private final UserManager userManager;
private final IndexQueue queue;
private final SearchEngine searchEngine;
@Inject
public UserIndexer(UserManager userManager, IndexQueue queue) {
public UserIndexer(UserManager userManager, SearchEngine searchEngine) {
this.userManager = userManager;
this.queue = queue;
this.searchEngine = searchEngine;
}
@Override
@@ -60,11 +57,6 @@ public class UserIndexer implements Indexer<User> {
return User.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Override
public int getVersion() {
return VERSION;
@@ -77,15 +69,15 @@ public class UserIndexer implements Indexer<User> {
@Override
public Updater<User> open() {
return new UserIndexUpdater(userManager, queue.getQueuedIndex(INDEX));
return new UserIndexUpdater(userManager, searchEngine.forType(User.class).getOrCreate());
}
public static class UserIndexUpdater implements Updater<User> {
private final UserManager userManager;
private final Index index;
private final Index<User> index;
private UserIndexUpdater(UserManager userManager, Index index) {
private UserIndexUpdater(UserManager userManager, Index<User> index) {
this.userManager = userManager;
this.index = index;
}
@@ -97,12 +89,12 @@ public class UserIndexer implements Indexer<User> {
@Override
public void delete(User user) {
index.delete(Id.of(user), User.class);
index.delete().byType().byId(Id.of(user));
}
@Override
public void reIndexAll() {
index.deleteByType(User.class);
index.delete().byType().all();
for (User user : userManager.getAll()) {
store(user);
}

11
scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java
View File

@@ -41,7 +41,6 @@ import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryTestData;
import sonia.scm.search.Hit;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
@@ -200,8 +199,9 @@ class SearchResourceTest {
@Test
void shouldReturnCountOnly() throws URISyntaxException {
when(
searchEngine.search(IndexNames.DEFAULT)
.count("string", "Hello")
searchEngine.forType("string")
.search()
.count("Hello")
).thenReturn(new QueryCountResult(String.class, 2L));
MockHttpRequest request = MockHttpRequest.get("/v2/search/query/string?q=Hello&countOnly=true");
@@ -271,10 +271,11 @@ class SearchResourceTest {
private void mockQueryResult(int start, int limit, String query, QueryResult result) {
when(
searchEngine.search(IndexNames.DEFAULT)
searchEngine.forType("string")
.search()
.start(start)
.limit(limit)
.execute("string", query)
.execute(query)
).thenReturn(result);
}

30
scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java
View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
@@ -47,22 +48,17 @@ class GroupIndexerTest {
@Mock
private GroupManager groupManager;
@Mock
private IndexQueue indexQueue;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private SearchEngine searchEngine;
@InjectMocks
private GroupIndexer indexer;
@Test
void shouldReturnRepositoryClass() {
void shouldReturnClass() {
assertThat(indexer.getType()).isEqualTo(Group.class);
}
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(GroupIndexer.INDEX);
}
@Test
void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(GroupIndexer.VERSION);
@@ -71,28 +67,28 @@ class GroupIndexerTest {
@Nested
class UpdaterTests {
@Mock
private Index index;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index<Group> index;
private final Group group = new Group("xml", "astronauts");
@BeforeEach
void open() {
when(indexQueue.getQueuedIndex(GroupIndexer.INDEX)).thenReturn(index);
when(searchEngine.forType(Group.class).getOrCreate()).thenReturn(index);
}
@Test
void shouldStoreRepository() {
void shouldStore() {
indexer.open().store(group);
verify(index).store(Id.of(group), "group:read:astronauts", group);
}
@Test
void shouldDeleteByRepository() {
void shouldDeleteById() {
indexer.open().delete(group);
verify(index).delete(Id.of(group), Group.class);
verify(index.delete().byType()).byId(Id.of(group));
}
@Test
@@ -101,7 +97,7 @@ class GroupIndexerTest {
indexer.open().reIndexAll();
verify(index).deleteByType(Group.class);
verify(index.delete().byType()).all();
verify(index).store(Id.of(group), "group:read:astronauts", group);
}
@@ -111,7 +107,7 @@ class GroupIndexerTest {
indexer.handleEvent(event);
verify(index).delete(Id.of(group), Group.class);
verify(index.delete().byType()).byId(Id.of(group));
}
@Test

26
scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java
View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
@@ -47,8 +48,8 @@ class RepositoryIndexerTest {
@Mock
private RepositoryManager repositoryManager;
@Mock
private IndexQueue indexQueue;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private SearchEngine searchEngine;
@InjectMocks
private RepositoryIndexer indexer;
@@ -58,11 +59,6 @@ class RepositoryIndexerTest {
assertThat(indexer.getType()).isEqualTo(Repository.class);
}
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(RepositoryIndexer.INDEX);
}
@Test
void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(RepositoryIndexer.VERSION);
@@ -71,14 +67,14 @@ class RepositoryIndexerTest {
@Nested
class UpdaterTests {
@Mock
private Index index;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index<Repository> index;
private Repository repository;
@BeforeEach
void open() {
when(indexQueue.getQueuedIndex(RepositoryIndexer.INDEX)).thenReturn(index);
when(searchEngine.forType(Repository.class).getOrCreate()).thenReturn(index);
repository = new Repository();
repository.setId("42");
}
@@ -94,7 +90,7 @@ class RepositoryIndexerTest {
void shouldDeleteByRepository() {
indexer.open().delete(repository);
verify(index).deleteByRepository("42");
verify(index.delete().allTypes()).byRepository("42");
}
@Test
@@ -103,8 +99,8 @@ class RepositoryIndexerTest {
indexer.open().reIndexAll();
verify(index).deleteByTypeName(Repository.class.getName());
verify(index).deleteByType(Repository.class);
verify(index.delete().allTypes()).byTypeName(Repository.class.getName());
verify(index.delete().byType()).all();
verify(index).store(Id.of(repository), "repository:read:42", repository);
}
@@ -115,7 +111,7 @@ class RepositoryIndexerTest {
indexer.handleEvent(event);
verify(index).deleteByRepository("42");
verify(index.delete().allTypes()).byRepository("42");
}
@Test

97
scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java
View File

@@ -24,13 +24,17 @@
package sonia.scm.search;
import lombok.Getter;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat;
@@ -39,40 +43,79 @@ class AnalyzerFactoryTest {
private final AnalyzerFactory analyzerFactory = new AnalyzerFactory();
@Test
void shouldReturnStandardAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults());
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
@Nested
class FromIndexOptionsTests {
@Test
void shouldReturnStandardAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults());
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnStandardAnalyzerForUnknownLocale() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE));
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnEnglishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH));
assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzerForLocaleGermany() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnSpanishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES")));
assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class);
}
}
@Test
void shouldReturnStandardAnalyzerForUnknownLocale() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE));
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
@Nested
class FromSearchableTypeTests {
@Test
void shouldUseDefaultAnalyzerIfNotSpecified() throws IOException {
analyze(Account.class, "simple_text", "description", "simple_text");
}
@Test
void shouldUseNonNaturalLanguageAnalyzer() throws IOException {
analyze(Account.class, "simple_text", "username", "simple", "text", "simple_text");
}
private void analyze(Class<?> type, String text, String field, String... expectedTokens) throws IOException {
LuceneSearchableType searchableType = SearchableTypes.create(type);
IndexOptions defaults = IndexOptions.defaults();
Analyzer analyzer = analyzerFactory.create(searchableType, defaults);
List<String> tokens = Analyzers.tokenize(analyzer, field, text);
assertThat(tokens).containsOnly(expectedTokens);
}
}
@Test
void shouldReturnEnglishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH));
assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class);
}
@Getter
@IndexedType
public static class Account {
@Test
void shouldReturnGermanAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Indexed(analyzer = Indexed.Analyzer.IDENTIFIER)
private String username;
@Test
void shouldReturnGermanAnalyzerForLocaleGermany() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Indexed
private String description;
@Test
void shouldReturnSpanishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES")));
assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class);
}
}

52
scm-core/src/main/java/sonia/scm/search/IndexQueue.java → scm-webapp/src/test/java/sonia/scm/search/Analyzers.java
View File

@@ -24,37 +24,33 @@
package sonia.scm.search;
import com.google.common.annotations.Beta;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Queue the work of indexing.
* An index can't be opened in parallel, so the queue coordinates the work of indexing in an asynchronous manner.
* {@link IndexQueue} should be used most of the time to index content.
*
* @since 2.21.0
*/
@Beta
public interface IndexQueue {
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Returns an index which queues every change to the content.
*
* @param name name of index
* @param indexOptions options for the index
*
* @return index which queues changes
*/
Index getQueuedIndex(String name, IndexOptions indexOptions);
class Analyzers {
/**
* Returns an index which with default options which queues every change to the content.
* @param name name of index
*
* @return index with default options which queues changes
* @see IndexOptions#defaults()
*/
default Index getQueuedIndex(String name) {
return getQueuedIndex(name, IndexOptions.defaults());
private Analyzers() {
}
static List<String> tokenize(Analyzer analyzer, String text) throws IOException {
return tokenize(analyzer, "default", text);
}
static List<String> tokenize(Analyzer analyzer, String field, String text) throws IOException {
List<String> tokens = new ArrayList<>();
try (TokenStream stream = analyzer.tokenStream(field, text)) {
CharTermAttribute attr = stream.addAttribute(CharTermAttribute.class);
stream.reset();
while (stream.incrementToken()) {
tokens.add(attr.toString());
}
}
return tokens;
}
}

6
scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java
View File

@@ -44,14 +44,14 @@ class DefaultIndexLogStoreTest {
@Test
void shouldReturnEmptyOptional() {
Optional<IndexLog> indexLog = indexLogStore.get("index", String.class);
Optional<IndexLog> indexLog = indexLogStore.forIndex("index").get(String.class);
assertThat(indexLog).isEmpty();
}
@Test
void shouldStoreLog() {
indexLogStore.log("index", String.class, 42);
Optional<IndexLog> index = indexLogStore.get("index", String.class);
indexLogStore.forIndex("index").log(String.class, 42);
Optional<IndexLog> index = indexLogStore.forIndex("index").get(String.class);
assertThat(index).hasValueSatisfying(log -> {
assertThat(log.getVersion()).isEqualTo(42);
assertThat(log.getDate()).isNotNull();

12
scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java
View File

@@ -26,6 +26,7 @@ package sonia.scm.search;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.group.Group;
@@ -53,7 +54,7 @@ class IndexBootstrapListenerTest {
@Mock
private AdministrationContext administrationContext;
@Mock
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private IndexLogStore indexLogStore;
@Test
@@ -67,7 +68,7 @@ class IndexBootstrapListenerTest {
verify(updater).reIndexAll();
verify(updater).close();
verify(indexLogStore).log(IndexNames.DEFAULT, Repository.class, 1);
verify(indexLogStore.defaultIndex()).log(Repository.class, 1);
}
@Test
@@ -81,7 +82,7 @@ class IndexBootstrapListenerTest {
verify(updater).reIndexAll();
verify(updater).close();
verify(indexLogStore).log(IndexNames.DEFAULT, User.class, 2);
verify(indexLogStore.defaultIndex()).log(User.class, 2);
}
@Test
@@ -103,7 +104,7 @@ class IndexBootstrapListenerTest {
}
private <T> void mockIndexLog(Class<T> type, @Nullable IndexLog indexLog) {
when(indexLogStore.get(IndexNames.DEFAULT, type)).thenReturn(Optional.ofNullable(indexLog));
when(indexLogStore.defaultIndex().get(type)).thenReturn(Optional.ofNullable(indexLog));
}
private void mockAdminContext() {
@@ -128,14 +129,15 @@ class IndexBootstrapListenerTest {
);
}
@SuppressWarnings("unchecked")
private <T> Indexer<T> indexer(Class<T> type, int version) {
Indexer<T> indexer = mock(Indexer.class);
when(indexer.getType()).thenReturn(type);
when(indexer.getVersion()).thenReturn(version);
when(indexer.getIndex()).thenReturn(IndexNames.DEFAULT);
return indexer;
}
@SuppressWarnings("unchecked")
private <T> Indexer.Updater<T> updater(Indexer<T> indexer) {
Indexer.Updater<T> updater = mock(Indexer.Updater.class);
when(indexer.open()).thenReturn(updater);

29
scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java
View File

@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -54,6 +55,9 @@ class IndexOpenerTest {
@Mock
private AnalyzerFactory analyzerFactory;
@Mock
private LuceneSearchableType searchableType;
private IndexOpener indexOpener;
@BeforeEach
@@ -61,35 +65,50 @@ class IndexOpenerTest {
this.directory = tempDirectory;
SCMContextProvider context = mock(SCMContextProvider.class);
when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory);
when(analyzerFactory.create(any(IndexOptions.class))).thenReturn(new SimpleAnalyzer());
when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer());
indexOpener = new IndexOpener(context, analyzerFactory);
}
@Test
void shouldCreateNewIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) {
try (IndexWriter writer = open("new-index")) {
addDoc(writer, "Trillian");
}
assertThat(directory.resolve("new-index")).exists();
}
private IndexWriter open(String index) throws IOException {
return indexOpener.openForWrite(new IndexParams(index, searchableType, IndexOptions.defaults()));
}
@Test
void shouldOpenExistingIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) {
try (IndexWriter writer = open("reused")) {
addDoc(writer, "Dent");
}
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) {
try (IndexWriter writer = open("reused")) {
assertThat(writer.getFieldNames()).contains("hitchhiker");
}
}
@Test
void shouldUseAnalyzerFromFactory() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) {
try (IndexWriter writer = open("new-index")) {
assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class);
}
}
@Test
void shouldOpenIndexForRead() throws IOException {
try (IndexWriter writer = open("idx-for-read")) {
addDoc(writer, "Dent");
}
try (IndexReader reader = indexOpener.openForRead("idx-for-read")) {
assertThat(reader.numDocs()).isOne();
}
}
private void addDoc(IndexWriter writer, String name) throws IOException {
Document doc = new Document();
doc.add(new TextField("hitchhiker", name, Field.Store.YES));

31
scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java → scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java
View File

@@ -43,35 +43,31 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.in;
import static org.awaitility.Awaitility.await;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class DefaultIndexQueueTest {
class IndexQueueTest {
private Directory directory;
private DefaultIndexQueue queue;
@Mock
private LuceneQueryBuilderFactory queryBuilderFactory;
private IndexQueue queue;
@BeforeEach
void createQueue() throws IOException {
directory = new ByteBuffersDirectory();
IndexOpener opener = mock(IndexOpener.class);
when(opener.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
when(opener.openForWrite(any(IndexParams.class))).thenAnswer(ic -> {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, config);
});
SearchableTypeResolver resolver = new SearchableTypeResolver(Account.class, IndexedNumber.class);
LuceneIndexFactory indexFactory = new LuceneIndexFactory(resolver, opener);
SearchEngine engine = new LuceneSearchEngine(resolver, indexFactory, queryBuilderFactory);
queue = new DefaultIndexQueue(engine);
LuceneIndexFactory indexFactory = new LuceneIndexFactory(opener);
queue = new IndexQueue(indexFactory);
}
@AfterEach
@@ -82,13 +78,20 @@ class DefaultIndexQueueTest {
@Test
void shouldWriteToIndex() throws Exception {
try (Index index = queue.getQueuedIndex("default")) {
try (Index<Account> index = getIndex(Account.class)) {
index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan"));
index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent"));
}
assertDocCount(2);
}
private <T> Index<T> getIndex(Class<T> type) {
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneSearchableType searchableType = resolver.resolve(type);
IndexParams indexParams = new IndexParams("default", searchableType, IndexOptions.defaults());
return queue.getQueuedIndex(indexParams);
}
@Test
void shouldWriteMultiThreaded() throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(4);
@@ -96,8 +99,8 @@ class DefaultIndexQueueTest {
executorService.execute(new IndexNumberTask(i));
}
executorService.execute(() -> {
try (Index index = queue.getQueuedIndex("default")) {
index.delete(Id.of(String.valueOf(12)), IndexedNumber.class);
try (Index<IndexedNumber> index = getIndex(IndexedNumber.class)) {
index.delete().byType().byId(Id.of(String.valueOf(12)));
}
});
executorService.shutdown();
@@ -141,7 +144,7 @@ class DefaultIndexQueueTest {
@Override
public void run() {
try (Index index = queue.getQueuedIndex("default")) {
try (Index<IndexedNumber> index = getIndex(IndexedNumber.class)) {
index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number));
}
}

109
scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java
View File

@@ -49,6 +49,7 @@ import static sonia.scm.search.FieldNames.*;
class LuceneIndexTest {
private static final Id ONE = Id.of("one");
private static final Id TWO = Id.of("two");
private Directory directory;
@@ -59,7 +60,7 @@ class LuceneIndexTest {
@Test
void shouldStoreObject() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
}
@@ -68,7 +69,7 @@ class LuceneIndexTest {
@Test
void shouldUpdateObject() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
index.store(ONE, null, new Storable("Awesome content"));
}
@@ -78,7 +79,7 @@ class LuceneIndexTest {
@Test
void shouldStoreUidOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
}
@@ -87,7 +88,7 @@ class LuceneIndexTest {
@Test
void shouldStoreIdOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some text"));
}
@@ -96,7 +97,7 @@ class LuceneIndexTest {
@Test
void shouldStoreRepositoryOfId() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("Some text"));
}
@@ -105,7 +106,7 @@ class LuceneIndexTest {
@Test
void shouldStoreTypeOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text"));
}
@@ -114,12 +115,12 @@ class LuceneIndexTest {
@Test
void shouldDeleteById() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text"));
}
try (LuceneIndex index = createIndex()) {
index.delete(ONE, Storable.class);
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byId(ONE);
}
assertHits(ID, "one", 0);
@@ -127,14 +128,17 @@ class LuceneIndexTest {
@Test
void shouldDeleteAllByType() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("content"));
index.store(Id.of("two"), null, new Storable("content"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(Id.of("three"), null, new OtherStorable("content"));
}
try (LuceneIndex index = createIndex()) {
index.deleteByType(Storable.class);
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().all();
}
assertHits("value", "content", 1);
@@ -142,13 +146,16 @@ class LuceneIndexTest {
@Test
void shouldDeleteByIdAnyType() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE, null, new OtherStorable("Some other text"));
}
try (LuceneIndex index = createIndex()) {
index.delete(ONE, Storable.class);
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byId(ONE);
}
assertHits(ID, "one", 1);
@@ -160,13 +167,13 @@ class LuceneIndexTest {
@Test
void shouldDeleteByIdAndRepository() throws IOException {
Id withRepository = ONE.withRepository("4211");
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text"));
index.store(withRepository, null, new Storable("New stuff"));
}
try (LuceneIndex index = createIndex()) {
index.delete(withRepository, Storable.class);
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byId(withRepository);
}
ScoreDoc[] docs = assertHits(ID, "one", 1);
@@ -176,21 +183,73 @@ class LuceneIndexTest {
@Test
void shouldDeleteByRepository() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("Some other text"));
index.store(ONE.withRepository("4212"), null, new Storable("New stuff"));
}
try (LuceneIndex index = createIndex()) {
index.deleteByRepository("4212");
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byRepository("4212");
}
assertHits(ID, "one", 1);
}
@Test
void shouldDeleteByRepositoryAndType() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("some text"));
index.store(TWO.withRepository("4211"), null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE.withRepository("4211"), null, new OtherStorable("some text"));
}
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byRepository("4211");
}
ScoreDoc[] docs = assertHits("value", "text", 1);
Document doc = doc(docs[0].doc);
assertThat(doc.get(TYPE)).isEqualTo("otherStorable");
}
@Test
void shouldDeleteAllByRepository() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("some text"));
index.store(TWO.withRepository("4211"), null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE.withRepository("4211"), null, new OtherStorable("some text"));
}
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().allTypes().byRepository("4211");
}
assertHits("value", "text", 0);
}
@Test
void shouldDeleteAllByTypeName() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("some text"));
index.store(TWO, null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.delete().allTypes().byTypeName("storable");
}
assertHits("value", "text", 0);
}
@Test
void shouldStorePermission() throws IOException {
try (LuceneIndex index = createIndex()) {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text"));
}
@@ -213,9 +272,9 @@ class LuceneIndexTest {
}
}
private LuceneIndex createIndex() throws IOException {
SearchableTypeResolver resolver = new SearchableTypeResolver(Storable.class, OtherStorable.class);
return new LuceneIndex(resolver, createWriter());
private <T> LuceneIndex<T> createIndex(Class<T> type) throws IOException {
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
return new LuceneIndex<>(resolver.resolve(type), createWriter());
}
private IndexWriter createWriter() throws IOException {

43
scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java
View File

@@ -40,7 +40,6 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.shiro.authz.AuthorizationException;
@@ -305,10 +304,10 @@ class LuceneQueryBuilderTest {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, resolver, "default", new StandardAnalyzer()
LuceneQueryBuilder<Simple> builder = new LuceneQueryBuilder<>(
opener, "default", resolver.resolve(Simple.class), new StandardAnalyzer()
);
result = builder.repository("cde").execute(Simple.class, "content:awesome");
result = builder.repository("cde").execute("content:awesome");
}
assertThat(result.getTotalHits()).isOne();
@@ -434,24 +433,6 @@ class LuceneQueryBuilderTest {
assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something"));
}
@Test
void shouldFailWithoutPermissionForTheSearchedType() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(denyDoc("awesome"));
}
assertThrows(AuthorizationException.class, () -> query(Deny.class, "awesome"));
}
@Test
@SubjectAware(value = "marvin", permissions = "deny:4711")
void shouldNotFailWithRequiredPermissionForTheSearchedType() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(denyDoc("awesome"));
}
QueryResult result = query(Deny.class, "awesome");
assertThat(result.getTotalHits()).isOne();
}
@Test
void shouldLimitHitsByDefaultSize() throws IOException {
try (IndexWriter writer = writer()) {
@@ -577,23 +558,25 @@ class LuceneQueryBuilderTest {
return query(type, queryString, null, null);
}
private long count(Class<?> type, String queryString) throws IOException {
private <T> long count(Class<T> type, String queryString) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, resolver, "default", new StandardAnalyzer()
LuceneSearchableType searchableType = resolver.resolve(type);
LuceneQueryBuilder<T> builder = new LuceneQueryBuilder<T>(
opener, "default", searchableType, new StandardAnalyzer()
);
return builder.count(type, queryString).getTotalHits();
return builder.count(queryString).getTotalHits();
}
}
private QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
private <T> QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, resolver, "default", new StandardAnalyzer()
LuceneSearchableType searchableType = resolver.resolve(type);
LuceneQueryBuilder<T> builder = new LuceneQueryBuilder<T>(
opener, "default", searchableType, new StandardAnalyzer()
);
if (start != null) {
builder.start(start);
@@ -601,7 +584,7 @@ class LuceneQueryBuilderTest {
if (limit != null) {
builder.limit(limit);
}
return builder.execute(type, queryString);
return builder.execute(queryString);
}
}

109
scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java
View File

@@ -24,6 +24,7 @@
package sonia.scm.search;
import org.apache.shiro.authz.AuthorizationException;
import org.github.sdorra.jse.ShiroExtension;
import org.github.sdorra.jse.SubjectAware;
import org.junit.jupiter.api.Test;
@@ -31,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.Repository;
import java.util.Arrays;
import java.util.Collection;
@@ -40,6 +42,8 @@ import java.util.Locale;
import java.util.Optional;
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -52,7 +56,7 @@ class LuceneSearchEngineTest {
private SearchableTypeResolver resolver;
@Mock
private LuceneIndexFactory indexFactory;
private IndexQueue indexQueue;
@Mock
private LuceneQueryBuilderFactory queryBuilderFactory;
@@ -60,6 +64,9 @@ class LuceneSearchEngineTest {
@InjectMocks
private LuceneSearchEngine searchEngine;
@Mock
private LuceneSearchableType searchableType;
@Test
void shouldDelegateGetSearchableTypes() {
List<LuceneSearchableType> mockedTypes = Collections.singletonList(searchableType("repository"));
@@ -96,43 +103,119 @@ class LuceneSearchEngineTest {
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateWithDefaultIndex() {
Index<Repository> index = mock(Index.class);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults());
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).getOrCreate();
assertThat(idx).isSameAs(index);
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateIndexWithDefaults() {
LuceneIndex index = mock(LuceneIndex.class);
when(indexFactory.create("idx", IndexOptions.defaults())).thenReturn(index);
Index<Repository> index = mock(Index.class);
Index idx = searchEngine.getOrCreate("idx");
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("idx", searchableType, IndexOptions.defaults());
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).withIndex("idx").getOrCreate();
assertThat(idx).isSameAs(index);
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateIndex() {
LuceneIndex index = mock(LuceneIndex.class);
Index<Repository> index = mock(Index.class);
IndexOptions options = IndexOptions.naturalLanguage(Locale.ENGLISH);
when(indexFactory.create("idx", options)).thenReturn(index);
Index idx = searchEngine.getOrCreate("idx", options);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("default", searchableType, options);
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).withOptions(options).getOrCreate();
assertThat(idx).isSameAs(index);
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateSearchWithDefaults() {
LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create("idx", IndexOptions.defaults())).thenReturn(mockedBuilder);
LuceneQueryBuilder<Repository> mockedBuilder = mock(LuceneQueryBuilder.class);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
QueryBuilder queryBuilder = searchEngine.search("idx");
IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults());
when(queryBuilderFactory.<Repository>create(params)).thenReturn(mockedBuilder);
QueryBuilder<Repository> queryBuilder = searchEngine.forType(Repository.class).search();
assertThat(queryBuilder).isSameAs(mockedBuilder);
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateSearch() {
LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class);
IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN);
when(queryBuilderFactory.create("idx", options)).thenReturn(mockedBuilder);
QueryBuilder queryBuilder = searchEngine.search("idx", options);
LuceneQueryBuilder<Repository> mockedBuilder = mock(LuceneQueryBuilder.class);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("idx", searchableType, options);
when(queryBuilderFactory.<Repository>create(params)).thenReturn(mockedBuilder);
QueryBuilder<Repository> queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search();
assertThat(queryBuilder).isSameAs(mockedBuilder);
}
@Test
void shouldFailWithoutRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
SearchEngine.ForType<Repository> forType = searchEngine.forType(Repository.class);
assertThrows(AuthorizationException.class, forType::search);
}
@Test
@SuppressWarnings("unchecked")
@SubjectAware(permissions = "repository:read")
void shouldNotFailWithRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
LuceneQueryBuilder<Object> mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder);
SearchEngine.ForType<Repository> forType = searchEngine.forType(Repository.class);
assertThat(forType.search()).isNotNull();
}
@Test
void shouldFailWithTypeNameWithoutRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolveByName("repository")).thenReturn(searchableType);
SearchEngine.ForType<Object> forType = searchEngine.forType("repository");
assertThrows(AuthorizationException.class, forType::search);
}
@Test
@SuppressWarnings("unchecked")
@SubjectAware(permissions = "repository:read")
void shouldNotFailWithTypeNameAndRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolveByName("repository")).thenReturn(searchableType);
LuceneQueryBuilder<Object> mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder);
SearchEngine.ForType<Object> forType = searchEngine.forType("repository");
assertThat(forType.search()).isNotNull();
}
}

75
scm-webapp/src/test/java/sonia/scm/search/NonNaturalLanguageAnalyzerTest.java Normal file
View File

@@ -0,0 +1,75 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat;
class NonNaturalLanguageAnalyzerTest {
private final NonNaturalLanguageAnalyzer analyzer = new NonNaturalLanguageAnalyzer();
@ParameterizedTest
@ValueSource(strings = {
"simple text", "simple-text", "simple(text)", "simple[text]",
"simple{text}", "simple/text", "simple;text", "simple$text",
"simple\\text"
})
void shouldTokenize(String value) throws IOException {
List<String> tokens = tokenize(value);
assertThat(tokens).containsOnly("simple", "text");
}
@ParameterizedTest
@ValueSource(strings = {
"simple.text", "simple_text", "simpleText", "simple:text", "SimpleText"
})
void shouldTokenizeAndPreserveOriginal(String value) throws IOException {
List<String> tokens = tokenize(value);
assertThat(tokens).containsOnly("simple", "text", value.toLowerCase(Locale.ENGLISH));
}
@Test
void shouldSplitOnNumeric() throws IOException {
List<String> tokens = tokenize("simple42text");
assertThat(tokens).containsOnly("simple", "42", "text", "simple42text");
}
private List<String> tokenize(String text) throws IOException {
return Analyzers.tokenize(analyzer, text);
}
}

30
scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java
View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
@@ -47,22 +48,17 @@ class UserIndexerTest {
@Mock
private UserManager userManager;
@Mock
private IndexQueue indexQueue;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private SearchEngine searchEngine;
@InjectMocks
private UserIndexer indexer;
@Test
void shouldReturnRepositoryClass() {
void shouldReturnType() {
assertThat(indexer.getType()).isEqualTo(User.class);
}
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(UserIndexer.INDEX);
}
@Test
void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(UserIndexer.VERSION);
@@ -71,28 +67,28 @@ class UserIndexerTest {
@Nested
class UpdaterTests {
@Mock
private Index index;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index<User> index;
private final User user = UserTestData.createTrillian();
@BeforeEach
void open() {
when(indexQueue.getQueuedIndex(UserIndexer.INDEX)).thenReturn(index);
when(searchEngine.forType(User.class).getOrCreate()).thenReturn(index);
}
@Test
void shouldStoreRepository() {
void shouldStore() {
indexer.open().store(user);
verify(index).store(Id.of(user), "user:read:trillian", user);
}
@Test
void shouldDeleteByRepository() {
void shouldDeleteById() {
indexer.open().delete(user);
verify(index).delete(Id.of(user), User.class);
verify(index.delete().byType()).byId(Id.of(user));
}
@Test
@@ -101,7 +97,7 @@ class UserIndexerTest {
indexer.open().reIndexAll();
verify(index).deleteByType(User.class);
verify(index.delete().byType()).all();
verify(index).store(Id.of(user), "user:read:trillian", user);
}
@@ -111,7 +107,7 @@ class UserIndexerTest {
indexer.handleEvent(event);
verify(index).delete(Id.of(user), User.class);
verify(index.delete().byType()).byId(Id.of(user));
}
@Test