Refactor Search API and allow analyzer per field (#1755)

The Search api is now simpler, because it provides useful defaults. Only if you want to deviate from the defaults, you can set these values. This is mostly reached by using the builder pattern. Furthermore it is now possible to configure an analyzer per field. The default analyzer is still the one which is derived from the index options, but it is possible to configure a new indexer with the analyzer attribute of the indexed annotation. The attribute allows the configuration for code, identifiers and path. The current implementation uses the same analyzer code, identifiers and path. The new implemented splits tokens on more delimiters as the default analyzer e.g.: dots, underscores etc.

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>
This commit is contained in:
Sebastian Sdorra
2021-08-05 08:21:46 +02:00
committed by GitHub
parent 4fa5ad1f0d
commit 21a6943980
47 changed files with 1069 additions and 574 deletions

View File

@@ -0,0 +1,4 @@
- type: Added
description: Support for different types of analyzer per field ([#1755](https://github.com/scm-manager/scm-manager/pull/1755))
- type: Changed
description: Improve Search API ([#1755](https://github.com/scm-manager/scm-manager/pull/1755))

View File

@@ -80,6 +80,42 @@ public @interface Indexed {
*/ */
boolean highlighted() default false; boolean highlighted() default false;
/**
* Describes how the field is analyzed and tokenized.
*
* @return type of analyzer
* @since 2.23.0
*/
Analyzer analyzer() default Analyzer.DEFAULT;
/**
* Describes how fields are analyzed and tokenized.
*
* @since 2.23.0
*/
enum Analyzer {
/**
* Uses the analyzer which was used to open the index.
*/
DEFAULT,
/**
* Uses an analyzer which is specialized for identifiers like repository names.
*/
IDENTIFIER,
/**
* Uses an analyzer which is specialized for paths.
*/
PATH,
/**
* Uses an analyzer which is specialized for source code.
*/
CODE
}
/** /**
* Describes how the field is indexed. * Describes how the field is indexed.
*/ */

View File

@@ -70,10 +70,10 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per
private String id; private String id;
@Indexed(defaultQuery = true, boost = 1.25f) @Indexed(defaultQuery = true, boost = 1.25f, analyzer = Indexed.Analyzer.IDENTIFIER)
private String namespace; private String namespace;
@Indexed(defaultQuery = true, boost = 1.5f) @Indexed(defaultQuery = true, boost = 1.5f, analyzer = Indexed.Analyzer.IDENTIFIER)
private String name; private String name;
@Indexed(type = Indexed.Type.SEARCHABLE) @Indexed(type = Indexed.Type.SEARCHABLE)
private String type; private String type;

View File

@@ -28,11 +28,11 @@ import com.google.common.annotations.Beta;
/** /**
* Can be used to index objects for full text searches. * Can be used to index objects for full text searches.
* * @param <T> type of indexed objects
* @since 2.21.0 * @since 2.21.0
*/ */
@Beta @Beta
public interface Index extends AutoCloseable { public interface Index<T> extends AutoCloseable {
/** /**
* Store the given object in the index. * Store the given object in the index.
@@ -44,39 +44,87 @@ public interface Index extends AutoCloseable {
* *
* @see Indexed * @see Indexed
*/ */
void store(Id id, String permission, Object object); void store(Id id, String permission, T object);
/** /**
* Delete the object with the given id and type from index. * Delete provides an api to delete objects from the index
* * @return delete api
* @param id id of object * @since 2.23.0
* @param type type of object
*/ */
void delete(Id id, Class<?> type); Deleter delete();
/**
* Delete all objects which are related the given repository from index.
*
* @param repositoryId id of repository
*/
void deleteByRepository(String repositoryId);
/**
* Delete all objects with the given type from index.
* @param type type of objects
*/
void deleteByType(Class<?> type);
/**
* Delete all objects with the given type from index.
* This method is mostly if the index type has changed and the old type (in form of class) is no longer available.
* @param typeName type name of objects
*/
void deleteByTypeName(String typeName);
/** /**
* Close index and commit changes. * Close index and commit changes.
*/ */
@Override @Override
void close(); void close();
/**
* Deleter provides an api to delete object from index.
*
* @since 2.23.0
*/
interface Deleter {
/**
* Returns an api which allows deletion of objects from the type of this index.
* @return type restricted delete api
*/
ByTypeDeleter byType();
/**
* Returns an api which allows deletion of objects of every type.
* @return unrestricted delete api for all types.
*/
AllTypesDeleter allTypes();
}
/**
* Delete api for the type of the index. This means, that only entries for this
* type will be deleted.
*
* @since 2.23.0
*/
interface ByTypeDeleter {
/**
* Delete the object with the given id and type from index.
* @param id id of object
*/
void byId(Id id);
/**
* Delete all objects of the given type from index.
*/
void all();
/**
* Delete all objects which are related the given type and repository from index.
*
* @param repositoryId id of repository
*/
void byRepository(String repositoryId);
}
/**
* Delete api for the overall index regarding all types.
*
* @since 2.23.0
*/
interface AllTypesDeleter {
/**
* Delete all objects which are related to the given repository from index regardless of their type.
* @param repositoryId repository id
*/
void byRepository(String repositoryId);
/**
* Delete all objects with the given type from index.
* This method is mostly useful if the index type has changed and the old type (in form of a class)
* is no longer available.
* @param typeName type name of objects
*/
void byTypeName(String typeName);
}
} }

View File

@@ -36,7 +36,8 @@ import java.time.Instant;
/** /**
* A marker keeping track of when and with which model version an object type was last indexed. * A marker keeping track of when and with which model version an object type was last indexed.
* @since 2.21 *
* @since 2.21.0
*/ */
@Beta @Beta
@Data @Data

View File

@@ -39,23 +39,42 @@ import java.util.Optional;
public interface IndexLogStore { public interface IndexLogStore {
/** /**
* Log index and version of a type which is now indexed. * Returns an index log store for the given index.
*
* @param index name of index * @param index name of index
* @param type type which was indexed * @return index log store for given index
* @param version model version * @since 2.23.0
*/ */
void log(String index, Class<?> type, int version); ForIndex forIndex(String index);
/** /**
* Returns version and date of the indexed type or an empty object, * Returns the index log store for the default index.
* if the object was not indexed at all. * @return index log store for default index
* * @since 2.23.0
* @param index name if index
* @param type type of object
*
* @return log entry or empty
*/ */
Optional<IndexLog> get(String index, Class<?> type); ForIndex defaultIndex();
/**
* Index log store for a specific index.
* @since 2.23.0
*/
interface ForIndex {
/**
* Log index and version of a type which is now indexed.
*
* @param type type which was indexed
* @param version model version
*/
void log(Class<?> type, int version);
/**
* Returns version and date of the indexed type or an empty object,
* if the object was not indexed at all.
*
* @param type type of object
*
* @return log entry or empty
*/
Optional<IndexLog> get(Class<?> type);
}
} }

View File

@@ -45,13 +45,6 @@ public interface Indexer<T> {
*/ */
Class<T> getType(); Class<T> getType();
/**
* Returns name of index.
*
* @return name of index
*/
String getIndex();
/** /**
* Returns version of index type. * Returns version of index type.
* The version should be increased if the type changes and should be re indexed. * The version should be increased if the type changes and should be re indexed.

View File

@@ -37,22 +37,22 @@ import static sonia.scm.NotFoundException.notFound;
/** /**
* Build and execute queries against an index. * Build and execute queries against an index.
* *
* @param <T> type of indexed objects
* @since 2.21.0 * @since 2.21.0
*/ */
@Beta @Beta
public abstract class QueryBuilder { public abstract class QueryBuilder<T> {
private String repositoryId; private String repositoryId;
private int start = 0; private int start = 0;
private int limit = 10; private int limit = 10;
/** /**
* Return only results which are related to the given repository. * Return only results which are related to the given repository.
* @param repository repository * @param repository repository
* @return {@code this} * @return {@code this}
*/ */
public QueryBuilder repository(Repository repository) { public QueryBuilder<T> repository(Repository repository) {
return repository(repository.getId()); return repository(repository.getId());
} }
@@ -61,7 +61,7 @@ public abstract class QueryBuilder {
* @param repositoryId id of the repository * @param repositoryId id of the repository
* @return {@code this} * @return {@code this}
*/ */
public QueryBuilder repository(String repositoryId) { public QueryBuilder<T> repository(String repositoryId) {
this.repositoryId = repositoryId; this.repositoryId = repositoryId;
return this; return this;
} }
@@ -72,7 +72,7 @@ public abstract class QueryBuilder {
* @param start start of result * @param start start of result
* @return {@code this} * @return {@code this}
*/ */
public QueryBuilder start(int start) { public QueryBuilder<T> start(int start) {
this.start = start; this.start = start;
return this; return this;
} }
@@ -82,7 +82,7 @@ public abstract class QueryBuilder {
* @param limit limit of hits * @param limit limit of hits
* @return {@code this} * @return {@code this}
*/ */
public QueryBuilder limit(int limit) { public QueryBuilder<T> limit(int limit) {
this.limit = limit; this.limit = limit;
return this; return this;
} }
@@ -90,61 +90,26 @@ public abstract class QueryBuilder {
/** /**
* Executes the query and returns the matches. * Executes the query and returns the matches.
* *
* @param type type of objects which are searched
* @param queryString searched query * @param queryString searched query
* @return result of query * @return result of query
*/ */
public QueryResult execute(Class<?> type, String queryString){ public QueryResult execute(String queryString){
return execute(new QueryParams(type, repositoryId, queryString, start, limit)); return execute(new QueryParams(repositoryId, queryString, start, limit));
} }
/** /**
* Executes the query and returns the total count of hits. * Executes the query and returns the total count of hits.
* *
* @param type type of objects which are searched
* @param queryString searched query * @param queryString searched query
* *
* @return total count of hits * @return total count of hits
* @since 2.22.0 * @since 2.22.0
*/ */
public QueryCountResult count(Class<?> type, String queryString) { public QueryCountResult count(String queryString) {
return count(new QueryParams(type, repositoryId, queryString, start, limit)); return count(new QueryParams(repositoryId, queryString, start, limit));
} }
/**
* Executes the query and returns the matches.
*
* @param typeName type name of objects which are searched
* @param queryString searched query
* @return result of query
*
* @throws NotFoundException if type could not be found
*/
public QueryResult execute(String typeName, String queryString){
return execute(resolveTypeByName(typeName), queryString);
}
/**
* Executes the query and returns the total count of hits.
*
* @param typeName type name of objects which are searched
* @param queryString searched query
*
* @return total count of hits
* @since 2.22.0
*/
public QueryCountResult count(String typeName, String queryString) {
return count(resolveTypeByName(typeName), queryString);
}
/**
* Resolves the type by its name. Returns optional with class of type or an empty optional.
*
* @param typeName name of type
* @return optional with class of type or empty
*/
protected abstract Optional<Class<?>> resolveByName(String typeName);
/** /**
* Executes the query and returns the matches. * Executes the query and returns the matches.
@@ -163,16 +128,11 @@ public abstract class QueryBuilder {
return execute(queryParams); return execute(queryParams);
} }
private Class<?> resolveTypeByName(String typeName) {
return resolveByName(typeName).orElseThrow(() -> notFound(entity("type", typeName)));
}
/** /**
* The searched query and all parameters, which belong to the query. * The searched query and all parameters, which belong to the query.
*/ */
@Value @Value
static class QueryParams { static class QueryParams {
Class<?> type;
String repositoryId; String repositoryId;
String queryString; String queryString;
int start; int start;

View File

@@ -30,10 +30,7 @@ import java.util.Collection;
/** /**
* The {@link SearchEngine} is the main entry point for indexing and searching. * The {@link SearchEngine} is the main entry point for indexing and searching.
* Note that this is kind of a low level api for indexing.
* For non expert indexing the {@link IndexQueue} should be used.
* *
* @see IndexQueue
* @since 2.21.0 * @since 2.21.0
*/ */
@Beta @Beta
@@ -47,46 +44,58 @@ public interface SearchEngine {
Collection<SearchableType> getSearchableTypes(); Collection<SearchableType> getSearchableTypes();
/** /**
* Returns the index with the given name and the given options. * Returns a type specific api which can be used to index objects of that specific type.
* The index is created if it does not exist.
* Warning: Be careful, because an index can't be opened multiple times in parallel.
* If you are not sure how you should index your objects, use the {@link IndexQueue}.
* *
* @param name name of the index * @param type type of object
* @param options index options * @param <T> type of object
* @return existing index or a new one if none exists * @return type specific index and search api
* @since 2.23.0
*/ */
Index getOrCreate(String name, IndexOptions options); <T> ForType<T> forType(Class<T> type);
/** /**
* Same as {@link #getOrCreate(String, IndexOptions)} with default options. * Returns an api which can be used to index and search object of that type.
* * @param name name of type
* @param name name of the index * @return search and index api
* @return existing index or a new one if none exists * @since 2.23.0
* @see IndexOptions#defaults()
*/ */
default Index getOrCreate(String name) { ForType<Object> forType(String name);
return getOrCreate(name, IndexOptions.defaults());
}
/** /**
* Search the index. * Search and index api.
* Returns a {@link QueryBuilder} which allows to query the index.
* *
* @param name name of the index * @param <T> type of searchable objects
* @param options options for searching the index * @since 2.23.0
* @return query builder
*/ */
QueryBuilder search(String name, IndexOptions options); interface ForType<T> {
/** /**
* Same as {@link #search(String, IndexOptions)} with default options. * Specify options for the index.
* * If not used the default options will be used.
* @param name name of the index * @param options index options
* @return query builder * @return {@code this}
* @see IndexOptions#defaults() * @see IndexOptions#defaults()
*/ */
default QueryBuilder search(String name) { ForType<T> withOptions(IndexOptions options);
return search(name, IndexOptions.defaults());
/**
* Name of the index which should be used.
* If not specified the default index will be used.
* @param name name of index
* @return {@code this}
*/
ForType<T> withIndex(String name);
/**
* Returns an index object which provides method to update the search index.
* @return index object
*/
Index<T> getOrCreate();
/**
* Returns a query builder object which can be used to search the index.
* @return query builder
*/
QueryBuilder<T> search();
} }
} }

View File

@@ -24,28 +24,37 @@
import React, { FC } from "react"; import React, { FC } from "react";
import { Hit as HitType } from "@scm-manager/ui-types"; import { Hit as HitType } from "@scm-manager/ui-types";
import classNames from "classnames";
export type HitProps = { export type HitProps = {
hit: HitType; hit: HitType;
}; };
type Props = {
className?: string;
};
type SearchResultType = FC & { type SearchResultType = FC & {
Title: FC; Title: FC<Props>;
Left: FC; Left: FC<Props>;
Content: FC; Content: FC<Props>;
Right: FC; Right: FC<Props>;
}; };
const Hit: SearchResultType = ({ children }) => { const Hit: SearchResultType = ({ children }) => {
return <article className="media p-1">{children}</article>; return <article className="media p-1">{children}</article>;
}; };
Hit.Title = ({ children }) => <h3 className="has-text-weight-bold is-ellipsis-overflow">{children}</h3>; Hit.Title = ({ className, children }) => (
<h3 className={classNames("has-text-weight-bold is-ellipsis-overflow", className)}>{children}</h3>
);
Hit.Left = ({ children }) => <div className="media-left">{children}</div>; Hit.Left = ({ className, children }) => <div className={classNames("media-left", className)}>{children}</div>;
Hit.Right = ({ children }) => <div className="media-right is-size-7 has-text-right">{children}</div>; Hit.Right = ({ className, children }) => (
<div className={classNames("media-right is-size-7 has-text-right", className)}>{children}</div>
);
Hit.Content = ({ children }) => <div className="media-content">{children}</div>; Hit.Content = ({ className, children }) => <div className={classNames("media-content", className)}> {children}</div>;
export default Hit; export default Hit;

View File

@@ -30,6 +30,7 @@ import { isHighlightedHitField } from "./fields";
type Props = { type Props = {
hit: Hit; hit: Hit;
field: string; field: string;
truncateValueAt?: number;
}; };
type HighlightedTextFieldProps = { type HighlightedTextFieldProps = {
@@ -48,14 +49,18 @@ const HighlightedTextField: FC<HighlightedTextFieldProps> = ({ field }) => (
</> </>
); );
const TextHitField: FC<Props> = ({ hit, field: fieldName }) => { const TextHitField: FC<Props> = ({ hit, field: fieldName, truncateValueAt = 0 }) => {
const field = hit.fields[fieldName]; const field = hit.fields[fieldName];
if (!field) { if (!field) {
return null; return null;
} else if (isHighlightedHitField(field)) { } else if (isHighlightedHitField(field)) {
return <HighlightedTextField field={field} />; return <HighlightedTextField field={field} />;
} else { } else {
return <>{field.value}</>; let value = field.value;
if (typeof value === "string" && truncateValueAt > 0 && value.length > truncateValueAt) {
value = value.substring(0, truncateValueAt) + "...";
}
return <>{value}</>;
} }
}; };

View File

@@ -31,7 +31,6 @@ import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryCountResult; import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult; import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine; import sonia.scm.search.SearchEngine;
@@ -112,17 +111,19 @@ public class SearchResource {
} }
private QueryResultDto search(SearchParameters params) { private QueryResultDto search(SearchParameters params) {
QueryResult result = engine.search(IndexNames.DEFAULT) QueryResult result = engine.forType(params.getType())
.search()
.start(params.getPage() * params.getPageSize()) .start(params.getPage() * params.getPageSize())
.limit(params.getPageSize()) .limit(params.getPageSize())
.execute(params.getType(), params.getQuery()); .execute(params.getQuery());
return mapper.map(params, result); return mapper.map(params, result);
} }
private QueryResultDto count(SearchParameters params) { private QueryResultDto count(SearchParameters params) {
QueryCountResult result = engine.search(IndexNames.DEFAULT) QueryCountResult result = engine.forType(params.getType())
.count(params.getType(), params.getQuery()); .search()
.count(params.getQuery());
return mapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList())); return mapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList()));
} }

View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer; import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
@@ -41,18 +40,16 @@ import javax.inject.Singleton;
@Singleton @Singleton
public class GroupIndexer implements Indexer<Group> { public class GroupIndexer implements Indexer<Group> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting @VisibleForTesting
static final int VERSION = 1; static final int VERSION = 1;
private final GroupManager groupManager; private final GroupManager groupManager;
private final IndexQueue indexQueue; private final SearchEngine searchEngine;
@Inject @Inject
public GroupIndexer(GroupManager groupManager, IndexQueue indexQueue) { public GroupIndexer(GroupManager groupManager, SearchEngine searchEngine) {
this.groupManager = groupManager; this.groupManager = groupManager;
this.indexQueue = indexQueue; this.searchEngine = searchEngine;
} }
@Override @Override
@@ -60,11 +57,6 @@ public class GroupIndexer implements Indexer<Group> {
return Group.class; return Group.class;
} }
@Override
public String getIndex() {
return INDEX;
}
@Override @Override
public int getVersion() { public int getVersion() {
return VERSION; return VERSION;
@@ -77,15 +69,15 @@ public class GroupIndexer implements Indexer<Group> {
@Override @Override
public Updater<Group> open() { public Updater<Group> open() {
return new GroupIndexUpdater(groupManager, indexQueue.getQueuedIndex(INDEX)); return new GroupIndexUpdater(groupManager, searchEngine.forType(Group.class).getOrCreate());
} }
public static class GroupIndexUpdater implements Updater<Group> { public static class GroupIndexUpdater implements Updater<Group> {
private final GroupManager groupManager; private final GroupManager groupManager;
private final Index index; private final Index<Group> index;
private GroupIndexUpdater(GroupManager groupManager, Index index) { private GroupIndexUpdater(GroupManager groupManager, Index<Group> index) {
this.groupManager = groupManager; this.groupManager = groupManager;
this.index = index; this.index = index;
} }
@@ -97,12 +89,12 @@ public class GroupIndexer implements Indexer<Group> {
@Override @Override
public void delete(Group group) { public void delete(Group group) {
index.delete(Id.of(group), Group.class); index.delete().byType().byId(Id.of(group));
} }
@Override @Override
public void reIndexAll() { public void reIndexAll() {
index.deleteByType(Group.class); index.delete().byType().all();
for (Group group : groupManager.getAll()) { for (Group group : groupManager.getAll()) {
store(group); store(group);
} }

View File

@@ -101,9 +101,7 @@ import sonia.scm.repository.xml.XmlRepositoryRoleDAO;
import sonia.scm.schedule.CronScheduler; import sonia.scm.schedule.CronScheduler;
import sonia.scm.schedule.Scheduler; import sonia.scm.schedule.Scheduler;
import sonia.scm.search.DefaultIndexLogStore; import sonia.scm.search.DefaultIndexLogStore;
import sonia.scm.search.DefaultIndexQueue;
import sonia.scm.search.IndexLogStore; import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.LuceneSearchEngine; import sonia.scm.search.LuceneSearchEngine;
import sonia.scm.search.SearchEngine; import sonia.scm.search.SearchEngine;
import sonia.scm.security.AccessTokenCookieIssuer; import sonia.scm.security.AccessTokenCookieIssuer;
@@ -289,7 +287,6 @@ class ScmServletModule extends ServletModule {
bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class); bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class);
// bind search stuff // bind search stuff
bind(IndexQueue.class, DefaultIndexQueue.class);
bind(SearchEngine.class, LuceneSearchEngine.class); bind(SearchEngine.class, LuceneSearchEngine.class);
bind(IndexLogStore.class, DefaultIndexLogStore.class); bind(IndexLogStore.class, DefaultIndexLogStore.class);

View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer; import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
@@ -42,18 +41,15 @@ import javax.inject.Singleton;
public class RepositoryIndexer implements Indexer<Repository> { public class RepositoryIndexer implements Indexer<Repository> {
@VisibleForTesting @VisibleForTesting
static final int VERSION = 2; static final int VERSION = 3;
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
private final RepositoryManager repositoryManager; private final RepositoryManager repositoryManager;
private final IndexQueue indexQueue; private final SearchEngine searchEngine;
@Inject @Inject
public RepositoryIndexer(RepositoryManager repositoryManager, IndexQueue indexQueue) { public RepositoryIndexer(RepositoryManager repositoryManager, SearchEngine searchEngine) {
this.repositoryManager = repositoryManager; this.repositoryManager = repositoryManager;
this.indexQueue = indexQueue; this.searchEngine = searchEngine;
} }
@Override @Override
@@ -66,11 +62,6 @@ public class RepositoryIndexer implements Indexer<Repository> {
return Repository.class; return Repository.class;
} }
@Override
public String getIndex() {
return INDEX;
}
@Subscribe(async = false) @Subscribe(async = false)
public void handleEvent(RepositoryEvent event) { public void handleEvent(RepositoryEvent event) {
new HandlerEventIndexSyncer<>(this).handleEvent(event); new HandlerEventIndexSyncer<>(this).handleEvent(event);
@@ -78,15 +69,15 @@ public class RepositoryIndexer implements Indexer<Repository> {
@Override @Override
public Updater<Repository> open() { public Updater<Repository> open() {
return new RepositoryIndexUpdater(repositoryManager, indexQueue.getQueuedIndex(INDEX)); return new RepositoryIndexUpdater(repositoryManager, searchEngine.forType(getType()).getOrCreate());
} }
public static class RepositoryIndexUpdater implements Updater<Repository> { public static class RepositoryIndexUpdater implements Updater<Repository> {
private final RepositoryManager repositoryManager; private final RepositoryManager repositoryManager;
private final Index index; private final Index<Repository> index;
public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index index) { public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index<Repository> index) {
this.repositoryManager = repositoryManager; this.repositoryManager = repositoryManager;
this.index = index; this.index = index;
} }
@@ -98,14 +89,14 @@ public class RepositoryIndexer implements Indexer<Repository> {
@Override @Override
public void delete(Repository repository) { public void delete(Repository repository) {
index.deleteByRepository(repository.getId()); index.delete().allTypes().byRepository(repository.getId());
} }
@Override @Override
public void reIndexAll() { public void reIndexAll() {
// v1 used the whole classname as type // v1 used the whole classname as type
index.deleteByTypeName(Repository.class.getName()); index.delete().allTypes().byTypeName(Repository.class.getName());
index.deleteByType(Repository.class); index.delete().byType().all();
for (Repository repository : repositoryManager.getAll()) { for (Repository repository : repositoryManager.getAll()) {
store(repository); store(repository);
} }

View File

@@ -28,9 +28,12 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer; import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer; import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.util.HashMap;
import java.util.Map;
public class AnalyzerFactory { public class AnalyzerFactory {
@@ -59,4 +62,21 @@ public class AnalyzerFactory {
} }
} }
public Analyzer create(LuceneSearchableType type, IndexOptions options) {
Analyzer defaultAnalyzer = create(options);
Map<String, Analyzer> analyzerMap = new HashMap<>();
for (SearchableField field : type.getFields()) {
addFieldAnalyzer(analyzerMap, field);
}
return new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap);
}
private void addFieldAnalyzer(Map<String, Analyzer> analyzerMap, SearchableField field) {
if (field.getAnalyzer() != Indexed.Analyzer.DEFAULT) {
analyzerMap.put(field.getName(), new NonNaturalLanguageAnalyzer());
}
}
} }

View File

@@ -42,18 +42,36 @@ public class DefaultIndexLogStore implements IndexLogStore {
} }
@Override @Override
public void log(String index,Class<?> type, int version) { public ForIndex forIndex(String index) {
String id = id(index, type); return new DefaultForIndex(index);
dataStore.put(id, new IndexLog(version));
}
private String id(String index, Class<?> type) {
return index + "_" + type.getName();
} }
@Override @Override
public Optional<IndexLog> get(String index, Class<?> type) { public ForIndex defaultIndex() {
String id = id(index, type); // constant
return dataStore.getOptional(id); return new DefaultForIndex("default");
}
class DefaultForIndex implements ForIndex {
private final String index;
private DefaultForIndex(String index) {
this.index = index;
}
private String id(Class<?> type) {
return index + "_" + type.getName();
}
@Override
public void log(Class<?> type, int version) {
dataStore.put(id(type), new IndexLog(version));
}
@Override
public Optional<IndexLog> get(Class<?> type) {
return dataStore.getOptional(id(type));
}
} }
} }

View File

@@ -62,7 +62,7 @@ public class IndexBootstrapListener implements ServletContextListener {
} }
private void bootstrap(Indexer indexer) { private void bootstrap(Indexer indexer) {
Optional<IndexLog> indexLog = indexLogStore.get(indexer.getIndex(), indexer.getType()); Optional<IndexLog> indexLog = indexLogStore.defaultIndex().get(indexer.getType());
if (indexLog.isPresent()) { if (indexLog.isPresent()) {
int version = indexLog.get().getVersion(); int version = indexLog.get().getVersion();
if (version < indexer.getVersion()) { if (version < indexer.getVersion()) {
@@ -82,7 +82,7 @@ public class IndexBootstrapListener implements ServletContextListener {
} }
}); });
indexLogStore.log(indexer.getIndex(), indexer.getType(), indexer.getVersion()); indexLogStore.defaultIndex().log(indexer.getType(), indexer.getVersion());
} }
@Override @Override

View File

@@ -52,10 +52,10 @@ public class IndexOpener {
return DirectoryReader.open(directory(name)); return DirectoryReader.open(directory(name));
} }
public IndexWriter openForWrite(String name, IndexOptions options) throws IOException { public IndexWriter openForWrite(IndexParams indexParams) throws IOException {
IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options)); IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions()));
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(name), config); return new IndexWriter(directory(indexParams.getIndex()), config);
} }
private Directory directory(String name) throws IOException { private Directory directory(String name) throws IOException {

View File

@@ -24,20 +24,13 @@
package sonia.scm.search; package sonia.scm.search;
import com.google.common.annotations.Beta; import lombok.Value;
/** @Value
* Names of predefined indexes. public class IndexParams {
* @since 2.21.0
*/
@Beta
public final class IndexNames {
/** String index;
* The default index. LuceneSearchableType searchableType;
*/ IndexOptions options;
public static final String DEFAULT = "_default";
private IndexNames() {
}
} }

View File

@@ -35,29 +35,28 @@ import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
@Singleton @Singleton
public class DefaultIndexQueue implements IndexQueue, Closeable { public class IndexQueue implements Closeable {
private final ExecutorService executor = Executors.newSingleThreadExecutor(); private final ExecutorService executor = Executors.newSingleThreadExecutor();
private final AtomicLong size = new AtomicLong(0); private final AtomicLong size = new AtomicLong(0);
private final SearchEngine searchEngine; private final LuceneIndexFactory indexFactory;
@Inject @Inject
public DefaultIndexQueue(SearchEngine searchEngine) { public IndexQueue(LuceneIndexFactory indexFactory) {
this.searchEngine = searchEngine; this.indexFactory = indexFactory;
} }
@Override public <T> Index<T> getQueuedIndex(IndexParams indexParams) {
public Index getQueuedIndex(String name, IndexOptions indexOptions) { return new QueuedIndex<>(this, indexParams);
return new QueuedIndex(this, name, indexOptions);
} }
public SearchEngine getSearchEngine() { public LuceneIndexFactory getIndexFactory() {
return searchEngine; return indexFactory;
} }
void enqueue(IndexQueueTaskWrapper task) { <T> void enqueue(IndexQueueTaskWrapper<T> task) {
size.incrementAndGet(); size.incrementAndGet();
executor.execute(() -> { executor.execute(() -> {
task.run(); task.run();

View File

@@ -25,8 +25,8 @@
package sonia.scm.search; package sonia.scm.search;
@FunctionalInterface @FunctionalInterface
public interface IndexQueueTask { public interface IndexQueueTask<T> {
void updateIndex(Index index); void updateIndex(Index<T> index);
} }

View File

@@ -27,30 +27,28 @@ package sonia.scm.search;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public final class IndexQueueTaskWrapper implements Runnable { public final class IndexQueueTaskWrapper<T> implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class); private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class);
private final SearchEngine searchEngine; private final LuceneIndexFactory indexFactory;
private final String indexName; private final IndexParams indexParams;
private final IndexOptions options; private final Iterable<IndexQueueTask<T>> tasks;
private final Iterable<IndexQueueTask> tasks;
IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable<IndexQueueTask> tasks) { IndexQueueTaskWrapper(LuceneIndexFactory indexFactory, IndexParams indexParams, Iterable<IndexQueueTask<T>> tasks) {
this.searchEngine = searchEngine; this.indexFactory = indexFactory;
this.indexName = indexName; this.indexParams = indexParams;
this.options = options;
this.tasks = tasks; this.tasks = tasks;
} }
@Override @Override
public void run() { public void run() {
try (Index index = searchEngine.getOrCreate(this.indexName, options)) { try (Index<T> index = indexFactory.create(indexParams)) {
for (IndexQueueTask task : tasks) { for (IndexQueueTask<T> task : tasks) {
task.updateIndex(index); task.updateIndex(index);
} }
} catch (Exception e) { } catch (Exception e) {
LOG.warn("failure during execution of index task for index {}", indexName, e); LOG.warn("failure during execution of index task for index {}", indexParams.getIndex(), e);
} }
} }
} }

View File

@@ -30,6 +30,9 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField; import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import java.io.IOException; import java.io.IOException;
@@ -39,26 +42,25 @@ import static sonia.scm.search.FieldNames.REPOSITORY;
import static sonia.scm.search.FieldNames.TYPE; import static sonia.scm.search.FieldNames.TYPE;
import static sonia.scm.search.FieldNames.UID; import static sonia.scm.search.FieldNames.UID;
public class LuceneIndex implements Index { class LuceneIndex<T> implements Index<T> {
private final SearchableTypeResolver resolver; private final LuceneSearchableType searchableType;
private final IndexWriter writer; private final IndexWriter writer;
LuceneIndex(SearchableTypeResolver resolver, IndexWriter writer) { LuceneIndex(LuceneSearchableType searchableType, IndexWriter writer) {
this.resolver = resolver; this.searchableType = searchableType;
this.writer = writer; this.writer = writer;
} }
@Override @Override
public void store(Id id, String permission, Object object) { public void store(Id id, String permission, Object object) {
LuceneSearchableType type = resolver.resolve(object); String uid = createUid(id, searchableType);
String uid = createUid(id, type); Document document = searchableType.getTypeConverter().convert(object);
Document document = type.getTypeConverter().convert(object);
try { try {
field(document, UID, uid); field(document, UID, uid);
field(document, ID, id.getValue()); field(document, ID, id.getValue());
id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository)); id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository));
field(document, TYPE, type.getName()); field(document, TYPE, searchableType.getName());
if (!Strings.isNullOrEmpty(permission)) { if (!Strings.isNullOrEmpty(permission)) {
field(document, PERMISSION, permission); field(document, PERMISSION, permission);
} }
@@ -77,37 +79,8 @@ public class LuceneIndex implements Index {
} }
@Override @Override
public void delete(Id id, Class<?> type) { public Deleter delete() {
LuceneSearchableType searchableType = resolver.resolve(type); return new LuceneDeleter();
try {
writer.deleteDocuments(new Term(UID, createUid(id, searchableType)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void deleteByRepository(String repository) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repository));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex);
}
}
@Override
public void deleteByType(Class<?> type) {
LuceneSearchableType searchableType = resolver.resolve(type);
deleteByTypeName(searchableType.getName());
}
@Override
public void deleteByTypeName(String typeName) {
try {
writer.deleteDocuments(new Term(TYPE, typeName));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + typeName + " from index", ex);
}
} }
@Override @Override
@@ -118,4 +91,73 @@ public class LuceneIndex implements Index {
throw new SearchEngineException("failed to close index writer", e); throw new SearchEngineException("failed to close index writer", e);
} }
} }
private class LuceneDeleter implements Deleter {
@Override
public ByTypeDeleter byType() {
return new LuceneByTypeDeleter();
}
@Override
public AllTypesDeleter allTypes() {
return new LuceneAllTypesDelete();
}
}
@SuppressWarnings("java:S1192")
private class LuceneByTypeDeleter implements ByTypeDeleter {
@Override
public void byId(Id id) {
try {
writer.deleteDocuments(new Term(UID, createUid(id, searchableType)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void all() {
try {
writer.deleteDocuments(new Term(TYPE, searchableType.getName()));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by type " + searchableType.getName() + " from index", ex);
}
}
@Override
public void byRepository(String repositoryId) {
try {
BooleanQuery query = new BooleanQuery.Builder()
.add(new TermQuery(new Term(TYPE, searchableType.getName())), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(REPOSITORY, repositoryId)), BooleanClause.Occur.MUST)
.build();
writer.deleteDocuments(query);
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repositoryId + " from index", ex);
}
}
}
private class LuceneAllTypesDelete implements AllTypesDeleter {
@Override
public void byRepository(String repositoryId) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repositoryId));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete all documents by repository " + repositoryId + " from index", ex);
}
}
@Override
public void byTypeName(String typeName) {
try {
writer.deleteDocuments(new Term(TYPE, typeName));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by type " + typeName + " from index", ex);
}
}
}
} }

View File

@@ -29,20 +29,18 @@ import java.io.IOException;
public class LuceneIndexFactory { public class LuceneIndexFactory {
private final SearchableTypeResolver typeResolver;
private final IndexOpener indexOpener; private final IndexOpener indexOpener;
@Inject @Inject
public LuceneIndexFactory(SearchableTypeResolver typeResolver, IndexOpener indexOpener) { public LuceneIndexFactory(IndexOpener indexOpener) {
this.typeResolver = typeResolver;
this.indexOpener = indexOpener; this.indexOpener = indexOpener;
} }
public LuceneIndex create(String name, IndexOptions options) { public <T> LuceneIndex<T> create(IndexParams indexParams) {
try { try {
return new LuceneIndex(typeResolver, indexOpener.openForWrite(name, options)); return new LuceneIndex<>(indexParams.getSearchableType(), indexOpener.openForWrite(indexParams));
} catch (IOException ex) { } catch (IOException ex) {
throw new SearchEngineException("failed to open index " + name, ex); throw new SearchEngineException("failed to open index " + indexParams.getIndex(), ex);
} }
} }
} }

View File

@@ -43,62 +43,50 @@ import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.shiro.SecurityUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.Locale; import java.util.Locale;
import java.util.Optional;
public class LuceneQueryBuilder extends QueryBuilder { public class LuceneQueryBuilder<T> extends QueryBuilder<T> {
private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class); private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class);
private final IndexOpener opener; private final IndexOpener opener;
private final SearchableTypeResolver resolver; private final LuceneSearchableType searchableType;
private final String indexName; private final String indexName;
private final Analyzer analyzer; private final Analyzer analyzer;
LuceneQueryBuilder(IndexOpener opener, SearchableTypeResolver resolver, String indexName, Analyzer analyzer) { LuceneQueryBuilder(IndexOpener opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) {
this.opener = opener; this.opener = opener;
this.resolver = resolver;
this.indexName = indexName; this.indexName = indexName;
this.searchableType = searchableType;
this.analyzer = analyzer; this.analyzer = analyzer;
} }
@Override
protected Optional<Class<?>> resolveByName(String typeName) {
return resolver.resolveClassByName(typeName);
}
@Override @Override
protected QueryCountResult count(QueryParams queryParams) { protected QueryCountResult count(QueryParams queryParams) {
TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
return search( return search(
queryParams, totalHitCountCollector, queryParams, totalHitCountCollector,
(searcher, type, query) -> new QueryCountResult(type.getType(), totalHitCountCollector.getTotalHits()) (searcher, query) -> new QueryCountResult(searchableType.getType(), totalHitCountCollector.getTotalHits())
); );
} }
@Override @Override
protected QueryResult execute(QueryParams queryParams) { protected QueryResult execute(QueryParams queryParams) {
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams); TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
return search(queryParams, topScoreCollector, (searcher, searchableType, query) -> { return search(queryParams, topScoreCollector, (searcher, query) -> {
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query); QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector)); return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
}); });
} }
private <T> T search(QueryParams queryParams, Collector collector, ResultBuilder<T> resultBuilder) { private <R> R search(QueryParams queryParams, Collector collector, ResultBuilder<R> resultBuilder) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString()); String queryString = Strings.nullToEmpty(queryParams.getQueryString());
LuceneSearchableType searchableType = resolver.resolve(queryParams.getType());
searchableType.getPermission().ifPresent(
permission -> SecurityUtils.getSubject().checkPermission(permission)
);
Query parsedQuery = createQuery(searchableType, queryParams, queryString); Query parsedQuery = createQuery(searchableType, queryParams, queryString);
Query query = Queries.filter(parsedQuery, searchableType, queryParams); Query query = Queries.filter(parsedQuery, searchableType, queryParams);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@@ -109,7 +97,7 @@ public class LuceneQueryBuilder extends QueryBuilder {
searcher.search(query, new PermissionAwareCollector(reader, collector)); searcher.search(query, new PermissionAwareCollector(reader, collector));
return resultBuilder.create(searcher, searchableType, parsedQuery); return resultBuilder.create(searcher, parsedQuery);
} catch (IOException e) { } catch (IOException e) {
throw new SearchEngineException("failed to search index", e); throw new SearchEngineException("failed to search index", e);
} catch (InvalidTokenOffsetsException e) { } catch (InvalidTokenOffsetsException e) {
@@ -196,6 +184,6 @@ public class LuceneQueryBuilder extends QueryBuilder {
@FunctionalInterface @FunctionalInterface
private interface ResultBuilder<T> { private interface ResultBuilder<T> {
T create(IndexSearcher searcher, LuceneSearchableType searchableType, Query query) throws IOException, InvalidTokenOffsetsException; T create(IndexSearcher searcher, Query query) throws IOException, InvalidTokenOffsetsException;
} }
} }

View File

@@ -29,18 +29,21 @@ import javax.inject.Inject;
public class LuceneQueryBuilderFactory { public class LuceneQueryBuilderFactory {
private final IndexOpener indexOpener; private final IndexOpener indexOpener;
private final SearchableTypeResolver searchableTypeResolver;
private final AnalyzerFactory analyzerFactory; private final AnalyzerFactory analyzerFactory;
@Inject @Inject
public LuceneQueryBuilderFactory(IndexOpener indexOpener, SearchableTypeResolver searchableTypeResolver, AnalyzerFactory analyzerFactory) { public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) {
this.indexOpener = indexOpener; this.indexOpener = indexOpener;
this.searchableTypeResolver = searchableTypeResolver;
this.analyzerFactory = analyzerFactory; this.analyzerFactory = analyzerFactory;
} }
public LuceneQueryBuilder create(String name, IndexOptions options) { public <T> LuceneQueryBuilder<T> create(IndexParams indexParams) {
return new LuceneQueryBuilder(indexOpener, searchableTypeResolver, name, analyzerFactory.create(options)); return new LuceneQueryBuilder<>(
indexOpener,
indexParams.getIndex(),
indexParams.getSearchableType(),
analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())
);
} }
} }

View File

@@ -34,13 +34,13 @@ import java.util.stream.Collectors;
public class LuceneSearchEngine implements SearchEngine { public class LuceneSearchEngine implements SearchEngine {
private final SearchableTypeResolver resolver; private final SearchableTypeResolver resolver;
private final LuceneIndexFactory indexFactory; private final IndexQueue indexQueue;
private final LuceneQueryBuilderFactory queryBuilderFactory; private final LuceneQueryBuilderFactory queryBuilderFactory;
@Inject @Inject
public LuceneSearchEngine(SearchableTypeResolver resolver, LuceneIndexFactory indexFactory, LuceneQueryBuilderFactory queryBuilderFactory) { public LuceneSearchEngine(SearchableTypeResolver resolver, IndexQueue indexQueue, LuceneQueryBuilderFactory queryBuilderFactory) {
this.resolver = resolver; this.resolver = resolver;
this.indexFactory = indexFactory; this.indexQueue = indexQueue;
this.queryBuilderFactory = queryBuilderFactory; this.queryBuilderFactory = queryBuilderFactory;
} }
@@ -54,13 +54,57 @@ public class LuceneSearchEngine implements SearchEngine {
} }
@Override @Override
public Index getOrCreate(String name, IndexOptions options) { public <T> ForType<T> forType(Class<T> type) {
return indexFactory.create(name, options); return forType(resolver.resolve(type));
} }
@Override @Override
public QueryBuilder search(String name, IndexOptions options) { public ForType<Object> forType(String typeName) {
return queryBuilderFactory.create(name, options); return forType(resolver.resolveByName(typeName));
}
private <T> ForType<T> forType(LuceneSearchableType searchableType) {
return new LuceneForType<>(searchableType);
}
class LuceneForType<T> implements ForType<T> {
private final LuceneSearchableType searchableType;
private IndexOptions options = IndexOptions.defaults();
private String index = "default";
private LuceneForType(LuceneSearchableType searchableType) {
this.searchableType = searchableType;
}
@Override
public ForType<T> withOptions(IndexOptions options) {
this.options = options;
return this;
}
@Override
public ForType<T> withIndex(String index) {
this.index = index;
return this;
}
private IndexParams params() {
return new IndexParams(index, searchableType, options);
}
@Override
public Index<T> getOrCreate() {
return indexQueue.getQueuedIndex(params());
}
@Override
public QueryBuilder<T> search() {
searchableType.getPermission().ifPresent(
permission -> SecurityUtils.getSubject().checkPermission(permission)
);
return queryBuilderFactory.create(params());
}
} }
} }

View File

@@ -0,0 +1,52 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_WORD_PARTS;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.PRESERVE_ORIGINAL;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_NUMERICS;
public class NonNaturalLanguageAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
StandardTokenizer tokenizer = new StandardTokenizer();
TokenStream stream = new WordDelimiterGraphFilter(
tokenizer,
GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | PRESERVE_ORIGINAL,
null // list of words which are protected from being delimited
);
stream = new LowerCaseFilter(stream);
return new TokenStreamComponents(tokenizer, stream);
}
}

View File

@@ -27,50 +27,76 @@ package sonia.scm.search;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class QueuedIndex implements Index { class QueuedIndex<T> implements Index<T> {
private final DefaultIndexQueue queue; private final IndexQueue queue;
private final String indexName; private final IndexParams indexParams;
private final IndexOptions indexOptions; private final List<IndexQueueTask<T>> tasks = new ArrayList<>();
private final List<IndexQueueTask> tasks = new ArrayList<>(); QueuedIndex(IndexQueue queue, IndexParams indexParams) {
QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) {
this.queue = queue; this.queue = queue;
this.indexName = indexName; this.indexParams = indexParams;
this.indexOptions = indexOptions; }
}
@Override @Override
public void store(Id id, String permission, Object object) { public void store(Id id, String permission, T object) {
tasks.add(index -> index.store(id, permission, object)); tasks.add(index -> index.store(id, permission, object));
} }
@Override @Override
public void delete(Id id, Class<?> type) { public Deleter delete() {
tasks.add(index -> index.delete(id, type)); return new QueueDeleter();
}
@Override
public void deleteByRepository(String repository) {
tasks.add(index -> index.deleteByRepository(repository));
}
@Override
public void deleteByType(Class<?> type) {
tasks.add(index -> index.deleteByType(type));
}
@Override
public void deleteByTypeName(String typeName) {
tasks.add(index -> index.deleteByTypeName(typeName));
} }
@Override @Override
public void close() { public void close() {
IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper( IndexQueueTaskWrapper<T> wrappedTask = new IndexQueueTaskWrapper<>(
queue.getSearchEngine(), indexName, indexOptions, tasks queue.getIndexFactory(), indexParams, tasks
); );
queue.enqueue(wrappedTask); queue.enqueue(wrappedTask);
} }
private class QueueDeleter implements Deleter {
@Override
public ByTypeDeleter byType() {
return new QueueByTypeDeleter();
}
@Override
public AllTypesDeleter allTypes() {
return new QueueAllTypesDeleter();
}
}
private class QueueByTypeDeleter implements ByTypeDeleter {
@Override
public void byId(Id id) {
tasks.add(index -> index.delete().byType().byId(id));
}
@Override
public void all() {
tasks.add(index -> index.delete().byType().all());
}
@Override
public void byRepository(String repositoryId) {
tasks.add(index -> index.delete().byType().byRepository(repositoryId));
}
}
private class QueueAllTypesDeleter implements AllTypesDeleter {
@Override
public void byRepository(String repositoryId) {
tasks.add(index -> index.delete().allTypes().byRepository(repositoryId));
}
@Override
public void byTypeName(String typeName) {
tasks.add(index -> index.delete().allTypes().byTypeName(typeName));
}
}
} }

View File

@@ -41,6 +41,7 @@ class SearchableField {
private final boolean defaultQuery; private final boolean defaultQuery;
private final boolean highlighted; private final boolean highlighted;
private final PointsConfig pointsConfig; private final PointsConfig pointsConfig;
private final Indexed.Analyzer analyzer;
SearchableField(Field field, Indexed indexed) { SearchableField(Field field, Indexed indexed) {
this.name = name(field, indexed); this.name = name(field, indexed);
@@ -50,6 +51,7 @@ class SearchableField {
this.defaultQuery = indexed.defaultQuery(); this.defaultQuery = indexed.defaultQuery();
this.highlighted = indexed.highlighted(); this.highlighted = indexed.highlighted();
this.pointsConfig = IndexableFields.pointConfig(field); this.pointsConfig = IndexableFields.pointConfig(field);
this.analyzer = indexed.analyzer();
} }
Object value(Document document) { Object value(Document document) {

View File

@@ -35,7 +35,6 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
@@ -94,8 +93,12 @@ class SearchableTypeResolver {
return searchableType; return searchableType;
} }
public Optional<Class<?>> resolveClassByName(String typeName) { public LuceneSearchableType resolveByName(String typeName) {
return Optional.ofNullable(nameToClass.get(typeName)); Class<?> type = nameToClass.get(typeName);
if (type == null) {
throw notFound(entity("type", typeName));
}
return resolve(type);
} }
} }

View File

@@ -30,9 +30,8 @@ import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer; import sonia.scm.search.Indexer;
import sonia.scm.search.SearchEngine;
import javax.inject.Inject; import javax.inject.Inject;
import javax.inject.Singleton; import javax.inject.Singleton;
@@ -41,18 +40,16 @@ import javax.inject.Singleton;
@Singleton @Singleton
public class UserIndexer implements Indexer<User> { public class UserIndexer implements Indexer<User> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting @VisibleForTesting
static final int VERSION = 1; static final int VERSION = 1;
private final UserManager userManager; private final UserManager userManager;
private final IndexQueue queue; private final SearchEngine searchEngine;
@Inject @Inject
public UserIndexer(UserManager userManager, IndexQueue queue) { public UserIndexer(UserManager userManager, SearchEngine searchEngine) {
this.userManager = userManager; this.userManager = userManager;
this.queue = queue; this.searchEngine = searchEngine;
} }
@Override @Override
@@ -60,11 +57,6 @@ public class UserIndexer implements Indexer<User> {
return User.class; return User.class;
} }
@Override
public String getIndex() {
return INDEX;
}
@Override @Override
public int getVersion() { public int getVersion() {
return VERSION; return VERSION;
@@ -77,15 +69,15 @@ public class UserIndexer implements Indexer<User> {
@Override @Override
public Updater<User> open() { public Updater<User> open() {
return new UserIndexUpdater(userManager, queue.getQueuedIndex(INDEX)); return new UserIndexUpdater(userManager, searchEngine.forType(User.class).getOrCreate());
} }
public static class UserIndexUpdater implements Updater<User> { public static class UserIndexUpdater implements Updater<User> {
private final UserManager userManager; private final UserManager userManager;
private final Index index; private final Index<User> index;
private UserIndexUpdater(UserManager userManager, Index index) { private UserIndexUpdater(UserManager userManager, Index<User> index) {
this.userManager = userManager; this.userManager = userManager;
this.index = index; this.index = index;
} }
@@ -97,12 +89,12 @@ public class UserIndexer implements Indexer<User> {
@Override @Override
public void delete(User user) { public void delete(User user) {
index.delete(Id.of(user), User.class); index.delete().byType().byId(Id.of(user));
} }
@Override @Override
public void reIndexAll() { public void reIndexAll() {
index.deleteByType(User.class); index.delete().byType().all();
for (User user : userManager.getAll()) { for (User user : userManager.getAll()) {
store(user); store(user);
} }

View File

@@ -41,7 +41,6 @@ import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager; import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryTestData; import sonia.scm.repository.RepositoryTestData;
import sonia.scm.search.Hit; import sonia.scm.search.Hit;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryCountResult; import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult; import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine; import sonia.scm.search.SearchEngine;
@@ -200,8 +199,9 @@ class SearchResourceTest {
@Test @Test
void shouldReturnCountOnly() throws URISyntaxException { void shouldReturnCountOnly() throws URISyntaxException {
when( when(
searchEngine.search(IndexNames.DEFAULT) searchEngine.forType("string")
.count("string", "Hello") .search()
.count("Hello")
).thenReturn(new QueryCountResult(String.class, 2L)); ).thenReturn(new QueryCountResult(String.class, 2L));
MockHttpRequest request = MockHttpRequest.get("/v2/search/query/string?q=Hello&countOnly=true"); MockHttpRequest request = MockHttpRequest.get("/v2/search/query/string?q=Hello&countOnly=true");
@@ -271,10 +271,11 @@ class SearchResourceTest {
private void mockQueryResult(int start, int limit, String query, QueryResult result) { private void mockQueryResult(int start, int limit, String query, QueryResult result) {
when( when(
searchEngine.search(IndexNames.DEFAULT) searchEngine.forType("string")
.search()
.start(start) .start(start)
.limit(limit) .limit(limit)
.execute("string", query) .execute(query)
).thenReturn(result); ).thenReturn(result);
} }

View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType; import sonia.scm.HandlerEventType;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue; import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -47,22 +48,17 @@ class GroupIndexerTest {
@Mock @Mock
private GroupManager groupManager; private GroupManager groupManager;
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private IndexQueue indexQueue; private SearchEngine searchEngine;
@InjectMocks @InjectMocks
private GroupIndexer indexer; private GroupIndexer indexer;
@Test @Test
void shouldReturnRepositoryClass() { void shouldReturnClass() {
assertThat(indexer.getType()).isEqualTo(Group.class); assertThat(indexer.getType()).isEqualTo(Group.class);
} }
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(GroupIndexer.INDEX);
}
@Test @Test
void shouldReturnVersion() { void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(GroupIndexer.VERSION); assertThat(indexer.getVersion()).isEqualTo(GroupIndexer.VERSION);
@@ -71,28 +67,28 @@ class GroupIndexerTest {
@Nested @Nested
class UpdaterTests { class UpdaterTests {
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index index; private Index<Group> index;
private final Group group = new Group("xml", "astronauts"); private final Group group = new Group("xml", "astronauts");
@BeforeEach @BeforeEach
void open() { void open() {
when(indexQueue.getQueuedIndex(GroupIndexer.INDEX)).thenReturn(index); when(searchEngine.forType(Group.class).getOrCreate()).thenReturn(index);
} }
@Test @Test
void shouldStoreRepository() { void shouldStore() {
indexer.open().store(group); indexer.open().store(group);
verify(index).store(Id.of(group), "group:read:astronauts", group); verify(index).store(Id.of(group), "group:read:astronauts", group);
} }
@Test @Test
void shouldDeleteByRepository() { void shouldDeleteById() {
indexer.open().delete(group); indexer.open().delete(group);
verify(index).delete(Id.of(group), Group.class); verify(index.delete().byType()).byId(Id.of(group));
} }
@Test @Test
@@ -101,7 +97,7 @@ class GroupIndexerTest {
indexer.open().reIndexAll(); indexer.open().reIndexAll();
verify(index).deleteByType(Group.class); verify(index.delete().byType()).all();
verify(index).store(Id.of(group), "group:read:astronauts", group); verify(index).store(Id.of(group), "group:read:astronauts", group);
} }
@@ -111,7 +107,7 @@ class GroupIndexerTest {
indexer.handleEvent(event); indexer.handleEvent(event);
verify(index).delete(Id.of(group), Group.class); verify(index.delete().byType()).byId(Id.of(group));
} }
@Test @Test

View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType; import sonia.scm.HandlerEventType;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue; import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -47,8 +48,8 @@ class RepositoryIndexerTest {
@Mock @Mock
private RepositoryManager repositoryManager; private RepositoryManager repositoryManager;
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private IndexQueue indexQueue; private SearchEngine searchEngine;
@InjectMocks @InjectMocks
private RepositoryIndexer indexer; private RepositoryIndexer indexer;
@@ -58,11 +59,6 @@ class RepositoryIndexerTest {
assertThat(indexer.getType()).isEqualTo(Repository.class); assertThat(indexer.getType()).isEqualTo(Repository.class);
} }
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(RepositoryIndexer.INDEX);
}
@Test @Test
void shouldReturnVersion() { void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(RepositoryIndexer.VERSION); assertThat(indexer.getVersion()).isEqualTo(RepositoryIndexer.VERSION);
@@ -71,14 +67,14 @@ class RepositoryIndexerTest {
@Nested @Nested
class UpdaterTests { class UpdaterTests {
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index index; private Index<Repository> index;
private Repository repository; private Repository repository;
@BeforeEach @BeforeEach
void open() { void open() {
when(indexQueue.getQueuedIndex(RepositoryIndexer.INDEX)).thenReturn(index); when(searchEngine.forType(Repository.class).getOrCreate()).thenReturn(index);
repository = new Repository(); repository = new Repository();
repository.setId("42"); repository.setId("42");
} }
@@ -94,7 +90,7 @@ class RepositoryIndexerTest {
void shouldDeleteByRepository() { void shouldDeleteByRepository() {
indexer.open().delete(repository); indexer.open().delete(repository);
verify(index).deleteByRepository("42"); verify(index.delete().allTypes()).byRepository("42");
} }
@Test @Test
@@ -103,8 +99,8 @@ class RepositoryIndexerTest {
indexer.open().reIndexAll(); indexer.open().reIndexAll();
verify(index).deleteByTypeName(Repository.class.getName()); verify(index.delete().allTypes()).byTypeName(Repository.class.getName());
verify(index).deleteByType(Repository.class); verify(index.delete().byType()).all();
verify(index).store(Id.of(repository), "repository:read:42", repository); verify(index).store(Id.of(repository), "repository:read:42", repository);
} }
@@ -115,7 +111,7 @@ class RepositoryIndexerTest {
indexer.handleEvent(event); indexer.handleEvent(event);
verify(index).deleteByRepository("42"); verify(index.delete().allTypes()).byRepository("42");
} }
@Test @Test

View File

@@ -24,13 +24,17 @@
package sonia.scm.search; package sonia.scm.search;
import lombok.Getter;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer; import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer; import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -39,40 +43,79 @@ class AnalyzerFactoryTest {
private final AnalyzerFactory analyzerFactory = new AnalyzerFactory(); private final AnalyzerFactory analyzerFactory = new AnalyzerFactory();
@Test @Nested
void shouldReturnStandardAnalyzer() { class FromIndexOptionsTests {
Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults());
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); @Test
void shouldReturnStandardAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults());
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnStandardAnalyzerForUnknownLocale() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE));
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnEnglishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH));
assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzerForLocaleGermany() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnSpanishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES")));
assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class);
}
} }
@Test @Nested
void shouldReturnStandardAnalyzerForUnknownLocale() { class FromSearchableTypeTests {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE));
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); @Test
void shouldUseDefaultAnalyzerIfNotSpecified() throws IOException {
analyze(Account.class, "simple_text", "description", "simple_text");
}
@Test
void shouldUseNonNaturalLanguageAnalyzer() throws IOException {
analyze(Account.class, "simple_text", "username", "simple", "text", "simple_text");
}
private void analyze(Class<?> type, String text, String field, String... expectedTokens) throws IOException {
LuceneSearchableType searchableType = SearchableTypes.create(type);
IndexOptions defaults = IndexOptions.defaults();
Analyzer analyzer = analyzerFactory.create(searchableType, defaults);
List<String> tokens = Analyzers.tokenize(analyzer, field, text);
assertThat(tokens).containsOnly(expectedTokens);
}
} }
@Test @Getter
void shouldReturnEnglishAnalyzer() { @IndexedType
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH)); public static class Account {
assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class);
}
@Test @Indexed(analyzer = Indexed.Analyzer.IDENTIFIER)
void shouldReturnGermanAnalyzer() { private String username;
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test @Indexed
void shouldReturnGermanAnalyzerForLocaleGermany() { private String description;
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnSpanishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES")));
assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class);
} }
} }

View File

@@ -24,37 +24,33 @@
package sonia.scm.search; package sonia.scm.search;
import com.google.common.annotations.Beta; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/** import java.io.IOException;
* Queue the work of indexing. import java.util.ArrayList;
* An index can't be opened in parallel, so the queue coordinates the work of indexing in an asynchronous manner. import java.util.List;
* {@link IndexQueue} should be used most of the time to index content.
*
* @since 2.21.0
*/
@Beta
public interface IndexQueue {
/** class Analyzers {
* Returns an index which queues every change to the content.
*
* @param name name of index
* @param indexOptions options for the index
*
* @return index which queues changes
*/
Index getQueuedIndex(String name, IndexOptions indexOptions);
/** private Analyzers() {
* Returns an index which with default options which queues every change to the content. }
* @param name name of index
* static List<String> tokenize(Analyzer analyzer, String text) throws IOException {
* @return index with default options which queues changes return tokenize(analyzer, "default", text);
* @see IndexOptions#defaults() }
*/
default Index getQueuedIndex(String name) { static List<String> tokenize(Analyzer analyzer, String field, String text) throws IOException {
return getQueuedIndex(name, IndexOptions.defaults()); List<String> tokens = new ArrayList<>();
try (TokenStream stream = analyzer.tokenStream(field, text)) {
CharTermAttribute attr = stream.addAttribute(CharTermAttribute.class);
stream.reset();
while (stream.incrementToken()) {
tokens.add(attr.toString());
}
}
return tokens;
} }
} }

View File

@@ -44,14 +44,14 @@ class DefaultIndexLogStoreTest {
@Test @Test
void shouldReturnEmptyOptional() { void shouldReturnEmptyOptional() {
Optional<IndexLog> indexLog = indexLogStore.get("index", String.class); Optional<IndexLog> indexLog = indexLogStore.forIndex("index").get(String.class);
assertThat(indexLog).isEmpty(); assertThat(indexLog).isEmpty();
} }
@Test @Test
void shouldStoreLog() { void shouldStoreLog() {
indexLogStore.log("index", String.class, 42); indexLogStore.forIndex("index").log(String.class, 42);
Optional<IndexLog> index = indexLogStore.get("index", String.class); Optional<IndexLog> index = indexLogStore.forIndex("index").get(String.class);
assertThat(index).hasValueSatisfying(log -> { assertThat(index).hasValueSatisfying(log -> {
assertThat(log.getVersion()).isEqualTo(42); assertThat(log.getVersion()).isEqualTo(42);
assertThat(log.getDate()).isNotNull(); assertThat(log.getDate()).isNotNull();

View File

@@ -26,6 +26,7 @@ package sonia.scm.search;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.group.Group; import sonia.scm.group.Group;
@@ -53,7 +54,7 @@ class IndexBootstrapListenerTest {
@Mock @Mock
private AdministrationContext administrationContext; private AdministrationContext administrationContext;
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private IndexLogStore indexLogStore; private IndexLogStore indexLogStore;
@Test @Test
@@ -67,7 +68,7 @@ class IndexBootstrapListenerTest {
verify(updater).reIndexAll(); verify(updater).reIndexAll();
verify(updater).close(); verify(updater).close();
verify(indexLogStore).log(IndexNames.DEFAULT, Repository.class, 1); verify(indexLogStore.defaultIndex()).log(Repository.class, 1);
} }
@Test @Test
@@ -81,7 +82,7 @@ class IndexBootstrapListenerTest {
verify(updater).reIndexAll(); verify(updater).reIndexAll();
verify(updater).close(); verify(updater).close();
verify(indexLogStore).log(IndexNames.DEFAULT, User.class, 2); verify(indexLogStore.defaultIndex()).log(User.class, 2);
} }
@Test @Test
@@ -103,7 +104,7 @@ class IndexBootstrapListenerTest {
} }
private <T> void mockIndexLog(Class<T> type, @Nullable IndexLog indexLog) { private <T> void mockIndexLog(Class<T> type, @Nullable IndexLog indexLog) {
when(indexLogStore.get(IndexNames.DEFAULT, type)).thenReturn(Optional.ofNullable(indexLog)); when(indexLogStore.defaultIndex().get(type)).thenReturn(Optional.ofNullable(indexLog));
} }
private void mockAdminContext() { private void mockAdminContext() {
@@ -128,14 +129,15 @@ class IndexBootstrapListenerTest {
); );
} }
@SuppressWarnings("unchecked")
private <T> Indexer<T> indexer(Class<T> type, int version) { private <T> Indexer<T> indexer(Class<T> type, int version) {
Indexer<T> indexer = mock(Indexer.class); Indexer<T> indexer = mock(Indexer.class);
when(indexer.getType()).thenReturn(type); when(indexer.getType()).thenReturn(type);
when(indexer.getVersion()).thenReturn(version); when(indexer.getVersion()).thenReturn(version);
when(indexer.getIndex()).thenReturn(IndexNames.DEFAULT);
return indexer; return indexer;
} }
@SuppressWarnings("unchecked")
private <T> Indexer.Updater<T> updater(Indexer<T> indexer) { private <T> Indexer.Updater<T> updater(Indexer<T> indexer) {
Indexer.Updater<T> updater = mock(Indexer.Updater.class); Indexer.Updater<T> updater = mock(Indexer.Updater.class);
when(indexer.open()).thenReturn(updater); when(indexer.open()).thenReturn(updater);

View File

@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -54,6 +55,9 @@ class IndexOpenerTest {
@Mock @Mock
private AnalyzerFactory analyzerFactory; private AnalyzerFactory analyzerFactory;
@Mock
private LuceneSearchableType searchableType;
private IndexOpener indexOpener; private IndexOpener indexOpener;
@BeforeEach @BeforeEach
@@ -61,35 +65,50 @@ class IndexOpenerTest {
this.directory = tempDirectory; this.directory = tempDirectory;
SCMContextProvider context = mock(SCMContextProvider.class); SCMContextProvider context = mock(SCMContextProvider.class);
when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory); when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory);
when(analyzerFactory.create(any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer());
indexOpener = new IndexOpener(context, analyzerFactory); indexOpener = new IndexOpener(context, analyzerFactory);
} }
@Test @Test
void shouldCreateNewIndex() throws IOException { void shouldCreateNewIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { try (IndexWriter writer = open("new-index")) {
addDoc(writer, "Trillian"); addDoc(writer, "Trillian");
} }
assertThat(directory.resolve("new-index")).exists(); assertThat(directory.resolve("new-index")).exists();
} }
private IndexWriter open(String index) throws IOException {
return indexOpener.openForWrite(new IndexParams(index, searchableType, IndexOptions.defaults()));
}
@Test @Test
void shouldOpenExistingIndex() throws IOException { void shouldOpenExistingIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { try (IndexWriter writer = open("reused")) {
addDoc(writer, "Dent"); addDoc(writer, "Dent");
} }
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { try (IndexWriter writer = open("reused")) {
assertThat(writer.getFieldNames()).contains("hitchhiker"); assertThat(writer.getFieldNames()).contains("hitchhiker");
} }
} }
@Test @Test
void shouldUseAnalyzerFromFactory() throws IOException { void shouldUseAnalyzerFromFactory() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { try (IndexWriter writer = open("new-index")) {
assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class);
} }
} }
@Test
void shouldOpenIndexForRead() throws IOException {
try (IndexWriter writer = open("idx-for-read")) {
addDoc(writer, "Dent");
}
try (IndexReader reader = indexOpener.openForRead("idx-for-read")) {
assertThat(reader.numDocs()).isOne();
}
}
private void addDoc(IndexWriter writer, String name) throws IOException { private void addDoc(IndexWriter writer, String name) throws IOException {
Document doc = new Document(); Document doc = new Document();
doc.add(new TextField("hitchhiker", name, Field.Store.YES)); doc.add(new TextField("hitchhiker", name, Field.Store.YES));

View File

@@ -43,35 +43,31 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.in;
import static org.awaitility.Awaitility.await; import static org.awaitility.Awaitility.await;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
class DefaultIndexQueueTest { class IndexQueueTest {
private Directory directory; private Directory directory;
private DefaultIndexQueue queue; private IndexQueue queue;
@Mock
private LuceneQueryBuilderFactory queryBuilderFactory;
@BeforeEach @BeforeEach
void createQueue() throws IOException { void createQueue() throws IOException {
directory = new ByteBuffersDirectory(); directory = new ByteBuffersDirectory();
IndexOpener opener = mock(IndexOpener.class); IndexOpener opener = mock(IndexOpener.class);
when(opener.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> { when(opener.openForWrite(any(IndexParams.class))).thenAnswer(ic -> {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, config); return new IndexWriter(directory, config);
}); });
SearchableTypeResolver resolver = new SearchableTypeResolver(Account.class, IndexedNumber.class); LuceneIndexFactory indexFactory = new LuceneIndexFactory(opener);
LuceneIndexFactory indexFactory = new LuceneIndexFactory(resolver, opener); queue = new IndexQueue(indexFactory);
SearchEngine engine = new LuceneSearchEngine(resolver, indexFactory, queryBuilderFactory);
queue = new DefaultIndexQueue(engine);
} }
@AfterEach @AfterEach
@@ -82,13 +78,20 @@ class DefaultIndexQueueTest {
@Test @Test
void shouldWriteToIndex() throws Exception { void shouldWriteToIndex() throws Exception {
try (Index index = queue.getQueuedIndex("default")) { try (Index<Account> index = getIndex(Account.class)) {
index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan")); index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan"));
index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent")); index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent"));
} }
assertDocCount(2); assertDocCount(2);
} }
private <T> Index<T> getIndex(Class<T> type) {
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneSearchableType searchableType = resolver.resolve(type);
IndexParams indexParams = new IndexParams("default", searchableType, IndexOptions.defaults());
return queue.getQueuedIndex(indexParams);
}
@Test @Test
void shouldWriteMultiThreaded() throws Exception { void shouldWriteMultiThreaded() throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(4); ExecutorService executorService = Executors.newFixedThreadPool(4);
@@ -96,8 +99,8 @@ class DefaultIndexQueueTest {
executorService.execute(new IndexNumberTask(i)); executorService.execute(new IndexNumberTask(i));
} }
executorService.execute(() -> { executorService.execute(() -> {
try (Index index = queue.getQueuedIndex("default")) { try (Index<IndexedNumber> index = getIndex(IndexedNumber.class)) {
index.delete(Id.of(String.valueOf(12)), IndexedNumber.class); index.delete().byType().byId(Id.of(String.valueOf(12)));
} }
}); });
executorService.shutdown(); executorService.shutdown();
@@ -141,7 +144,7 @@ class DefaultIndexQueueTest {
@Override @Override
public void run() { public void run() {
try (Index index = queue.getQueuedIndex("default")) { try (Index<IndexedNumber> index = getIndex(IndexedNumber.class)) {
index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number)); index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number));
} }
} }

View File

@@ -49,6 +49,7 @@ import static sonia.scm.search.FieldNames.*;
class LuceneIndexTest { class LuceneIndexTest {
private static final Id ONE = Id.of("one"); private static final Id ONE = Id.of("one");
private static final Id TWO = Id.of("two");
private Directory directory; private Directory directory;
@@ -59,7 +60,7 @@ class LuceneIndexTest {
@Test @Test
void shouldStoreObject() throws IOException { void shouldStoreObject() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed")); index.store(ONE, null, new Storable("Awesome content which should be indexed"));
} }
@@ -68,7 +69,7 @@ class LuceneIndexTest {
@Test @Test
void shouldUpdateObject() throws IOException { void shouldUpdateObject() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed")); index.store(ONE, null, new Storable("Awesome content which should be indexed"));
index.store(ONE, null, new Storable("Awesome content")); index.store(ONE, null, new Storable("Awesome content"));
} }
@@ -78,7 +79,7 @@ class LuceneIndexTest {
@Test @Test
void shouldStoreUidOfObject() throws IOException { void shouldStoreUidOfObject() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Awesome content which should be indexed")); index.store(ONE, null, new Storable("Awesome content which should be indexed"));
} }
@@ -87,7 +88,7 @@ class LuceneIndexTest {
@Test @Test
void shouldStoreIdOfObject() throws IOException { void shouldStoreIdOfObject() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some text")); index.store(ONE, null, new Storable("Some text"));
} }
@@ -96,7 +97,7 @@ class LuceneIndexTest {
@Test @Test
void shouldStoreRepositoryOfId() throws IOException { void shouldStoreRepositoryOfId() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("Some text")); index.store(ONE.withRepository("4211"), null, new Storable("Some text"));
} }
@@ -105,7 +106,7 @@ class LuceneIndexTest {
@Test @Test
void shouldStoreTypeOfObject() throws IOException { void shouldStoreTypeOfObject() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text")); index.store(ONE, null, new Storable("Some other text"));
} }
@@ -114,12 +115,12 @@ class LuceneIndexTest {
@Test @Test
void shouldDeleteById() throws IOException { void shouldDeleteById() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text")); index.store(ONE, null, new Storable("Some other text"));
} }
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete(ONE, Storable.class); index.delete().byType().byId(ONE);
} }
assertHits(ID, "one", 0); assertHits(ID, "one", 0);
@@ -127,14 +128,17 @@ class LuceneIndexTest {
@Test @Test
void shouldDeleteAllByType() throws IOException { void shouldDeleteAllByType() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("content")); index.store(ONE, null, new Storable("content"));
index.store(Id.of("two"), null, new Storable("content")); index.store(Id.of("two"), null, new Storable("content"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(Id.of("three"), null, new OtherStorable("content")); index.store(Id.of("three"), null, new OtherStorable("content"));
} }
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.deleteByType(Storable.class); index.delete().byType().all();
} }
assertHits("value", "content", 1); assertHits("value", "content", 1);
@@ -142,13 +146,16 @@ class LuceneIndexTest {
@Test @Test
void shouldDeleteByIdAnyType() throws IOException { void shouldDeleteByIdAnyType() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some text")); index.store(ONE, null, new Storable("Some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE, null, new OtherStorable("Some other text")); index.store(ONE, null, new OtherStorable("Some other text"));
} }
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete(ONE, Storable.class); index.delete().byType().byId(ONE);
} }
assertHits(ID, "one", 1); assertHits(ID, "one", 1);
@@ -160,13 +167,13 @@ class LuceneIndexTest {
@Test @Test
void shouldDeleteByIdAndRepository() throws IOException { void shouldDeleteByIdAndRepository() throws IOException {
Id withRepository = ONE.withRepository("4211"); Id withRepository = ONE.withRepository("4211");
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("Some other text")); index.store(ONE, null, new Storable("Some other text"));
index.store(withRepository, null, new Storable("New stuff")); index.store(withRepository, null, new Storable("New stuff"));
} }
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete(withRepository, Storable.class); index.delete().byType().byId(withRepository);
} }
ScoreDoc[] docs = assertHits(ID, "one", 1); ScoreDoc[] docs = assertHits(ID, "one", 1);
@@ -176,21 +183,73 @@ class LuceneIndexTest {
@Test @Test
void shouldDeleteByRepository() throws IOException { void shouldDeleteByRepository() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("Some other text")); index.store(ONE.withRepository("4211"), null, new Storable("Some other text"));
index.store(ONE.withRepository("4212"), null, new Storable("New stuff")); index.store(ONE.withRepository("4212"), null, new Storable("New stuff"));
} }
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.deleteByRepository("4212"); index.delete().byType().byRepository("4212");
} }
assertHits(ID, "one", 1); assertHits(ID, "one", 1);
} }
@Test
void shouldDeleteByRepositoryAndType() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("some text"));
index.store(TWO.withRepository("4211"), null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE.withRepository("4211"), null, new OtherStorable("some text"));
}
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().byType().byRepository("4211");
}
ScoreDoc[] docs = assertHits("value", "text", 1);
Document doc = doc(docs[0].doc);
assertThat(doc.get(TYPE)).isEqualTo("otherStorable");
}
@Test
void shouldDeleteAllByRepository() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), null, new Storable("some text"));
index.store(TWO.withRepository("4211"), null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.store(ONE.withRepository("4211"), null, new OtherStorable("some text"));
}
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.delete().allTypes().byRepository("4211");
}
assertHits("value", "text", 0);
}
@Test
void shouldDeleteAllByTypeName() throws IOException {
try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE, null, new Storable("some text"));
index.store(TWO, null, new Storable("some text"));
}
try (LuceneIndex<OtherStorable> index = createIndex(OtherStorable.class)) {
index.delete().allTypes().byTypeName("storable");
}
assertHits("value", "text", 0);
}
@Test @Test
void shouldStorePermission() throws IOException { void shouldStorePermission() throws IOException {
try (LuceneIndex index = createIndex()) { try (LuceneIndex<Storable> index = createIndex(Storable.class)) {
index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text")); index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text"));
} }
@@ -213,9 +272,9 @@ class LuceneIndexTest {
} }
} }
private LuceneIndex createIndex() throws IOException { private <T> LuceneIndex<T> createIndex(Class<T> type) throws IOException {
SearchableTypeResolver resolver = new SearchableTypeResolver(Storable.class, OtherStorable.class); SearchableTypeResolver resolver = new SearchableTypeResolver(type);
return new LuceneIndex(resolver, createWriter()); return new LuceneIndex<>(resolver.resolve(type), createWriter());
} }
private IndexWriter createWriter() throws IOException { private IndexWriter createWriter() throws IOException {

View File

@@ -40,7 +40,6 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.shiro.authz.AuthorizationException; import org.apache.shiro.authz.AuthorizationException;
@@ -305,10 +304,10 @@ class LuceneQueryBuilderTest {
try (DirectoryReader reader = DirectoryReader.open(directory)) { try (DirectoryReader reader = DirectoryReader.open(directory)) {
when(opener.openForRead("default")).thenReturn(reader); when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class); SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class);
LuceneQueryBuilder builder = new LuceneQueryBuilder( LuceneQueryBuilder<Simple> builder = new LuceneQueryBuilder<>(
opener, resolver, "default", new StandardAnalyzer() opener, "default", resolver.resolve(Simple.class), new StandardAnalyzer()
); );
result = builder.repository("cde").execute(Simple.class, "content:awesome"); result = builder.repository("cde").execute("content:awesome");
} }
assertThat(result.getTotalHits()).isOne(); assertThat(result.getTotalHits()).isOne();
@@ -434,24 +433,6 @@ class LuceneQueryBuilderTest {
assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something")); assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something"));
} }
@Test
void shouldFailWithoutPermissionForTheSearchedType() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(denyDoc("awesome"));
}
assertThrows(AuthorizationException.class, () -> query(Deny.class, "awesome"));
}
@Test
@SubjectAware(value = "marvin", permissions = "deny:4711")
void shouldNotFailWithRequiredPermissionForTheSearchedType() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(denyDoc("awesome"));
}
QueryResult result = query(Deny.class, "awesome");
assertThat(result.getTotalHits()).isOne();
}
@Test @Test
void shouldLimitHitsByDefaultSize() throws IOException { void shouldLimitHitsByDefaultSize() throws IOException {
try (IndexWriter writer = writer()) { try (IndexWriter writer = writer()) {
@@ -577,23 +558,25 @@ class LuceneQueryBuilderTest {
return query(type, queryString, null, null); return query(type, queryString, null, null);
} }
private long count(Class<?> type, String queryString) throws IOException { private <T> long count(Class<T> type, String queryString) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) { try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader); lenient().when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(type); SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneQueryBuilder builder = new LuceneQueryBuilder( LuceneSearchableType searchableType = resolver.resolve(type);
opener, resolver, "default", new StandardAnalyzer() LuceneQueryBuilder<T> builder = new LuceneQueryBuilder<T>(
opener, "default", searchableType, new StandardAnalyzer()
); );
return builder.count(type, queryString).getTotalHits(); return builder.count(queryString).getTotalHits();
} }
} }
private QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException { private <T> QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) { try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader); lenient().when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(type); SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneQueryBuilder builder = new LuceneQueryBuilder( LuceneSearchableType searchableType = resolver.resolve(type);
opener, resolver, "default", new StandardAnalyzer() LuceneQueryBuilder<T> builder = new LuceneQueryBuilder<T>(
opener, "default", searchableType, new StandardAnalyzer()
); );
if (start != null) { if (start != null) {
builder.start(start); builder.start(start);
@@ -601,7 +584,7 @@ class LuceneQueryBuilderTest {
if (limit != null) { if (limit != null) {
builder.limit(limit); builder.limit(limit);
} }
return builder.execute(type, queryString); return builder.execute(queryString);
} }
} }

View File

@@ -24,6 +24,7 @@
package sonia.scm.search; package sonia.scm.search;
import org.apache.shiro.authz.AuthorizationException;
import org.github.sdorra.jse.ShiroExtension; import org.github.sdorra.jse.ShiroExtension;
import org.github.sdorra.jse.SubjectAware; import org.github.sdorra.jse.SubjectAware;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -31,6 +32,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.Repository;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@@ -40,6 +42,8 @@ import java.util.Locale;
import java.util.Optional; import java.util.Optional;
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -52,7 +56,7 @@ class LuceneSearchEngineTest {
private SearchableTypeResolver resolver; private SearchableTypeResolver resolver;
@Mock @Mock
private LuceneIndexFactory indexFactory; private IndexQueue indexQueue;
@Mock @Mock
private LuceneQueryBuilderFactory queryBuilderFactory; private LuceneQueryBuilderFactory queryBuilderFactory;
@@ -60,6 +64,9 @@ class LuceneSearchEngineTest {
@InjectMocks @InjectMocks
private LuceneSearchEngine searchEngine; private LuceneSearchEngine searchEngine;
@Mock
private LuceneSearchableType searchableType;
@Test @Test
void shouldDelegateGetSearchableTypes() { void shouldDelegateGetSearchableTypes() {
List<LuceneSearchableType> mockedTypes = Collections.singletonList(searchableType("repository")); List<LuceneSearchableType> mockedTypes = Collections.singletonList(searchableType("repository"));
@@ -96,43 +103,119 @@ class LuceneSearchEngineTest {
} }
@Test @Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateWithDefaultIndex() {
Index<Repository> index = mock(Index.class);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults());
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).getOrCreate();
assertThat(idx).isSameAs(index);
}
@Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateIndexWithDefaults() { void shouldDelegateGetOrCreateIndexWithDefaults() {
LuceneIndex index = mock(LuceneIndex.class); Index<Repository> index = mock(Index.class);
when(indexFactory.create("idx", IndexOptions.defaults())).thenReturn(index);
Index idx = searchEngine.getOrCreate("idx"); when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("idx", searchableType, IndexOptions.defaults());
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).withIndex("idx").getOrCreate();
assertThat(idx).isSameAs(index); assertThat(idx).isSameAs(index);
} }
@Test @Test
@SuppressWarnings("unchecked")
void shouldDelegateGetOrCreateIndex() { void shouldDelegateGetOrCreateIndex() {
LuceneIndex index = mock(LuceneIndex.class); Index<Repository> index = mock(Index.class);
IndexOptions options = IndexOptions.naturalLanguage(Locale.ENGLISH); IndexOptions options = IndexOptions.naturalLanguage(Locale.ENGLISH);
when(indexFactory.create("idx", options)).thenReturn(index);
Index idx = searchEngine.getOrCreate("idx", options); when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("default", searchableType, options);
when(indexQueue.<Repository>getQueuedIndex(params)).thenReturn(index);
Index<Repository> idx = searchEngine.forType(Repository.class).withOptions(options).getOrCreate();
assertThat(idx).isSameAs(index); assertThat(idx).isSameAs(index);
} }
@Test @Test
@SuppressWarnings("unchecked")
void shouldDelegateSearchWithDefaults() { void shouldDelegateSearchWithDefaults() {
LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); LuceneQueryBuilder<Repository> mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create("idx", IndexOptions.defaults())).thenReturn(mockedBuilder); when(resolver.resolve(Repository.class)).thenReturn(searchableType);
QueryBuilder queryBuilder = searchEngine.search("idx"); IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults());
when(queryBuilderFactory.<Repository>create(params)).thenReturn(mockedBuilder);
QueryBuilder<Repository> queryBuilder = searchEngine.forType(Repository.class).search();
assertThat(queryBuilder).isSameAs(mockedBuilder); assertThat(queryBuilder).isSameAs(mockedBuilder);
} }
@Test @Test
@SuppressWarnings("unchecked")
void shouldDelegateSearch() { void shouldDelegateSearch() {
LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class);
IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN); IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN);
when(queryBuilderFactory.create("idx", options)).thenReturn(mockedBuilder);
QueryBuilder queryBuilder = searchEngine.search("idx", options); LuceneQueryBuilder<Repository> mockedBuilder = mock(LuceneQueryBuilder.class);
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
IndexParams params = new IndexParams("idx", searchableType, options);
when(queryBuilderFactory.<Repository>create(params)).thenReturn(mockedBuilder);
QueryBuilder<Repository> queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search();
assertThat(queryBuilder).isSameAs(mockedBuilder); assertThat(queryBuilder).isSameAs(mockedBuilder);
} }
@Test
void shouldFailWithoutRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
SearchEngine.ForType<Repository> forType = searchEngine.forType(Repository.class);
assertThrows(AuthorizationException.class, forType::search);
}
@Test
@SuppressWarnings("unchecked")
@SubjectAware(permissions = "repository:read")
void shouldNotFailWithRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolve(Repository.class)).thenReturn(searchableType);
LuceneQueryBuilder<Object> mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder);
SearchEngine.ForType<Repository> forType = searchEngine.forType(Repository.class);
assertThat(forType.search()).isNotNull();
}
@Test
void shouldFailWithTypeNameWithoutRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolveByName("repository")).thenReturn(searchableType);
SearchEngine.ForType<Object> forType = searchEngine.forType("repository");
assertThrows(AuthorizationException.class, forType::search);
}
@Test
@SuppressWarnings("unchecked")
@SubjectAware(permissions = "repository:read")
void shouldNotFailWithTypeNameAndRequiredPermission() {
when(searchableType.getPermission()).thenReturn(Optional.of("repository:read"));
when(resolver.resolveByName("repository")).thenReturn(searchableType);
LuceneQueryBuilder<Object> mockedBuilder = mock(LuceneQueryBuilder.class);
when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder);
SearchEngine.ForType<Object> forType = searchEngine.forType("repository");
assertThat(forType.search()).isNotNull();
}
} }

View File

@@ -0,0 +1,75 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat;
class NonNaturalLanguageAnalyzerTest {
private final NonNaturalLanguageAnalyzer analyzer = new NonNaturalLanguageAnalyzer();
@ParameterizedTest
@ValueSource(strings = {
"simple text", "simple-text", "simple(text)", "simple[text]",
"simple{text}", "simple/text", "simple;text", "simple$text",
"simple\\text"
})
void shouldTokenize(String value) throws IOException {
List<String> tokens = tokenize(value);
assertThat(tokens).containsOnly("simple", "text");
}
@ParameterizedTest
@ValueSource(strings = {
"simple.text", "simple_text", "simpleText", "simple:text", "SimpleText"
})
void shouldTokenizeAndPreserveOriginal(String value) throws IOException {
List<String> tokens = tokenize(value);
assertThat(tokens).containsOnly("simple", "text", value.toLowerCase(Locale.ENGLISH));
}
@Test
void shouldSplitOnNumeric() throws IOException {
List<String> tokens = tokenize("simple42text");
assertThat(tokens).containsOnly("simple", "42", "text", "simple42text");
}
private List<String> tokenize(String text) throws IOException {
return Analyzers.tokenize(analyzer, text);
}
}

View File

@@ -28,13 +28,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Answers;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType; import sonia.scm.HandlerEventType;
import sonia.scm.search.Id; import sonia.scm.search.Id;
import sonia.scm.search.Index; import sonia.scm.search.Index;
import sonia.scm.search.IndexQueue; import sonia.scm.search.SearchEngine;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
@@ -47,22 +48,17 @@ class UserIndexerTest {
@Mock @Mock
private UserManager userManager; private UserManager userManager;
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private IndexQueue indexQueue; private SearchEngine searchEngine;
@InjectMocks @InjectMocks
private UserIndexer indexer; private UserIndexer indexer;
@Test @Test
void shouldReturnRepositoryClass() { void shouldReturnType() {
assertThat(indexer.getType()).isEqualTo(User.class); assertThat(indexer.getType()).isEqualTo(User.class);
} }
@Test
void shouldReturnIndexName() {
assertThat(indexer.getIndex()).isEqualTo(UserIndexer.INDEX);
}
@Test @Test
void shouldReturnVersion() { void shouldReturnVersion() {
assertThat(indexer.getVersion()).isEqualTo(UserIndexer.VERSION); assertThat(indexer.getVersion()).isEqualTo(UserIndexer.VERSION);
@@ -71,28 +67,28 @@ class UserIndexerTest {
@Nested @Nested
class UpdaterTests { class UpdaterTests {
@Mock @Mock(answer = Answers.RETURNS_DEEP_STUBS)
private Index index; private Index<User> index;
private final User user = UserTestData.createTrillian(); private final User user = UserTestData.createTrillian();
@BeforeEach @BeforeEach
void open() { void open() {
when(indexQueue.getQueuedIndex(UserIndexer.INDEX)).thenReturn(index); when(searchEngine.forType(User.class).getOrCreate()).thenReturn(index);
} }
@Test @Test
void shouldStoreRepository() { void shouldStore() {
indexer.open().store(user); indexer.open().store(user);
verify(index).store(Id.of(user), "user:read:trillian", user); verify(index).store(Id.of(user), "user:read:trillian", user);
} }
@Test @Test
void shouldDeleteByRepository() { void shouldDeleteById() {
indexer.open().delete(user); indexer.open().delete(user);
verify(index).delete(Id.of(user), User.class); verify(index.delete().byType()).byId(Id.of(user));
} }
@Test @Test
@@ -101,7 +97,7 @@ class UserIndexerTest {
indexer.open().reIndexAll(); indexer.open().reIndexAll();
verify(index).deleteByType(User.class); verify(index.delete().byType()).all();
verify(index).store(Id.of(user), "user:read:trillian", user); verify(index).store(Id.of(user), "user:read:trillian", user);
} }
@@ -111,7 +107,7 @@ class UserIndexerTest {
indexer.handleEvent(event); indexer.handleEvent(event);
verify(index).delete(Id.of(user), User.class); verify(index.delete().byType()).byId(Id.of(user));
} }
@Test @Test