Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-02 19:45:51 +01:00
Code Issues Packages Projects Releases Activity

Prepare search api for different types (#1732)

Browse Source 
We introduced a new annotation '@IndexedType' which gets collected by the scm-annotation-processor. All classes which are annotated are index and searchable. This opens the search api for plugins.
This commit is contained in:
Sebastian Sdorra
2021-07-19 08:48:43 +02:00
committed by GitHub
parent 2de60a3007
commit e75d937ee5
36 changed files with 677 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
scm-core/src/main/java/sonia/scm/search/Indexed.java → scm-annotations/src/main/java/sonia/scm/search/Indexed.java
View File

54
scm-annotations/src/main/java/sonia/scm/search/IndexedType.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import sonia.scm.plugin.PluginAnnotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Mark an field object should be indexed.
*
* @since 2.21.0
*/
@Beta
@Documented
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@PluginAnnotation("indexed-type")
public @interface IndexedType {
/**
* Returns the name of the indexed object.
* Default is the simple name of the indexed class, with the first char is lowercase.
*
* @return name of the index object or an empty string which indicates that the default should be used.
*/
String value() default "";
}

21
scm-core/src/main/java/sonia/scm/plugin/ExtensionProcessor.java
View File

@@ -28,6 +28,8 @@ package sonia.scm.plugin;
import com.google.inject.Binder;
import java.util.Collections;
/**
* Process and resolve extensions.
*
@@ -46,8 +48,7 @@ public interface ExtensionProcessor
*
* @return extensions
*/
public <T> Iterable<Class<? extends T>> byExtensionPoint(
Class<T> extensionPoint);
<T> Iterable<Class<? extends T>> byExtensionPoint(Class<T> extensionPoint);
/**
* Returns single extension by its extension point.
@@ -58,7 +59,7 @@ public interface ExtensionProcessor
*
* @return extension
*/
public <T> Class<? extends T> oneByExtensionPoint(Class<T> extensionPoint);
<T> Class<? extends T> oneByExtensionPoint(Class<T> extensionPoint);
/**
* Process auto bind extensions.
@@ -66,7 +67,7 @@ public interface ExtensionProcessor
*
* @param binder injection binder
*/
public void processAutoBindExtensions(Binder binder);
void processAutoBindExtensions(Binder binder);
//~--- get methods ----------------------------------------------------------
@@ -76,5 +77,15 @@ public interface ExtensionProcessor
*
* @return collected web elements
*/
public Iterable<WebElementExtension> getWebElements();
Iterable<WebElementExtension> getWebElements();
/**
* Returns all collected indexable types.
*
* @return collected indexable types
* @since 2.21.0
*/
default Iterable<Class<?>> getIndexedTypes() {
return Collections.emptySet();
}
}

16
scm-core/src/main/java/sonia/scm/plugin/ScmModule.java
View File

@@ -26,18 +26,15 @@ package sonia.scm.plugin;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
//~--- JDK imports ------------------------------------------------------------
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.Set;
//~--- JDK imports ------------------------------------------------------------
/**
*
@@ -127,6 +124,10 @@ public class ScmModule
return nonNull(webElements);
}
public Iterable<ClassElement> getIndexedTypes() {
return nonNull(indexedTypes);
}
//~--- methods --------------------------------------------------------------
/**
@@ -151,6 +152,9 @@ public class ScmModule
//~--- fields ---------------------------------------------------------------
@XmlElement(name = "indexed-type")
private Set<ClassElement> indexedTypes;
/** Field description */
@XmlElement(name = "event")
private Set<ClassElement> events;

2
scm-core/src/main/java/sonia/scm/repository/Repository.java
View File

@@ -32,6 +32,7 @@ import com.google.common.base.Objects;
import sonia.scm.BasicPropertiesAware;
import sonia.scm.ModelObject;
import sonia.scm.search.Indexed;
import sonia.scm.search.IndexedType;
import sonia.scm.util.Util;
import sonia.scm.util.ValidationUtil;
@@ -52,6 +53,7 @@ import java.util.Set;
*
* @author Sebastian Sdorra
*/
@IndexedType
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "repositories")
@StaticPermissions(

7
scm-core/src/main/java/sonia/scm/search/Index.java
View File

@@ -67,6 +67,13 @@ public interface Index extends AutoCloseable {
*/
void deleteByType(Class<?> type);
/**
* Delete all objects with the given type from index.
* This method is mostly if the index type has changed and the old type (in form of class) is no longer available.
* @param typeName type name of objects
*/
void deleteByTypeName(String typeName);
/**
* Close index and commit changes.
*/

33
scm-core/src/main/java/sonia/scm/search/QueryBuilder.java
View File

@@ -26,10 +26,15 @@ package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.Value;
import sonia.scm.ContextEntry;
import sonia.scm.NotFoundException;
import sonia.scm.repository.Repository;
import java.util.Optional;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.NotFoundException.notFound;
/**
* Build and execute queries against an index.
*
@@ -42,6 +47,7 @@ public abstract class QueryBuilder {
private int start = 0;
private int limit = 10;
/**
* Return only results which are related to the given repository.
* @param repository repository
@@ -93,6 +99,33 @@ public abstract class QueryBuilder {
return execute(new QueryParams(type, repositoryId, queryString, start, limit));
}
/**
* Executes the query and returns the matches.
*
* @param typeName type name of objects which are searched
* @param queryString searched query
* @return result of query
*
* @throws NotFoundException if type could not be found
*/
public QueryResult execute(String typeName, String queryString){
Class<?> type = resolveByName(typeName).orElseThrow(() -> notFound(entity("type", typeName)));
return execute(type, queryString);
}
/**
* Resolves the type by its name. Returns optional with class of type or an empty optional.
*
* @param typeName name of type
* @return optional with class of type or empty
*/
protected abstract Optional<Class<?>> resolveByName(String typeName);
/**
* Executes the query and returns the matches.
* @param queryParams query parameter
* @return result of query
*/
protected abstract QueryResult execute(QueryParams queryParams);
/**

10
scm-ui/ui-api/src/search.ts
View File

@@ -29,14 +29,18 @@ import { createQueryString } from "./utils";
import { useQuery } from "react-query";
export type SearchOptions = {
type: string;
page?: number;
pageSize?: number;
};
const defaultSearchOptions: SearchOptions = {};
const defaultSearchOptions: SearchOptions = {
type: "repository",
};
export const useSearch = (query: string, options = defaultSearchOptions): ApiResult<QueryResult> => {
const link = useRequiredIndexLink("search");
export const useSearch = (query: string, optionParam = defaultSearchOptions): ApiResult<QueryResult> => {
const options = { ...defaultSearchOptions, ...optionParam };
const link = useRequiredIndexLink("search").replace("{type}", options.type);
const queryParams: Record<string, string> = {};
queryParams.q = query;

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java
View File

@@ -132,7 +132,7 @@ public class IndexDtoGenerator extends HalAppenderMapper {
builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self()));
builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}")));
builder.single(link("search", resourceLinks.search().search()));
builder.single(link("search", resourceLinks.search().search("INDEXED_TYPE").replace("INDEXED_TYPE", "{type}")));
} else {
builder.single(link("login", resourceLinks.authentication().jsonLogin()));
}

4
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java
View File

@@ -1125,8 +1125,8 @@ class ResourceLinks {
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class);
}
public String search() {
return searchLinkBuilder.method("search").parameters().href();
public String search(String type) {
return searchLinkBuilder.method("search").parameters(type).href();
}
}

4
scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java
View File

@@ -30,6 +30,7 @@ import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.Size;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
@@ -55,6 +56,9 @@ public class SearchParameters {
@DefaultValue("10")
private int pageSize = 10;
@PathParam("type")
private String type;
String getSelfLink() {
return uriInfo.getAbsolutePath().toASCIIString();
}

5
scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java
View File

@@ -31,7 +31,6 @@ import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.repository.Repository;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
@@ -62,7 +61,7 @@ public class SearchResource {
}
@GET
@Path("")
@Path("{type}")
@Produces(VndMediaType.QUERY_RESULT)
@Operation(
summary = "Query result",
@@ -103,7 +102,7 @@ public class SearchResource {
QueryResult result = engine.search(IndexNames.DEFAULT)
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
.execute(Repository.class, params.getQuery());
.execute(params.getType(), params.getQuery());
return mapper.map(params, result);
}

5
scm-webapp/src/main/java/sonia/scm/plugin/DefaultExtensionProcessor.java
View File

@@ -120,6 +120,11 @@ public class DefaultExtensionProcessor implements ExtensionProcessor
return collector.getWebElements();
}
@Override
public Iterable<Class<?>> getIndexedTypes() {
return collector.getIndexedTypes();
}
//~--- fields ---------------------------------------------------------------
/** Field description */

24
scm-webapp/src/main/java/sonia/scm/plugin/ExtensionCollector.java
View File

@@ -223,6 +223,10 @@ public final class ExtensionCollector
return webElements;
}
public Set<Class<?>> getIndexedTypes() {
return indexedTypes;
}
//~--- methods --------------------------------------------------------------
/**
@@ -255,7 +259,7 @@ public final class ExtensionCollector
private void collectExtensions(ClassLoader defaultClassLoader, ScmModule module) {
for (ClassElement extension : module.getExtensions()) {
if (isRequirementFulfilled(extension)) {
Class<?> extensionClass = loadExtension(defaultClassLoader, extension);
Class<?> extensionClass = load(defaultClassLoader, extension);
appendExtension(extensionClass);
}
}
@@ -265,25 +269,34 @@ public final class ExtensionCollector
Set<Class<?>> classes = new HashSet<>();
for (ClassElement element : classElements) {
if (isRequirementFulfilled(element)) {
Class<?> loadedClass = loadExtension(defaultClassLoader, element);
Class<?> loadedClass = load(defaultClassLoader, element);
classes.add(loadedClass);
}
}
return classes;
}
private Collection<Class<?>> collectIndexedTypes(ClassLoader defaultClassLoader, Iterable<ClassElement> descriptors) {
Set<Class<?>> types = new HashSet<>();
for (ClassElement descriptor : descriptors) {
Class<?> loadedClass = load(defaultClassLoader, descriptor);
types.add(loadedClass);
}
return types;
}
private Set<WebElementExtension> collectWebElementExtensions(ClassLoader defaultClassLoader, Iterable<WebElementDescriptor> descriptors) {
Set<WebElementExtension> webElementExtensions = new HashSet<>();
for (WebElementDescriptor descriptor : descriptors) {
if (isRequirementFulfilled(descriptor)) {
Class<?> loadedClass = loadExtension(defaultClassLoader, descriptor);
Class<?> loadedClass = load(defaultClassLoader, descriptor);
webElementExtensions.add(new WebElementExtension(loadedClass, descriptor));
}
}
return webElementExtensions;
}
private Class<?> loadExtension(ClassLoader classLoader, ClassElement extension) {
private Class<?> load(ClassLoader classLoader, ClassElement extension) {
try {
return classLoader.loadClass(extension.getClazz());
} catch (ClassNotFoundException ex) {
@@ -320,6 +333,7 @@ public final class ExtensionCollector
restResources.addAll(collectClasses(classLoader, module.getRestResources()));
webElements.addAll(collectWebElementExtensions(classLoader, module.getWebElements()));
indexedTypes.addAll(collectIndexedTypes(classLoader, module.getIndexedTypes()));
}
//~--- fields ---------------------------------------------------------------
@@ -327,6 +341,8 @@ public final class ExtensionCollector
/** Field description */
private final Set<WebElementExtension> webElements = Sets.newHashSet();
private final Set<Class<?>> indexedTypes = Sets.newHashSet();
/** Field description */
private final Set<Class> restResources = Sets.newHashSet();

18
scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java
View File

@@ -52,7 +52,7 @@ public class IndexUpdateListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class);
@VisibleForTesting
static final int INDEX_VERSION = 1;
static final int INDEX_VERSION = 2;
private final AdministrationContext administrationContext;
private final IndexQueue queue;
@@ -87,12 +87,22 @@ public class IndexUpdateListener implements ServletContextListener {
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
Optional<IndexLog> indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class);
if (!indexLog.isPresent()) {
if (indexLog.isPresent()) {
int version = indexLog.get().getVersion();
if (version < INDEX_VERSION) {
LOG.debug("repository index {} is older then {}, start reindexing of all repositories", version, INDEX_VERSION);
indexAll();
}
} else {
LOG.debug("could not find log entry for repository index, start reindexing of all repositories");
indexAll();
}
}
private void indexAll() {
administrationContext.runAsAdmin(ReIndexAll.class);
indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION);
}
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
@@ -117,6 +127,8 @@ public class IndexUpdateListener implements ServletContextListener {
@Override
public void run() {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
// delete v1 types
index.deleteByTypeName(Repository.class.getName());
for (Repository repository : repositoryManager.getAll()) {
store(index, repository);
}

144
scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java
View File

@@ -1,144 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import javax.inject.Singleton;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Collections.emptySet;
@Singleton
public class DocumentConverter {
private final Map<Class<?>, TypeConverter> typeConverter = new ConcurrentHashMap<>();
Document convert(Object object) {
TypeConverter converter = typeConverter.computeIfAbsent(object.getClass(), this::createTypeConverter);
try {
return converter.convert(object);
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new SearchEngineException("failed to create document", ex);
}
}
private TypeConverter createTypeConverter(Class<?> type) {
List<FieldConverter> fieldConverters = new ArrayList<>();
collectFields(fieldConverters, type);
return new TypeConverter(fieldConverters);
}
private void collectFields(List<FieldConverter> fieldConverters, Class<?> type) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(fieldConverters, parent);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed);
Method getter = findGetter(type, field);
fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory));
}
}
}
private Method findGetter(Class<?> type, Field field) {
String name = createGetterName(field);
try {
return type.getMethod(name);
} catch (NoSuchMethodException ex) {
throw new NonReadableFieldException("could not find getter for field", ex);
}
}
private String createGetterName(Field field) {
String fieldName = field.getName();
String prefix = "get";
if (field.getType() == Boolean.TYPE) {
prefix = "is";
}
return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1);
}
private static class TypeConverter {
private final List<FieldConverter> fieldConverters;
private TypeConverter(List<FieldConverter> fieldConverters) {
this.fieldConverters = fieldConverters;
}
public Document convert(Object object) throws IllegalAccessException, InvocationTargetException {
Document document = new Document();
for (FieldConverter fieldConverter : fieldConverters) {
for (IndexableField field : fieldConverter.convert(object)) {
document.add(field);
}
}
return document;
}
}
private static class FieldConverter {
private final Method getter;
private final IndexableFieldFactory fieldFactory;
private final String name;
private FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) {
this.getter = getter;
this.fieldFactory = fieldFactory;
this.name = createName(field, indexed);
}
private String createName(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (Strings.isNullOrEmpty(nameFromAnnotation)) {
return field.getName();
}
return nameFromAnnotation;
}
Iterable<IndexableField> convert(Object object) throws IllegalAccessException, InvocationTargetException {
Object value = getter.invoke(object);
if (value != null) {
return fieldFactory.create(name, value);
}
return emptySet();
}
}
}

64
scm-webapp/src/main/java/sonia/scm/search/FieldConverter.java Normal file
View File

@@ -0,0 +1,64 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.index.IndexableField;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import static java.util.Collections.emptySet;
final class FieldConverter {
private final Method getter;
private final IndexableFieldFactory fieldFactory;
private final String name;
FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) {
this.getter = getter;
this.fieldFactory = fieldFactory;
this.name = createName(field, indexed);
}
private String createName(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (Strings.isNullOrEmpty(nameFromAnnotation)) {
return field.getName();
}
return nameFromAnnotation;
}
Iterable<IndexableField> convert(Object object) throws IllegalAccessException, InvocationTargetException {
Object value = getter.invoke(object);
if (value != null) {
return fieldFactory.create(name, value);
}
return emptySet();
}
}

34
scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java
View File

@@ -33,27 +33,32 @@ import org.apache.lucene.index.Term;
import java.io.IOException;
import static sonia.scm.search.FieldNames.*;
import static sonia.scm.search.FieldNames.ID;
import static sonia.scm.search.FieldNames.PERMISSION;
import static sonia.scm.search.FieldNames.REPOSITORY;
import static sonia.scm.search.FieldNames.TYPE;
import static sonia.scm.search.FieldNames.UID;
public class LuceneIndex implements Index {
private final DocumentConverter converter;
private final SearchableTypeResolver resolver;
private final IndexWriter writer;
LuceneIndex(DocumentConverter converter, IndexWriter writer) {
this.converter = converter;
LuceneIndex(SearchableTypeResolver resolver, IndexWriter writer) {
this.resolver = resolver;
this.writer = writer;
}
@Override
public void store(Id id, String permission, Object object) {
String uid = createUid(id, object.getClass());
Document document = converter.convert(object);
SearchableType type = resolver.resolve(object);
String uid = createUid(id, type);
Document document = type.getTypeConverter().convert(object);
try {
field(document, UID, uid);
field(document, ID, id.getValue());
id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository));
field(document, TYPE, object.getClass().getName());
field(document, TYPE, type.getName());
if (!Strings.isNullOrEmpty(permission)) {
field(document, PERMISSION, permission);
}
@@ -63,7 +68,7 @@ public class LuceneIndex implements Index {
}
}
private String createUid(Id id, Class<?> type) {
private String createUid(Id id, SearchableType type) {
return id.asString() + "/" + type.getName();
}
@@ -73,8 +78,9 @@ public class LuceneIndex implements Index {
@Override
public void delete(Id id, Class<?> type) {
SearchableType searchableType = resolver.resolve(type);
try {
writer.deleteDocuments(new Term(UID, createUid(id, type)));
writer.deleteDocuments(new Term(UID, createUid(id, searchableType)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
@@ -91,10 +97,16 @@ public class LuceneIndex implements Index {
@Override
public void deleteByType(Class<?> type) {
SearchableType searchableType = resolver.resolve(type);
deleteByTypeName(searchableType.getName());
}
@Override
public void deleteByTypeName(String typeName) {
try {
writer.deleteDocuments(new Term(TYPE, type.getName()));
writer.deleteDocuments(new Term(TYPE, typeName));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + type + " from index", ex);
throw new SearchEngineException("failed to delete documents by repository " + typeName + " from index", ex);
}
}

48
scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import javax.inject.Inject;
import java.io.IOException;
public class LuceneIndexFactory {
private final SearchableTypeResolver typeResolver;
private final IndexOpener indexOpener;
@Inject
public LuceneIndexFactory(SearchableTypeResolver typeResolver, IndexOpener indexOpener) {
this.typeResolver = typeResolver;
this.indexOpener = indexOpener;
}
public LuceneIndex create(String name, IndexOptions options) {
try {
return new LuceneIndex(typeResolver, indexOpener.openForWrite(name, options));
} catch (IOException ex) {
throw new SearchEngineException("failed to open index " + name, ex);
}
}
}

19
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java
View File

@@ -46,32 +46,37 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Optional;
public class LuceneQueryBuilder extends QueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class);
private static final Map<Class<?>, SearchableType> CACHE = new ConcurrentHashMap<>();
private final IndexOpener opener;
private final SearchableTypeResolver resolver;
private final String indexName;
private final Analyzer analyzer;
LuceneQueryBuilder(IndexOpener opener, String indexName, Analyzer analyzer) {
LuceneQueryBuilder(IndexOpener opener, SearchableTypeResolver resolver, String indexName, Analyzer analyzer) {
this.opener = opener;
this.resolver = resolver;
this.indexName = indexName;
this.analyzer = analyzer;
}
@Override
protected Optional<Class<?>> resolveByName(String typeName) {
return resolver.resolveClassByName(typeName);
}
@Override
protected QueryResult execute(QueryParams queryParams) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString());
SearchableType searchableType = CACHE.computeIfAbsent(queryParams.getType(), SearchableTypes::create);
SearchableType searchableType = resolver.resolve(queryParams.getType());
Query query = Queries.filter(createQuery(searchableType, queryParams, queryString), queryParams);
Query parsedQuery = createQuery(searchableType, queryParams, queryString);
Query query = Queries.filter(parsedQuery, searchableType, queryParams);
if (LOG.isDebugEnabled()) {
LOG.debug("execute lucene query: {}", query);
}

6
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java
View File

@@ -29,16 +29,18 @@ import javax.inject.Inject;
public class LuceneQueryBuilderFactory {
private final IndexOpener indexOpener;
private final SearchableTypeResolver searchableTypeResolver;
private final AnalyzerFactory analyzerFactory;
@Inject
public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) {
public LuceneQueryBuilderFactory(IndexOpener indexOpener, SearchableTypeResolver searchableTypeResolver, AnalyzerFactory analyzerFactory) {
this.indexOpener = indexOpener;
this.searchableTypeResolver = searchableTypeResolver;
this.analyzerFactory = analyzerFactory;
}
public LuceneQueryBuilder create(String name, IndexOptions options) {
return new LuceneQueryBuilder(indexOpener, name, analyzerFactory.create(options));
return new LuceneQueryBuilder(indexOpener, searchableTypeResolver, name, analyzerFactory.create(options));
}
}

15
scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java
View File

@@ -25,28 +25,21 @@
package sonia.scm.search;
import javax.inject.Inject;
import java.io.IOException;
public class LuceneSearchEngine implements SearchEngine {
private final IndexOpener indexOpener;
private final DocumentConverter converter;
private final LuceneIndexFactory indexFactory;
private final LuceneQueryBuilderFactory queryBuilderFactory;
@Inject
public LuceneSearchEngine(IndexOpener indexOpener, DocumentConverter converter, LuceneQueryBuilderFactory queryBuilderFactory) {
this.indexOpener = indexOpener;
this.converter = converter;
public LuceneSearchEngine(LuceneIndexFactory indexFactory, LuceneQueryBuilderFactory queryBuilderFactory) {
this.indexFactory = indexFactory;
this.queryBuilderFactory = queryBuilderFactory;
}
@Override
public Index getOrCreate(String name, IndexOptions options) {
try {
return new LuceneIndex(converter, indexOpener.openForWrite(name, options));
} catch (IOException ex) {
throw new SearchEngineException("failed to open index", ex);
}
return indexFactory.create(name, options);
}
@Override

6
scm-webapp/src/main/java/sonia/scm/search/Queries.java
View File

@@ -36,7 +36,7 @@ final class Queries {
private Queries() {
}
private static Query typeQuery(Class<?> type) {
private static Query typeQuery(SearchableType type) {
return new TermQuery(new Term(FieldNames.TYPE, type.getName()));
}
@@ -44,10 +44,10 @@ final class Queries {
return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId));
}
static Query filter(Query query, QueryBuilder.QueryParams params) {
static Query filter(Query query, SearchableType searchableType, QueryBuilder.QueryParams params) {
BooleanQuery.Builder builder = new BooleanQuery.Builder()
.add(query, MUST)
.add(typeQuery(params.getType()), MUST);
.add(typeQuery(searchableType), MUST);
params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST));
return builder.build();
}

5
scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java
View File

@@ -61,6 +61,11 @@ public class QueuedIndex implements Index {
tasks.add(index -> index.deleteByType(type));
}
@Override
public void deleteByTypeName(String typeName) {
tasks.add(index -> index.deleteByTypeName(typeName));
}
@Override
public void close() {
IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper(

26
scm-webapp/src/main/java/sonia/scm/search/SearchableType.java
View File

@@ -24,6 +24,7 @@
package sonia.scm.search;
import com.google.common.base.Strings;
import lombok.Value;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
@@ -35,17 +36,40 @@ import java.util.Map;
public class SearchableType {
Class<?> type;
String name;
String[] fieldNames;
Map<String,Float> boosts;
Map<String, PointsConfig> pointsConfig;
List<SearchableField> fields;
TypeConverter typeConverter;
SearchableType(Class<?> type, String[] fieldNames, Map<String, Float> boosts, Map<String, PointsConfig> pointsConfig, List<SearchableField> fields) {
SearchableType(Class<?> type,
String[] fieldNames,
Map<String, Float> boosts,
Map<String, PointsConfig> pointsConfig,
List<SearchableField> fields,
TypeConverter typeConverter) {
this.type = type;
this.name = name(type);
this.fieldNames = fieldNames;
this.boosts = Collections.unmodifiableMap(boosts);
this.pointsConfig = Collections.unmodifiableMap(pointsConfig);
this.fields = Collections.unmodifiableList(fields);
this.typeConverter = typeConverter;
}
private String name(Class<?> type) {
IndexedType annotation = type.getAnnotation(IndexedType.class);
if (annotation == null) {
throw new IllegalArgumentException(
type.getName() + " has no " + IndexedType.class.getSimpleName() + " annotation"
);
}
String nameFromAnnotation = annotation.value();
if (Strings.isNullOrEmpty(nameFromAnnotation)) {
String simpleName = type.getSimpleName();
return Character.toLowerCase(simpleName.charAt(0)) + simpleName.substring(1);
}
return nameFromAnnotation;
}
}

95
scm-webapp/src/main/java/sonia/scm/search/SearchableTypeResolver.java Normal file
View File

@@ -0,0 +1,95 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.VisibleForTesting;
import sonia.scm.plugin.PluginLoader;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.NotFoundException.notFound;
@Singleton
class SearchableTypeResolver {
private final Map<Class<?>, SearchableType> classToSearchableType = new HashMap<>();
private final Map<String, Class<?>> nameToClass = new HashMap<>();
@Inject
public SearchableTypeResolver(PluginLoader pluginLoader) {
this(pluginLoader.getExtensionProcessor().getIndexedTypes());
}
@VisibleForTesting
SearchableTypeResolver(Class<?>... indexedTypes) {
this(Arrays.asList(indexedTypes));
}
@VisibleForTesting
SearchableTypeResolver(Iterable<Class<?>> indexedTypes) {
fillMaps(convert(indexedTypes));
}
private void fillMaps(Iterable<SearchableType> types) {
for (SearchableType type : types) {
classToSearchableType.put(type.getType(), type);
nameToClass.put(type.getName(), type.getType());
}
}
@Nonnull
private Set<SearchableType> convert(Iterable<Class<?>> indexedTypes) {
return StreamSupport.stream(indexedTypes.spliterator(), false)
.map(SearchableTypes::create)
.collect(Collectors.toSet());
}
public SearchableType resolve(Object object) {
return resolve(object.getClass());
}
public SearchableType resolve(Class<?> type) {
SearchableType searchableType = classToSearchableType.get(type);
if (searchableType == null) {
throw notFound(entity("type", type.getName()));
}
return searchableType;
}
public Optional<Class<?>> resolveClassByName(String typeName) {
return Optional.ofNullable(nameToClass.get(typeName));
}
}

2
scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java
View File

@@ -63,7 +63,7 @@ final class SearchableTypes {
}
}
return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields);
return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields, TypeConverters.create(type));
}
private static void collectFields(Class<?> type, List<SearchableField> fields) {

61
scm-webapp/src/main/java/sonia/scm/search/TypeConverter.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import javax.annotation.Nonnull;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
final class TypeConverter {
private final List<FieldConverter> fieldConverters;
TypeConverter(List<FieldConverter> fieldConverters) {
this.fieldConverters = fieldConverters;
}
public Document convert(Object object) {
try {
return doConversion(object);
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new SearchEngineException("failed to create document", ex);
}
}
@Nonnull
private Document doConversion(Object object) throws IllegalAccessException, InvocationTargetException {
Document document = new Document();
for (FieldConverter fieldConverter : fieldConverters) {
for (IndexableField field : fieldConverter.convert(object)) {
document.add(field);
}
}
return document;
}
}

77
scm-webapp/src/main/java/sonia/scm/search/TypeConverters.java Normal file
View File

@@ -0,0 +1,77 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
final class TypeConverters {
private TypeConverters() {
}
static TypeConverter create(Class<?> type) {
List<FieldConverter> fieldConverters = new ArrayList<>();
collectFields(fieldConverters, type);
return new TypeConverter(fieldConverters);
}
private static void collectFields(List<FieldConverter> fieldConverters, Class<?> type) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(fieldConverters, parent);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed);
Method getter = findGetter(type, field);
fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory));
}
}
}
private static Method findGetter(Class<?> type, Field field) {
String name = createGetterName(field);
try {
return type.getMethod(name);
} catch (NoSuchMethodException ex) {
throw new NonReadableFieldException("could not find getter for field", ex);
}
}
private static String createGetterName(Field field) {
String fieldName = field.getName();
String prefix = "get";
if (field.getType() == Boolean.TYPE) {
prefix = "is";
}
return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1);
}
}

2
scm-webapp/src/test/java/sonia/scm/api/v2/ContentTypeResolverTest.java → scm-webapp/src/test/java/sonia/scm/api/v2/ContentSearchableTypeResolverTest.java
View File

@@ -32,7 +32,7 @@ import java.nio.charset.StandardCharsets;
import static org.assertj.core.api.Assertions.assertThat;
class ContentTypeResolverTest {
class ContentSearchableTypeResolverTest {
@Test
void shouldResolveMarkdown() {

14
scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java
View File

@@ -135,11 +135,11 @@ class SearchResourceTest {
JsonMockHttpResponse response = search("paging", 1, 20);
JsonNode links = response.getContentAsJson().get("_links");
assertLink(links, "self", "/v2/search?q=paging&page=1&pageSize=20");
assertLink(links, "first", "/v2/search?q=paging&page=0&pageSize=20");
assertLink(links, "prev", "/v2/search?q=paging&page=0&pageSize=20");
assertLink(links, "next", "/v2/search?q=paging&page=2&pageSize=20");
assertLink(links, "last", "/v2/search?q=paging&page=4&pageSize=20");
assertLink(links, "self", "/v2/search/string?q=paging&page=1&pageSize=20");
assertLink(links, "first", "/v2/search/string?q=paging&page=0&pageSize=20");
assertLink(links, "prev", "/v2/search/string?q=paging&page=0&pageSize=20");
assertLink(links, "next", "/v2/search/string?q=paging&page=2&pageSize=20");
assertLink(links, "last", "/v2/search/string?q=paging&page=4&pageSize=20");
}
@Test
@@ -220,7 +220,7 @@ class SearchResourceTest {
searchEngine.search(IndexNames.DEFAULT)
.start(start)
.limit(limit)
.execute(Repository.class, query)
.execute("string", query)
).thenReturn(result);
}
@@ -233,7 +233,7 @@ class SearchResourceTest {
}
private JsonMockHttpResponse search(String query, Integer page, Integer pageSize) throws URISyntaxException, UnsupportedEncodingException {
String uri = "/v2/search?q=" + URLEncoder.encode(query, "UTF-8");
String uri = "/v2/search/string?q=" + URLEncoder.encode(query, "UTF-8");
if (page != null) {
uri += "&page=" + page;
}

11
scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java
View File

@@ -61,13 +61,16 @@ class DefaultIndexQueueTest {
@BeforeEach
void createQueue() throws IOException {
directory = new ByteBuffersDirectory();
IndexOpener factory = mock(IndexOpener.class);
when(factory.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
IndexOpener opener = mock(IndexOpener.class);
when(opener.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, config);
});
SearchEngine engine = new LuceneSearchEngine(factory, new DocumentConverter(), queryBuilderFactory);
SearchableTypeResolver resolver = new SearchableTypeResolver(Account.class, IndexedNumber.class);
LuceneIndexFactory indexFactory = new LuceneIndexFactory(resolver, opener);
SearchEngine engine = new LuceneSearchEngine(indexFactory, queryBuilderFactory);
queue = new DefaultIndexQueue(engine);
}
@@ -111,6 +114,7 @@ class DefaultIndexQueueTest {
}
@Value
@IndexedType
public static class Account {
@Indexed
String username;
@@ -121,6 +125,7 @@ class DefaultIndexQueueTest {
}
@Value
@IndexedType
public static class IndexedNumber {
@Indexed
int value;

9
scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java
View File

@@ -82,7 +82,7 @@ class LuceneIndexTest {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
}
assertHits(UID, "one/" + Storable.class.getName(), 1);
assertHits(UID, "one/storable", 1);
}
@Test
@@ -109,7 +109,7 @@ class LuceneIndexTest {
index.store(ONE, null, new Storable("Some other text"));
}
assertHits(TYPE, Storable.class.getName(), 1);
assertHits(TYPE, "storable", 1);
}
@Test
@@ -214,7 +214,8 @@ class LuceneIndexTest {
}
private LuceneIndex createIndex() throws IOException {
return new LuceneIndex(new DocumentConverter(), createWriter());
SearchableTypeResolver resolver = new SearchableTypeResolver(Storable.class, OtherStorable.class);
return new LuceneIndex(resolver, createWriter());
}
private IndexWriter createWriter() throws IOException {
@@ -224,12 +225,14 @@ class LuceneIndexTest {
}
@Value
@IndexedType
private static class Storable {
@Indexed
String value;
}
@Value
@IndexedType
private static class OtherStorable {
@Indexed
String value;

29
scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java
View File

@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.Getter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -153,7 +154,7 @@ class LuceneQueryBuilderTest {
try (IndexWriter writer = writer()) {
writer.addDocument(personDoc("Dent"));
}
assertThrows(QueryParseException.class, () -> query(String.class, ":~:~"));
assertThrows(QueryParseException.class, () -> query(InetOrgPerson.class, ":~:~"));
}
@Test
@@ -247,8 +248,9 @@ class LuceneQueryBuilderTest {
QueryResult result;
try (DirectoryReader reader = DirectoryReader.open(directory)) {
when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, "default", new StandardAnalyzer()
opener, resolver, "default", new StandardAnalyzer()
);
result = builder.repository("cde").execute(Simple.class, "content:awesome");
}
@@ -480,8 +482,9 @@ class LuceneQueryBuilderTest {
private QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader);
SearchableTypeResolver resolver = new SearchableTypeResolver(type);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, "default", new StandardAnalyzer()
opener, resolver, "default", new StandardAnalyzer()
);
if (start != null) {
builder.start(start);
@@ -502,14 +505,14 @@ class LuceneQueryBuilderTest {
private Document simpleDoc(String content) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
return document;
}
private Document permissionDoc(String content, String permission) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES));
return document;
}
@@ -517,7 +520,7 @@ class LuceneQueryBuilderTest {
private Document repositoryDoc(String content, String repository) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES));
return document;
}
@@ -529,14 +532,14 @@ class LuceneQueryBuilderTest {
document.add(new TextField("displayName", displayName, Field.Store.YES));
document.add(new TextField("carLicense", carLicense, Field.Store.YES));
document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, InetOrgPerson.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES));
return document;
}
private Document personDoc(String lastName) {
Document document = new Document();
document.add(new TextField("lastName", lastName, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Person.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES));
return document;
}
@@ -549,10 +552,12 @@ class LuceneQueryBuilderTest {
document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES));
document.add(new LongPoint("instantValue", instantValue.toEpochMilli()));
document.add(new StoredField("instantValue", instantValue.toEpochMilli()));
document.add(new StringField(FieldNames.TYPE, Types.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES));
return document;
}
@Getter
@IndexedType
static class Types {
@Indexed
@@ -566,12 +571,16 @@ class LuceneQueryBuilderTest {
}
@Getter
@IndexedType
static class Person {
@Indexed(defaultQuery = true)
private String lastName;
}
@Getter
@IndexedType
static class InetOrgPerson extends Person {
@Indexed(defaultQuery = true, boost = 2f)
@@ -584,6 +593,8 @@ class LuceneQueryBuilderTest {
private String carLicense;
}
@Getter
@IndexedType
static class Simple {
@Indexed(defaultQuery = true)
private String content;

58
scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java → scm-webapp/src/test/java/sonia/scm/search/TypeConvertersTest.java
View File

@@ -35,62 +35,61 @@ import org.apache.lucene.index.IndexableFieldType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import javax.annotation.Nonnull;
import java.time.Instant;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
class DocumentConverterTest {
private DocumentConverter documentConverter;
@BeforeEach
void prepare() {
documentConverter = new DocumentConverter();
}
class TypeConvertersTest {
@Test
void shouldConvertPersonToDocument() {
Person person = new Person("Arthur", "Dent");
Document document = documentConverter.convert(person);
Document document = convert(person);
assertThat(document.getField("firstName").stringValue()).isEqualTo("Arthur");
assertThat(document.getField("lastName").stringValue()).isEqualTo("Dent");
}
@Nonnull
private Document convert(Object object) {
return TypeConverters.create(object.getClass()).convert(object);
}
@Test
void shouldUseNameFromAnnotation() {
Document document = documentConverter.convert(new ParamSample());
Document document = convert(new ParamSample());
assertThat(document.getField("username").stringValue()).isEqualTo("dent");
}
@Test
void shouldBeIndexedAsTextFieldByDefault() {
Document document = documentConverter.convert(new ParamSample());
Document document = convert(new ParamSample());
assertThat(document.getField("username")).isInstanceOf(TextField.class);
}
@Test
void shouldBeIndexedAsStringField() {
Document document = documentConverter.convert(new ParamSample());
Document document = convert(new ParamSample());
assertThat(document.getField("searchable")).isInstanceOf(StringField.class);
}
@Test
void shouldBeIndexedAsStoredField() {
Document document = documentConverter.convert(new ParamSample());
Document document = convert(new ParamSample());
assertThat(document.getField("storedOnly")).isInstanceOf(StoredField.class);
}
@Test
void shouldIgnoreNonIndexedFields() {
Document document = documentConverter.convert(new ParamSample());
Document document = convert(new ParamSample());
assertThat(document.getField("notIndexed")).isNull();
}
@@ -99,7 +98,7 @@ class DocumentConverterTest {
void shouldSupportInheritance() {
Account account = new Account("Arthur", "Dent", "arthur@hitchhiker.com");
Document document = documentConverter.convert(account);
Document document = convert(account);
assertThat(document.getField("firstName")).isNotNull();
assertThat(document.getField("lastName")).isNotNull();
@@ -109,18 +108,18 @@ class DocumentConverterTest {
@Test
void shouldFailWithoutGetter() {
WithoutGetter withoutGetter = new WithoutGetter();
assertThrows(NonReadableFieldException.class, () -> documentConverter.convert(withoutGetter));
assertThrows(NonReadableFieldException.class, () -> convert(withoutGetter));
}
@Test
void shouldFailOnUnsupportedFieldType() {
UnsupportedFieldType unsupportedFieldType = new UnsupportedFieldType();
assertThrows(UnsupportedTypeOfFieldException.class, () -> documentConverter.convert(unsupportedFieldType));
assertThrows(UnsupportedTypeOfFieldException.class, () -> convert(unsupportedFieldType));
}
@Test
void shouldStoreLongFieldsAsPointAndStoredByDefault() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
assertPointField(document, "longType",
field -> assertThat(field.numericValue().longValue()).isEqualTo(42L)
@@ -129,7 +128,7 @@ class DocumentConverterTest {
@Test
void shouldStoreLongFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyLongType");
assertThat(field).isInstanceOf(StoredField.class);
@@ -138,7 +137,7 @@ class DocumentConverterTest {
@Test
void shouldStoreIntegerFieldsAsPointAndStoredByDefault() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
assertPointField(document, "intType",
field -> assertThat(field.numericValue().intValue()).isEqualTo(42)
@@ -147,7 +146,7 @@ class DocumentConverterTest {
@Test
void shouldStoreIntegerFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyIntegerType");
assertThat(field).isInstanceOf(StoredField.class);
@@ -156,7 +155,7 @@ class DocumentConverterTest {
@Test
void shouldStoreBooleanFieldsAsStringField() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
IndexableField field = document.getField("boolType");
assertThat(field).isInstanceOf(StringField.class);
@@ -166,7 +165,7 @@ class DocumentConverterTest {
@Test
void shouldStoreBooleanFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
Document document = convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyBoolType");
assertThat(field).isInstanceOf(StoredField.class);
@@ -176,7 +175,7 @@ class DocumentConverterTest {
@Test
void shouldStoreInstantFieldsAsPointAndStoredByDefault() {
Instant now = Instant.now();
Document document = documentConverter.convert(new DateTypes(now));
Document document = convert(new DateTypes(now));
assertPointField(document, "instant",
field -> assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli())
@@ -186,7 +185,7 @@ class DocumentConverterTest {
@Test
void shouldStoreInstantFieldAsStored() {
Instant now = Instant.now();
Document document = documentConverter.convert(new DateTypes(now));
Document document = convert(new DateTypes(now));
IndexableField field = document.getField("storedOnlyInstant");
assertThat(field).isInstanceOf(StoredField.class);
@@ -195,7 +194,7 @@ class DocumentConverterTest {
@Test
void shouldCreateNoFieldForNullValues() {
Document document = documentConverter.convert(new Person("Trillian", null));
Document document = convert(new Person("Trillian", null));
assertThat(document.getField("firstName")).isNotNull();
assertThat(document.getField("lastName")).isNull();
@@ -210,6 +209,7 @@ class DocumentConverterTest {
}
@Getter
@IndexedType
@AllArgsConstructor
public static class Person {
@Indexed
@@ -219,6 +219,7 @@ class DocumentConverterTest {
}
@Getter
@IndexedType
public static class Account extends Person {
@Indexed
private String mail;
@@ -230,6 +231,7 @@ class DocumentConverterTest {
}
@Getter
@IndexedType
public static class ParamSample {
@Indexed(name = "username")
private final String name = "dent";
@@ -243,18 +245,21 @@ class DocumentConverterTest {
private final String notIndexed = "--";
}
@IndexedType
public static class WithoutGetter {
@Indexed
private final String value = "one";
}
@Getter
@IndexedType
public static class UnsupportedFieldType {
@Indexed
private final Object value = "one";
}
@Getter
@IndexedType
public static class SupportedTypes {
@Indexed
private final Long longType = 42L;
@@ -273,6 +278,7 @@ class DocumentConverterTest {
}
@Getter
@IndexedType
private static class DateTypes {
@Indexed
private final Instant instant;