Add search engine and quick search for repositories (#1727)

Add a powerful search engine based on lucene to the scm-manager api.
The api can be used to index objects, simply by annotating them and add them to an index.
The first indexed object is the repository which could queried by quick search in the header.
This commit is contained in:
Sebastian Sdorra
2021-07-14 11:49:38 +02:00
committed by GitHub
parent ce4b869a7a
commit e321133ff7
88 changed files with 6052 additions and 25 deletions

View File

@@ -0,0 +1,4 @@
- type: Added
description: API to index and query objects ([#1727](https://github.com/scm-manager/scm-manager/pull/1727))
- type: Added
description: Quick search for repositories ([#1727](https://github.com/scm-manager/scm-manager/pull/1727))

View File

@@ -15,6 +15,7 @@ ext {
jjwtVersion = '0.11.2'
bouncycastleVersion = '1.67'
jettyVersion = '9.4.35.v20201120'
luceneVersion = '8.9.0'
junitJupiterVersion = '5.7.0'
hamcrestVersion = '2.1'
@@ -148,6 +149,12 @@ ext {
jettyJmx: "org.eclipse.jetty:jetty-jmx:${jettyVersion}",
jettyClient: "org.eclipse.jetty:jetty-client:${jettyVersion}",
// search
luceneCore: "org.apache.lucene:lucene-core:${luceneVersion}",
luceneQueryParser: "org.apache.lucene:lucene-queryparser:${luceneVersion}",
luceneHighlighter: "org.apache.lucene:lucene-highlighter:${luceneVersion}",
luceneAnalyzersCommon: "org.apache.lucene:lucene-analyzers-common:${luceneVersion}",
// tests
junitJupiterApi: "org.junit.jupiter:junit-jupiter-api:${junitJupiterVersion}",
junitJupiterParams: "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}",

View File

@@ -31,6 +31,7 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import sonia.scm.BasicPropertiesAware;
import sonia.scm.ModelObject;
import sonia.scm.search.Indexed;
import sonia.scm.util.Util;
import sonia.scm.util.ValidationUtil;
@@ -65,21 +66,28 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per
private static final long serialVersionUID = 3486560714961909711L;
private String contact;
private Long creationDate;
private String id;
@Indexed(defaultQuery = true, boost = 1.25f)
private String namespace;
@Indexed(defaultQuery = true, boost = 1.5f)
private String name;
@Indexed(type = Indexed.Type.SEARCHABLE)
private String type;
@Indexed(defaultQuery = true, highlighted = true)
private String description;
private String contact;
@Indexed
private Long creationDate;
@Indexed
private Long lastModified;
@XmlTransient
private List<HealthCheckFailure> healthCheckFailures;
private String id;
private Long lastModified;
private String namespace;
private String name;
@XmlElement(name = "permission")
private Set<RepositoryPermission> permissions = new HashSet<>();
private String type;
private boolean archived;
/**
* Constructs a new {@link Repository}.
* This constructor is used by JAXB.
@@ -162,10 +170,9 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per
* @return {@link List} of {@link HealthCheckFailure}s
* @since 1.36
*/
@SuppressWarnings("unchecked")
public List<HealthCheckFailure> getHealthCheckFailures() {
if (healthCheckFailures == null) {
healthCheckFailures = Collections.EMPTY_LIST;
healthCheckFailures = Collections.emptyList();
}
return healthCheckFailures;

View File

@@ -0,0 +1,95 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Value;
import java.util.Map;
/**
* Represents an object which matched the search query.
*
* @since 2.21.0
*/
@Beta
@Value
public class Hit {
/**
* Id of the matched object.
*/
String id;
/**
* The score describes how good the match was.
*/
float score;
/**
* Fields of the matched object.
* Key of the map is the name of the field and the value is either a {@link ValueField} or a {@link HighlightedField}.
*/
Map<String, Field> fields;
/**
* Base class of hit field types.
*/
@Getter
@AllArgsConstructor(access = AccessLevel.PRIVATE)
public abstract static class Field {
boolean highlighted;
}
/**
* A field holding a complete value.
*/
@Getter
public static class ValueField extends Field {
Object value;
public ValueField(Object value) {
super(false);
this.value = value;
}
}
/**
* A field which consists of fragments that contain a match of the search query.
*/
@Getter
public static class HighlightedField extends Field {
String[] fragments;
public HighlightedField(String[] fragments) {
super(true);
this.fragments = fragments;
}
}
}

View File

@@ -0,0 +1,158 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import sonia.scm.ModelObject;
import sonia.scm.repository.Repository;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Optional;
/**
* Describes the id of an indexed object.
*
* @since 2.21.0
*/
@Beta
@ToString
@EqualsAndHashCode
public final class Id {
private final String value;
private final String repository;
private Id(@Nonnull String value, @Nullable String repository) {
this.value = value;
this.repository = repository;
}
/**
* Returns the string representation of the id without the repository part.
*
* @return string representation without repository.
*/
public String getValue() {
return value;
}
/**
* Returns the repository id part of the id or an empty optional if the id does not belong to a repository.
* @return repository id or empty
*/
public Optional<String> getRepository() {
return Optional.ofNullable(repository);
}
/**
* Creates the id with the id of the given repository.
* @param repository repository
* @return id with repository id
*/
public Id withRepository(@Nonnull Repository repository) {
checkRepository(repository);
return withRepository(repository.getId());
}
/**
* Creates the id with the given repository id.
* @param repository repository id
* @return id with repository id
*/
public Id withRepository(@Nonnull String repository) {
checkRepository(repository);
return new Id(value, repository);
}
/**
* Returns the string representation of the id including the repository.
* @return string representation
*/
public String asString() {
if (repository != null) {
return value + "/" + repository;
}
return value;
}
/**
* Creates a new id.
*
* @param value primary value of the id
* @param others additional values which should be part of the id
*
* @return new id
*/
public static Id of(@Nonnull String value, String... others) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "primary value is required");
String idValue = value;
if (others.length > 0) {
idValue += ":" + Joiner.on(':').join(others);
}
return new Id(idValue, null);
}
/**
* Creates a new id for the given repository.
*
* @param repository repository
* @return id of repository
*/
public static Id of(@Nonnull Repository repository) {
checkRepository(repository);
String id = repository.getId();
checkRepository(id);
return new Id(id, id);
}
/**
* Creates a new id for the given model object.
* @param object model object
* @param others additional values which should be part of the id
* @return new id from model object
*/
public static Id of(@Nonnull ModelObject object, String... others) {
checkObject(object);
return of(object.getId(), others);
}
private static void checkRepository(Repository repository) {
Preconditions.checkArgument(repository != null, "repository is required");
}
private static void checkRepository(String repository) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(repository), "repository id is required");
}
private static void checkObject(@Nonnull ModelObject object) {
Preconditions.checkArgument(object != null, "object is required");
}
}

View File

@@ -0,0 +1,75 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* Can be used to index objects for full text searches.
*
* @since 2.21.0
*/
@Beta
public interface Index extends AutoCloseable {
/**
* Store the given object in the index.
* All fields of the object annotated with {@link Indexed} will be stored in the index.
*
* @param id identifier of the object in the index
* @param permission Shiro permission string representing the required permission to see this object as a result
* @param object object to index
*
* @see Indexed
*/
void store(Id id, String permission, Object object);
/**
* Delete the object with the given id and type from index.
*
* @param id id of object
* @param type type of object
*/
void delete(Id id, Class<?> type);
/**
* Delete all objects which are related the given repository from index.
*
* @param repositoryId id of repository
*/
void deleteByRepository(String repositoryId);
/**
* Delete all objects with the given type from index.
* @param type type of objects
*/
void deleteByType(Class<?> type);
/**
* Close index and commit changes.
*/
@Override
void close();
}

View File

@@ -0,0 +1,57 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.Data;
import sonia.scm.xml.XmlInstantAdapter;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.time.Instant;
/**
* A marker keeping track of when and with which model version an object type was last indexed.
* @since 2.21
*/
@Beta
@Data
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class IndexLog {
private int version = 1;
@XmlJavaTypeAdapter(XmlInstantAdapter.class)
private Instant date = Instant.now();
public IndexLog() {
}
public IndexLog(int version) {
this.version = version;
}
}

View File

@@ -0,0 +1,61 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import java.util.Optional;
/**
* Can be used to mark when a type of object was last indexed and with which version.
* This is useful to detect and mark if a bootstrap index was created for the kind of object
* or if the way how an object is indexed has changed.
*
* @since 2.21.0
*/
@Beta
public interface IndexLogStore {
/**
* Log index and version of a type which is now indexed.
*
* @param index name of index
* @param type type which was indexed
* @param version model version
*/
void log(String index, Class<?> type, int version);
/**
* Returns version and date of the indexed type or an empty object,
* if the object was not indexed at all.
*
* @param index name if index
* @param type type of object
*
* @return log entry or empty
*/
Optional<IndexLog> get(String index, Class<?> type);
}

View File

@@ -0,0 +1,43 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* Names of predefined indexes.
* @since 2.21.0
*/
@Beta
public final class IndexNames {
/**
* The default index.
*/
public static final String DEFAULT = "_default";
private IndexNames() {
}
}

View File

@@ -0,0 +1,101 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import java.util.Locale;
/**
* Options to configure how things are indexed and searched.
*
* @since 2.21.0
*/
@Beta
public class IndexOptions {
private final Type type;
private final Locale locale;
private IndexOptions(Type type, Locale locale) {
this.type = type;
this.locale = locale;
}
/**
* Returns the type of the index.
* @return type of index
*/
public Type getType() {
return type;
}
/**
* Returns the locale of the index content.
*
* @return locale of index content
*/
public Locale getLocale() {
return locale;
}
/**
* Returns the default index options which should match most of the use cases.
*
* @return default index options
*/
public static IndexOptions defaults() {
return new IndexOptions(Type.GENERIC, Locale.ENGLISH);
}
/**
* Returns index options for a specific language.
* This options should be used if the content is written in a specific natural language.
*
* @param locale natural language of content
*
* @return options for content in natural language
*/
public static IndexOptions naturalLanguage(Locale locale) {
return new IndexOptions(Type.NATURAL_LANGUAGE, locale);
}
/**
* Type of indexing.
*/
public enum Type {
/**
* Not specified content.
*/
GENERIC,
/**
* Content in natural language.
*/
NATURAL_LANGUAGE;
}
}

View File

@@ -0,0 +1,60 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* Queue the work of indexing.
* An index can't be opened in parallel, so the queue coordinates the work of indexing in an asynchronous manner.
* {@link IndexQueue} should be used most of the time to index content.
*
* @since 2.21.0
*/
@Beta
public interface IndexQueue {
/**
* Returns an index which queues every change to the content.
*
* @param name name of index
* @param indexOptions options for the index
*
* @return index which queues changes
*/
Index getQueuedIndex(String name, IndexOptions indexOptions);
/**
* Returns an index which with default options which queues every change to the content.
* @param name name of index
*
* @return index with default options which queues changes
* @see IndexOptions#defaults()
*/
default Index getQueuedIndex(String name) {
return getQueuedIndex(name, IndexOptions.defaults());
}
}

View File

@@ -0,0 +1,144 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Mark a field which should be indexed.
* @since 2.21.0
*/
@Beta
@Documented
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Indexed {
/**
* Name of the field.
* If not set the name of the annotated field is used.
*
* @return name of field
*/
String name() default "";
/**
* Describes how the field is indexed.
*
* @return type of indexing
*/
Type type() default Type.TOKENIZED;
/**
* {@code true} if this field should be part of default query for this type of object.
*
* @return {@code true} if field is part of default query
*/
boolean defaultQuery() default false;
/**
* Boost the object if the searched query matches this field.
* Greater than one brings the object further up in the search results.
* Smaller than one devalues the object in the search results.
*
* @return boost score
*/
float boost() default 1f;
/**
* {@code true} to search the field value for matches and returns fragments with those matches instead of the whole value.
*
* @return {@code true} to return matched fragments
*/
boolean highlighted() default false;
/**
* Describes how the field is indexed.
*/
enum Type {
/**
* The value of the field is analyzed and split into tokens, which allows searches for parts of the value.
* Tokenization only works for string values. If a field with another type is marked as tokenized,
* the field is indexed as if it was marked as {@link #SEARCHABLE}.
*/
TOKENIZED(true, true, true),
/**
* The value can only be searched as a whole.
* Numeric fields can also be search as part of a range,
* but strings are only found if the query contains the whole field value.
*/
SEARCHABLE(false, true, true),
/**
* Value of the field cannot be searched for, but is returned in the result.
*/
STORED_ONLY(false, false, true);
private final boolean tokenized;
private final boolean searchable;
private final boolean stored;
Type(boolean tokenized, boolean searchable, boolean stored) {
this.tokenized = tokenized;
this.searchable = searchable;
this.stored = stored;
}
/**
* Returns {@code true} if the field is tokenized.
*
* @return {@code true} if tokenized
* @see #TOKENIZED
*/
public boolean isTokenized() {
return tokenized;
}
/**
* Returns {@code true} if the field is searchable.
*
* @return {@code true} if searchable
* @see #SEARCHABLE
*/
public boolean isSearchable() {
return searchable;
}
/**
* Returns {@code true} if the field is stored.
* @return {@code true} if stored
*/
public boolean isStored() {
return stored;
}
}
}

View File

@@ -0,0 +1,41 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* Exception is thrown if a best guess query is executed
* and the search type does not contain a field which is marked for default query.
*
* @see Indexed#defaultQuery()
* @since 2.21.0
*/
@Beta
public class NoDefaultQueryFieldsFoundException extends SearchEngineException {
public NoDefaultQueryFieldsFoundException(Class<?> type) {
super("no default query fields defined for " + type);
}
}

View File

@@ -0,0 +1,113 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.Value;
import sonia.scm.repository.Repository;
import java.util.Optional;
/**
* Build and execute queries against an index.
*
* @since 2.21.0
*/
@Beta
public abstract class QueryBuilder {
private String repositoryId;
private int start = 0;
private int limit = 10;
/**
* Return only results which are related to the given repository.
* @param repository repository
* @return {@code this}
*/
public QueryBuilder repository(Repository repository) {
return repository(repository.getId());
}
/**
* Return only results which are related to the repository with the given id.
* @param repositoryId id of the repository
* @return {@code this}
*/
public QueryBuilder repository(String repositoryId) {
this.repositoryId = repositoryId;
return this;
}
/**
* The result should start at the given number.
* All matching objects before the given start are skipped.
* @param start start of result
* @return {@code this}
*/
public QueryBuilder start(int start) {
this.start = start;
return this;
}
/**
* Defines how many hits are returned.
* @param limit limit of hits
* @return {@code this}
*/
public QueryBuilder limit(int limit) {
this.limit = limit;
return this;
}
/**
* Executes the query and returns the matches.
*
* @param type type of objects which are searched
* @param queryString searched query
* @return result of query
*/
public QueryResult execute(Class<?> type, String queryString){
return execute(new QueryParams(type, repositoryId, queryString, start, limit));
}
protected abstract QueryResult execute(QueryParams queryParams);
/**
* The searched query and all parameters, which belong to the query.
*/
@Value
static class QueryParams {
Class<?> type;
String repositoryId;
String queryString;
int start;
int limit;
public Optional<String> getRepositoryId() {
return Optional.ofNullable(repositoryId);
}
}
}

View File

@@ -0,0 +1,51 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import sonia.scm.BadRequestException;
import static sonia.scm.ContextEntry.ContextBuilder.only;
/**
* Is thrown if a query with invalid syntax was executed.
*
* @since 2.21.0
*/
@Beta
@SuppressWarnings("java:S110") // large inheritance is ok for exceptions
public class QueryParseException extends BadRequestException {
private static final String CODE = "5VScek8Xp1";
public QueryParseException(String query, String message, Exception cause) {
super(only("query", query), message, cause);
}
@Override
public String getCode() {
return CODE;
}
}

View File

@@ -0,0 +1,57 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
import lombok.Value;
import java.util.List;
/**
* Result of a query.
* @since 2.21.0
*/
@Beta
@Value
public class QueryResult {
/**
* Total count of hits, which are matched by the query.
*/
long totalHits;
/**
* Searched type of object.
*/
Class<?> type;
/**
* List of hits found by the query.
* The list contains only those hits which are starting at start and they are limit by the given amount.
* @see QueryBuilder
*/
List<Hit> hits;
}

View File

@@ -0,0 +1,83 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* The {@link SearchEngine} is the main entry point for indexing and searching.
* Note that this is kind of a low level api for indexing.
* For non expert indexing the {@link IndexQueue} should be used.
*
* @see IndexQueue
* @since 2.21.0
*/
@Beta
public interface SearchEngine {
/**
* Returns the index with the given name and the given options.
* The index is created if it does not exist.
* Warning: Be careful, because an index can't be opened multiple times in parallel.
* If you are not sure how you should index your objects, use the {@link IndexQueue}.
*
* @param name name of the index
* @param options index options
* @return existing index or a new one if none exists
*/
Index getOrCreate(String name, IndexOptions options);
/**
* Same as {@link #getOrCreate(String, IndexOptions)} with default options.
*
* @param name name of the index
* @return existing index or a new one if none exists
* @see IndexOptions#defaults()
*/
default Index getOrCreate(String name) {
return getOrCreate(name, IndexOptions.defaults());
}
/**
* Search the index.
* Returns a {@link QueryBuilder} which allows to query the index.
*
* @param name name of the index
* @param options options for searching the index
* @return query builder
*/
QueryBuilder search(String name, IndexOptions options);
/**
* Same as {@link #search(String, IndexOptions)} with default options.
*
* @param name name of the index
* @return query builder
* @see IndexOptions#defaults()
*/
default QueryBuilder search(String name) {
return search(name, IndexOptions.defaults());
}
}

View File

@@ -0,0 +1,44 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.Beta;
/**
* Generic exception which could by thrown by any part of the search engine.
*
* @since 2.21.0
*/
@Beta
public class SearchEngineException extends RuntimeException {
public SearchEngineException(String message) {
super(message);
}
public SearchEngineException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@@ -95,6 +95,8 @@ public class VndMediaType {
public static final String NOTIFICATION_COLLECTION = PREFIX + "notificationCollection" + SUFFIX;
public static final String QUERY_RESULT = PREFIX + "queryResult" + SUFFIX;
private VndMediaType() {
}

View File

@@ -0,0 +1,128 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.junit.jupiter.api.Test;
import sonia.scm.ModelObject;
import sonia.scm.repository.Repository;
import sonia.scm.user.User;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
class IdTest {
@Test
void shouldCreateIdFromPrimary() {
Id id = Id.of("one");
assertThat(id.getValue()).isEqualTo("one");
}
@Test
void shouldCreateIdWithoutRepository() {
Id id = Id.of("one");
assertThat(id.getRepository()).isEmpty();
}
@Test
void shouldFailWithoutPrimaryValue() {
assertThrows(IllegalArgumentException.class, () -> Id.of((String) null));
}
@Test
void shouldFailWithEmptyPrimaryValue() {
assertThrows(IllegalArgumentException.class, () -> Id.of(""));
}
@Test
void shouldCreateCombinedValue() {
Id id = Id.of("one", "two", "three");
assertThat(id.getValue()).isEqualTo("one:two:three");
}
@Test
void shouldAddRepositoryId() {
Id id = Id.of("one").withRepository("4211");
assertThat(id.getRepository()).contains("4211");
}
@Test
void shouldAddRepository() {
Repository repository = new Repository();
repository.setId("4211");
Id id = Id.of("one").withRepository(repository);
assertThat(id.getRepository()).contains("4211");
}
@Test
void shouldCreateIdFromRepository() {
Repository repository = new Repository();
repository.setId("4211");
Id id = Id.of(repository);
assertThat(id.getRepository()).contains("4211");
}
@Test
void shouldFailWithoutRepository() {
Id id = Id.of("one");
assertThrows(IllegalArgumentException.class, () -> id.withRepository((Repository) null));
}
@Test
void shouldFailWithoutRepositoryId() {
Id id = Id.of("one");
assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null));
}
@Test
void shouldFailWithEmptyRepositoryId() {
Id id = Id.of("one");
assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null));
}
@Test
void shouldCreateIdFromModelObject() {
Id id = Id.of(new User("trillian"));
assertThat(id.getValue()).isEqualTo("trillian");
}
@Test
void shouldFailWithoutModelObject() {
assertThrows(IllegalArgumentException.class, () -> Id.of((ModelObject) null));
}
@Test
void shouldReturnSimpleIdAsString() {
Id id = Id.of("one", "two");
assertThat(id.asString()).isEqualTo("one:two");
}
@Test
void shouldReturnIdWithRepositoryAsString() {
Id id = Id.of("one", "two").withRepository("4211");
assertThat(id.asString()).isEqualTo("one:two/4211");
}
}

View File

@@ -56,6 +56,7 @@ export * from "./fileContent";
export * from "./history";
export * from "./contentType";
export * from "./annotations";
export * from "./search";
export { default as ApiProvider } from "./ApiProvider";
export * from "./ApiProvider";

View File

@@ -0,0 +1,56 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import { ApiResult, useRequiredIndexLink } from "./base";
import { QueryResult } from "@scm-manager/ui-types";
import { apiClient } from "./apiclient";
import { createQueryString } from "./utils";
import { useQuery } from "react-query";
export type SearchOptions = {
page?: number;
pageSize?: number;
};
const defaultSearchOptions: SearchOptions = {};
export const useSearch = (query: string, options = defaultSearchOptions): ApiResult<QueryResult> => {
const link = useRequiredIndexLink("search");
const queryParams: Record<string, string> = {};
queryParams.q = query;
if (options.page) {
queryParams.page = options.page.toString();
}
if (options.pageSize) {
queryParams.pageSize = options.pageSize.toString();
}
return useQuery<QueryResult, Error>(
["search", query],
() => apiClient.get(`${link}?${createQueryString(queryParams)}`).then((response) => response.json()),
{
enabled: query.length > 1,
}
);
};

View File

@@ -81,7 +81,7 @@ const FilterInput: FC<Props> = ({ filter, value, testId, placeholder, autoFocus,
autoFocus={autoFocus || false}
/>
<span className="icon is-small is-left">
<i className="fas fa-search" />
<i className="fas fa-filter" />
</span>
</div>
</form>

View File

@@ -0,0 +1,50 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import { HalRepresentation, PagedCollection } from "./hal";
export type ValueField = {
highlighted: false;
value: unknown;
};
export type HighligthedField = {
highlighted: true;
fragments: string[];
};
export type Field = ValueField | HighligthedField;
export type Hit = HalRepresentation & {
score: number;
fields: { [name: string]: Field };
};
export type HitEmbedded = {
hits: Hit[];
};
export type QueryResult = PagedCollection<HitEmbedded> & {
type: string;
};

View File

@@ -70,3 +70,4 @@ export * from "./Notifications";
export * from "./ApiKeys";
export * from "./PublicKeys";
export * from "./GlobalPermissions";
export * from "./Search";

View File

@@ -150,5 +150,12 @@
"d_plural": "{{count}} Tage",
"w": "{{count}} Woche",
"w_plural": "{{count}} Wochen"
},
"search": {
"quickSearch": {
"resultHeading": "Top-Ergebnisse Repositories",
"parseError": "Der Suchstring is ungültig",
"noResults": "Es konnten keine Repositories gefunden werden"
}
}
}

View File

@@ -26,7 +26,7 @@
}
},
"overview": {
"searchGroup": "Gruppe suchen"
"filterGroup": "Gruppen filtern"
},
"add-group": {
"title": "Gruppe erstellen",

View File

@@ -59,7 +59,7 @@
"noRepositories": "Keine Repositories gefunden.",
"invalidNamespace": "Keine Repositories gefunden. Möglicherweise existiert der ausgewählte Namespace nicht.",
"createButton": "Repository hinzufügen",
"searchRepository": "Repository suchen",
"filterRepositories": "Repositories filtern",
"allNamespaces": "Alle Namespaces"
},
"create": {

View File

@@ -31,7 +31,7 @@
"createButton": "Benutzer erstellen"
},
"overview": {
"searchUser": "Benutzer suchen"
"filterUser": "Benutzer filtern"
},
"singleUser": {
"errorTitle": "Fehler",

View File

@@ -151,5 +151,12 @@
"d_plural": "{{count}} days",
"w": "{{count}} week",
"w_plural": "{{count}} weeks"
},
"search": {
"quickSearch": {
"resultHeading": "Top repository results",
"parseError": "Failed to parse query",
"noResults": "Could not find matching repository"
}
}
}

View File

@@ -26,7 +26,7 @@
}
},
"overview": {
"searchGroup": "Search group"
"filterGroup": "Filter groups"
},
"add-group": {
"title": "Create Group",

View File

@@ -59,7 +59,7 @@
"noRepositories": "No repositories found.",
"invalidNamespace": "No repositories found. It's likely that the selected namespace does not exist.",
"createButton": "Add Repository",
"searchRepository": "Search repository",
"filterRepositories": "Filter repositories",
"allNamespaces": "All namespaces"
},
"create": {

View File

@@ -45,7 +45,7 @@
}
},
"overview": {
"searchUser": "Search user"
"filterUser": "Filter users"
},
"createUser": {
"title": "Create User",

View File

@@ -27,6 +27,7 @@ import classNames from "classnames";
import styled from "styled-components";
import { devices, Logo, PrimaryNavigation } from "@scm-manager/ui-components";
import Notifications from "./Notifications";
import OmniSearch from "./OmniSearch";
import LogoutButton from "./LogoutButton";
import LoginButton from "./LoginButton";
@@ -44,6 +45,7 @@ const StyledNavBar = styled.nav`
position: absolute;
top: 0;
left: 52px;
flex-direction: row-reverse;
}
}
@@ -142,6 +144,7 @@ const NavigationBar: FC<Props> = ({ links }) => {
</button>
</div>
<div className="is-active navbar-header-actions">
<OmniSearch links={links} />
<Notifications className="navbar-item" />
</div>
<StyledMenuBar className={classNames("navbar-menu", { "is-active": burgerActive })}>

View File

@@ -60,13 +60,18 @@ const DropDownMenu = styled.div`
min-width: 35rem;
@media screen and (max-width: ${devices.mobile.width}px) {
min-width: 25rem;
min-width: 20rem;
}
@media screen and (max-width: ${devices.desktop.width}px) {
margin-right: 1rem;
}
@media screen and (min-width: ${devices.desktop.width}px) {
right: 0;
left: auto;
}
&:before {
position: absolute;
content: "";
@@ -338,15 +343,12 @@ const Notifications: FC<NotificationProps> = ({ className }) => {
return () => window.removeEventListener("click", close);
}, []);
const isMobileView = window.matchMedia(`(max-width: ${devices.desktop.width - 1}px)`).matches;
return (
<>
<NotificationSubscription notifications={notifications} remove={remove} />
<div
className={classNames(
"dropdown",
isMobileView ? "is-left" : "is-right",
"is-hoverable",
{
"is-active": open,

View File

@@ -0,0 +1,254 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import React, { FC, KeyboardEvent, MouseEvent, useCallback, useState, useEffect } from "react";
import { Hit, Links, ValueField } from "@scm-manager/ui-types";
import styled from "styled-components";
import { BackendError, useSearch } from "@scm-manager/ui-api";
import classNames from "classnames";
import { Link, useHistory } from "react-router-dom";
import { useTranslation } from "react-i18next";
import { ErrorNotification, Notification } from "@scm-manager/ui-components";
const Field = styled.div`
margin-bottom: 0 !important;
`;
const Input = styled.input`
border-radius: 4px !important;
`;
type Props = {
links: Links;
};
const namespaceAndName = (hit: Hit) => {
const namespace = (hit.fields["namespace"] as ValueField).value as string;
const name = (hit.fields["name"] as ValueField).value as string;
return `${namespace}/${name}`;
};
type HitsProps = {
hits: Hit[];
index: number;
clear: () => void;
};
const QuickSearchNotification: FC = ({ children }) => <div className="dropdown-content p-4">{children}</div>;
const EmptyHits = () => {
const [t] = useTranslation("commons");
return (
<QuickSearchNotification>
<Notification type="info">{t("search.quickSearch.noResults")}</Notification>
</QuickSearchNotification>
);
};
type ErrorProps = {
error: Error;
};
const ParseErrorNotification: FC = () => {
const [t] = useTranslation("commons");
// TODO add link to query syntax page/modal
return (
<QuickSearchNotification>
<Notification type="warning">{t("search.quickSearch.parseError")}</Notification>
</QuickSearchNotification>
);
};
const isBackendError = (error: Error | BackendError): error is BackendError => {
return (error as BackendError).errorCode !== undefined;
};
const SearchErrorNotification: FC<ErrorProps> = ({ error }) => {
// 5VScek8Xp1 is the id of sonia.scm.search.QueryParseException
if (isBackendError(error) && error.errorCode === "5VScek8Xp1") {
return <ParseErrorNotification />;
}
return (
<QuickSearchNotification>
<ErrorNotification error={error} />
</QuickSearchNotification>
);
};
const ResultHeading = styled.div`
border-bottom: 1px solid lightgray;
margin: 0 0.5rem;
padding: 0.375rem 0.5rem;
`;
const Hits: FC<HitsProps> = ({ hits, index, clear }) => {
const id = useCallback(namespaceAndName, [hits]);
const [t] = useTranslation("commons");
if (hits.length === 0) {
return <EmptyHits />;
}
return (
<div className="dropdown-content">
<ResultHeading className="dropdown-item">{t("search.quickSearch.resultHeading")}</ResultHeading>
{hits.map((hit, idx) => (
<div key={id(hit)} onMouseDown={(e) => e.preventDefault()} onClick={clear}>
<Link
className={classNames("dropdown-item", "has-text-weight-medium", { "is-active": idx === index })}
to={`/repo/${id(hit)}`}
>
{id(hit)}
</Link>
</div>
))}
</div>
);
};
const useKeyBoardNavigation = (clear: () => void, hits?: Array<Hit>) => {
const [index, setIndex] = useState(-1);
const history = useHistory();
useEffect(() => {
setIndex(-1);
}, [hits]);
const onKeyDown = (e: KeyboardEvent<HTMLInputElement>) => {
// We use e.which, because ie 11 does not support e.code
// https://caniuse.com/keyboardevent-code
switch (e.which) {
case 40: // e.code: ArrowDown
if (hits) {
setIndex((idx) => {
if (idx + 1 < hits.length) {
return idx + 1;
}
return idx;
});
}
break;
case 38: // e.code: ArrowUp
if (hits) {
setIndex((idx) => {
if (idx > 0) {
return idx - 1;
}
return idx;
});
}
break;
case 13: // e.code: Enter
if (hits && index >= 0) {
const hit = hits[index];
history.push(`/repo/${namespaceAndName(hit)}`);
clear();
}
break;
case 27: // e.code: Escape
clear();
break;
}
};
return {
onKeyDown,
index,
};
};
const useDebounce = (value: string, delay: number) => {
const [debouncedValue, setDebouncedValue] = useState(value);
useEffect(() => {
const handler = setTimeout(() => {
setDebouncedValue(value);
}, delay);
return () => {
clearTimeout(handler);
};
}, [value, delay]);
return debouncedValue;
};
const useShowResultsOnFocus = () => {
const [showResults, setShowResults] = useState(false);
return {
showResults,
onClick: (e: MouseEvent<HTMLInputElement>) => e.stopPropagation(),
onFocus: () => setShowResults(true),
onBlur: () => setShowResults(false)
};
};
const OmniSearch: FC = () => {
const [query, setQuery] = useState("");
const debouncedQuery = useDebounce(query, 250);
const { data, isLoading, error } = useSearch(debouncedQuery, { pageSize: 5 });
const { showResults, ...handlers } = useShowResultsOnFocus();
const clearQuery = () => {
setQuery("");
};
const { onKeyDown, index } = useKeyBoardNavigation(clearQuery, data?._embedded.hits);
return (
<Field className="navbar-item field">
<div
className={classNames("control", "has-icons-right", {
"is-loading": isLoading,
})}
>
<div className={classNames("dropdown", { "is-active": (!!data || error) && showResults })}>
<div className="dropdown-trigger">
<Input
className="input is-small"
type="text"
placeholder="Search ..."
onChange={(e) => setQuery(e.target.value)}
onKeyDown={onKeyDown}
value={query}
{...handlers}
/>
{isLoading ? null : (
<span className="icon is-right">
<i className="fas fa-search" />
</span>
)}
</div>
<div className="dropdown-menu">
{error ? <SearchErrorNotification error={error} /> : null}
{!error && data ? <Hits clear={clearQuery} index={index} hits={data._embedded.hits} /> : null}
</div>
</div>
</div>
</Field>
);
};
const OmniSearchGuard: FC<Props> = ({ links }) => {
if (!links.search) {
return null;
}
return <OmniSearch />;
};
export default OmniSearchGuard;

View File

@@ -72,7 +72,7 @@ const Groups: FC = () => {
showCreateButton={canCreateGroups}
link="groups"
label={t("create-group-button.label")}
searchPlaceholder={t("overview.searchGroup")}
searchPlaceholder={t("overview.filterGroup")}
/>
</PageActions>
</Page>

View File

@@ -153,7 +153,7 @@ const Overview: FC = () => {
createLink="/repos/create/"
label={t("overview.createButton")}
testId="repository-overview"
searchPlaceholder={t("overview.searchRepository")}
searchPlaceholder={t("overview.filterRepositories")}
/>
) : null}
</PageActions>

View File

@@ -79,7 +79,7 @@ const Users: FC = () => {
showCreateButton={canAddUsers}
link="users"
label={t("users.createButton")}
searchPlaceholder={t("overview.searchUser")}
searchPlaceholder={t("overview.filterUser")}
/>
</PageActions>
</Page>

View File

@@ -120,6 +120,11 @@ dependencies {
// metrics
implementation libraries.micrometerExtra
implementation libraries.luceneCore
implementation libraries.luceneQueryParser
implementation libraries.luceneHighlighter
implementation libraries.luceneAnalyzersCommon
// lombok
compileOnly libraries.lombok
testCompileOnly libraries.lombok

View File

@@ -21,7 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
@@ -31,6 +31,7 @@ import lombok.Getter;
import lombok.Setter;
@Getter @Setter
@SuppressWarnings("squid:S2160") // we do not need equals for dto
class CollectionDto extends HalRepresentation {
private int page;

View File

@@ -0,0 +1,49 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.HalRepresentation;
import de.otto.edison.hal.Links;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import sonia.scm.search.Hit;
import java.util.Map;
@Getter
@Setter
@NoArgsConstructor
@SuppressWarnings("java:S2160") // we do not need this for dto
public class HitDto extends HalRepresentation {
private float score;
private Map<String, Hit.Field> fields;
public HitDto(Links links, Embedded embedded) {
super(links, embedded);
}
}

View File

@@ -131,6 +131,8 @@ public class IndexDtoGenerator extends HalAppenderMapper {
builder.single(link("namespaceStrategies", resourceLinks.namespaceStrategies().self()));
builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self()));
builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}")));
builder.single(link("search", resourceLinks.search().search()));
} else {
builder.single(link("login", resourceLinks.authentication().jsonLogin()));
}

View File

@@ -91,5 +91,7 @@ public class MapperModule extends AbstractModule {
bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class));
bind(RepositoryImportDtoToRepositoryImportParametersMapper.class).to(Mappers.getMapperClass(RepositoryImportDtoToRepositoryImportParametersMapper.class));
bind(QueryResultMapper.class).to(Mappers.getMapperClass(QueryResultMapper.class));
}
}

View File

@@ -0,0 +1,42 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.Links;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@SuppressWarnings("squid:S2160") // we do not need equals for dto
public class QueryResultDto extends CollectionDto {
private Class<?> type;
QueryResultDto(Links links, Embedded embedded) {
super(links, embedded);
}
}

View File

@@ -0,0 +1,120 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import com.damnhandy.uri.template.UriTemplate;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.Links;
import de.otto.edison.hal.paging.NumberedPaging;
import de.otto.edison.hal.paging.PagingRel;
import org.mapstruct.AfterMapping;
import org.mapstruct.Context;
import org.mapstruct.Mapper;
import org.mapstruct.MappingTarget;
import org.mapstruct.ObjectFactory;
import sonia.scm.search.Hit;
import sonia.scm.search.QueryResult;
import sonia.scm.web.EdisonHalAppender;
import javax.annotation.Nonnull;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
import static com.damnhandy.uri.template.UriTemplate.fromTemplate;
import static de.otto.edison.hal.Links.linkingTo;
import static de.otto.edison.hal.paging.NumberedPaging.zeroBasedNumberedPaging;
@Mapper
public abstract class QueryResultMapper extends HalAppenderMapper {
public abstract QueryResultDto map(@Context SearchParameters params, QueryResult result);
@AfterMapping
void setPageValues(@MappingTarget QueryResultDto dto, QueryResult result, @Context SearchParameters params) {
int totalHits = (int) result.getTotalHits();
dto.setPageTotal(computePageTotal(totalHits, params.getPageSize()));
dto.setPage(params.getPage());
}
@Nonnull
@ObjectFactory
QueryResultDto createDto(@Context SearchParameters params, QueryResult result) {
int totalHits = (int) result.getTotalHits();
Links.Builder links = links(params, totalHits);
Embedded.Builder embedded = hits(result);
applyEnrichers(new EdisonHalAppender(links, embedded), result);
return new QueryResultDto(links.build(), embedded.build());
}
@Nonnull
private QueryResultDto createDto(SearchParameters params, QueryResult result, int totalHits) {
Links.Builder links = links(params, totalHits);
Embedded.Builder embedded = hits(result);
applyEnrichers(new EdisonHalAppender(links, embedded), result);
return new QueryResultDto(links.build(), embedded.build());
}
private Links.Builder links(SearchParameters params, int totalHits) {
NumberedPaging paging = zeroBasedNumberedPaging(params.getPage(), params.getPageSize(), totalHits);
UriTemplate uriTemplate = fromTemplate(params.getSelfLink() + "{?q,page,pageSize}");
uriTemplate.set("q", params.getQuery());
return linkingTo()
.with(paging.links(
uriTemplate,
EnumSet.allOf(PagingRel.class))
);
}
@Nonnull
private Embedded.Builder hits(QueryResult result) {
List<HitDto> hits = result.getHits()
.stream()
.map(hit -> map(result, hit))
.collect(Collectors.toList());
return Embedded.embeddedBuilder().with("hits", hits);
}
@ObjectFactory
protected HitDto createHitDto(@Context QueryResult queryResult, Hit hit) {
Links.Builder links = linkingTo();
Embedded.Builder embedded = Embedded.embeddedBuilder();
applyEnrichers(new EdisonHalAppender(links, embedded), hit, queryResult);
return new HitDto(links.build(), embedded.build());
}
private int computePageTotal(int totalHits, int pageSize) {
if (totalHits % pageSize > 0) {
return totalHits / pageSize + 1;
} else {
return totalHits / pageSize;
}
}
protected abstract HitDto map(@Context QueryResult queryResult, Hit hit);
}

View File

@@ -1113,6 +1113,23 @@ class ResourceLinks {
}
}
public SearchLinks search() {
return new SearchLinks(scmPathInfoStore.get());
}
public static class SearchLinks {
private final LinkBuilder searchLinkBuilder;
SearchLinks(ScmPathInfo pathInfo) {
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class);
}
public String search() {
return searchLinkBuilder.method("search").parameters().href();
}
}
public InitialAdminAccountLinks initialAdminAccount() {
return new InitialAdminAccountLinks(new LinkBuilder(scmPathInfoStore.get(), InitializationResource.class, AdminAccountStartupResource.class));
}

View File

@@ -0,0 +1,61 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import lombok.Getter;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.Size;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
@Getter
public class SearchParameters {
@Context
private UriInfo uriInfo;
@Size(min = 2)
@QueryParam("q")
private String query;
@Min(0)
@QueryParam("page")
@DefaultValue("0")
private int page = 0;
@Min(1)
@Max(100)
@QueryParam("pageSize")
@DefaultValue("10")
private int pageSize = 10;
String getSelfLink() {
return uriInfo.getAbsolutePath().toASCIIString();
}
}

View File

@@ -0,0 +1,111 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.repository.Repository;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
import sonia.scm.web.VndMediaType;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.ws.rs.BeanParam;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@Path(SearchResource.PATH)
@OpenAPIDefinition(tags = {
@Tag(name = "Search", description = "Search related endpoints")
})
public class SearchResource {
static final String PATH = "v2/search";
private final SearchEngine engine;
private final QueryResultMapper mapper;
@Inject
public SearchResource(SearchEngine engine, QueryResultMapper mapper) {
this.engine = engine;
this.mapper = mapper;
}
@GET
@Path("")
@Produces(VndMediaType.QUERY_RESULT)
@Operation(
summary = "Query result",
description = "Returns a collection of matched hits.",
tags = "Search",
operationId = "search_query"
)
@ApiResponse(
responseCode = "200",
description = "success",
content = @Content(
mediaType = VndMediaType.QUERY_RESULT,
schema = @Schema(implementation = QueryResultDto.class)
)
)
@ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials")
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
))
@Parameter(
name = "query",
description = "The search expression",
required = true
)
@Parameter(
name = "page",
description = "The requested page number of the search results (zero based, defaults to 0)"
)
@Parameter(
name = "pageSize",
description = "The maximum number of results per page (defaults to 10)"
)
public QueryResultDto search(@Valid @BeanParam SearchParameters params) {
QueryResult result = engine.search(IndexNames.DEFAULT)
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
.execute(Repository.class, params.getQuery());
return mapper.map(params, result);
}
}

View File

@@ -98,6 +98,12 @@ import sonia.scm.repository.xml.XmlRepositoryDAO;
import sonia.scm.repository.xml.XmlRepositoryRoleDAO;
import sonia.scm.schedule.CronScheduler;
import sonia.scm.schedule.Scheduler;
import sonia.scm.search.DefaultIndexLogStore;
import sonia.scm.search.DefaultIndexQueue;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.LuceneSearchEngine;
import sonia.scm.search.SearchEngine;
import sonia.scm.security.AccessTokenCookieIssuer;
import sonia.scm.security.AuthorizationChangedEventProducer;
import sonia.scm.security.ConfigurableLoginAttemptHandler;
@@ -279,6 +285,11 @@ class ScmServletModule extends ServletModule {
bind(NotificationSender.class).to(DefaultNotificationSender.class);
bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class);
// bind search stuff
bind(IndexQueue.class, DefaultIndexQueue.class);
bind(SearchEngine.class, LuceneSearchEngine.class);
bind(IndexLogStore.class, DefaultIndexLogStore.class);
}
private <T> void bind(Class<T> clazz, Class<? extends T> defaultImplementation) {

View File

@@ -0,0 +1,128 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexLog;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.web.security.AdministrationContext;
import sonia.scm.web.security.PrivilegedAction;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.Optional;
@Extension
@Singleton
public class IndexUpdateListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class);
@VisibleForTesting
static final int INDEX_VERSION = 1;
private final AdministrationContext administrationContext;
private final IndexQueue queue;
private final IndexLogStore indexLogStore;
@Inject
public IndexUpdateListener(AdministrationContext administrationContext, IndexQueue queue, IndexLogStore indexLogStore) {
this.administrationContext = administrationContext;
this.queue = queue;
this.indexLogStore = indexLogStore;
}
@Subscribe(async = false)
public void handleEvent(RepositoryEvent event) {
HandlerEventType type = event.getEventType();
if (type.isPost()) {
updateIndex(type, event.getItem());
}
}
private void updateIndex(HandlerEventType type, Repository repository) {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
if (type == HandlerEventType.DELETE) {
index.deleteByRepository(repository.getId());
} else {
store(index, repository);
}
}
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
Optional<IndexLog> indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class);
if (!indexLog.isPresent()) {
LOG.debug("could not find log entry for repository index, start reindexing of all repositories");
administrationContext.runAsAdmin(ReIndexAll.class);
indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION);
}
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
// we have nothing to destroy
}
private static void store(Index index, Repository repository) {
index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository);
}
static class ReIndexAll implements PrivilegedAction {
private final RepositoryManager repositoryManager;
private final IndexQueue queue;
@Inject
public ReIndexAll(RepositoryManager repositoryManager, IndexQueue queue) {
this.repositoryManager = repositoryManager;
this.queue = queue;
}
@Override
public void run() {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
for (Repository repository : repositoryManager.getAll()) {
store(index, repository);
}
}
}
}
}

View File

@@ -0,0 +1,62 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import javax.annotation.Nonnull;
public class AnalyzerFactory {
@Nonnull
public Analyzer create(IndexOptions options) {
if (options.getType() == IndexOptions.Type.NATURAL_LANGUAGE) {
return createNaturalLanguageAnalyzer(options.getLocale().getLanguage());
}
return createDefaultAnalyzer();
}
private Analyzer createDefaultAnalyzer() {
return new StandardAnalyzer();
}
private Analyzer createNaturalLanguageAnalyzer(String lang) {
switch (lang) {
case "en":
return new EnglishAnalyzer();
case "de":
return new GermanAnalyzer();
case "es":
return new SpanishAnalyzer();
default:
return createDefaultAnalyzer();
}
}
}

View File

@@ -0,0 +1,59 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import sonia.scm.store.DataStore;
import sonia.scm.store.DataStoreFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.Optional;
@Singleton
public class DefaultIndexLogStore implements IndexLogStore {
private final DataStore<IndexLog> dataStore;
@Inject
public DefaultIndexLogStore(DataStoreFactory dataStoreFactory) {
this.dataStore = dataStoreFactory.withType(IndexLog.class).withName("index-log").build();
}
@Override
public void log(String index,Class<?> type, int version) {
String id = id(index, type);
dataStore.put(id, new IndexLog(version));
}
private String id(String index, Class<?> type) {
return index + "_" + type.getName();
}
@Override
public Optional<IndexLog> get(String index, Class<?> type) {
String id = id(index, type);
return dataStore.getOptional(id);
}
}

View File

@@ -0,0 +1,77 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.VisibleForTesting;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
@Singleton
public class DefaultIndexQueue implements IndexQueue, Closeable {
private final ExecutorService executor = Executors.newSingleThreadExecutor();
private final AtomicLong size = new AtomicLong(0);
private final SearchEngine searchEngine;
@Inject
public DefaultIndexQueue(SearchEngine searchEngine) {
this.searchEngine = searchEngine;
}
@Override
public Index getQueuedIndex(String name, IndexOptions indexOptions) {
return new QueuedIndex(this, name, indexOptions);
}
public SearchEngine getSearchEngine() {
return searchEngine;
}
void enqueue(IndexQueueTaskWrapper task) {
size.incrementAndGet();
executor.execute(() -> {
task.run();
size.decrementAndGet();
});
}
@VisibleForTesting
long getSize() {
return size.get();
}
@Override
public void close() throws IOException {
executor.shutdown();
}
}

View File

@@ -0,0 +1,144 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import javax.inject.Singleton;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Collections.emptySet;
@Singleton
public class DocumentConverter {
private final Map<Class<?>, TypeConverter> typeConverter = new ConcurrentHashMap<>();
Document convert(Object object) {
TypeConverter converter = typeConverter.computeIfAbsent(object.getClass(), this::createTypeConverter);
try {
return converter.convert(object);
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new SearchEngineException("failed to create document", ex);
}
}
private TypeConverter createTypeConverter(Class<?> type) {
List<FieldConverter> fieldConverters = new ArrayList<>();
collectFields(fieldConverters, type);
return new TypeConverter(fieldConverters);
}
private void collectFields(List<FieldConverter> fieldConverters, Class<?> type) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(fieldConverters, parent);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed);
Method getter = findGetter(type, field);
fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory));
}
}
}
private Method findGetter(Class<?> type, Field field) {
String name = createGetterName(field);
try {
return type.getMethod(name);
} catch (NoSuchMethodException ex) {
throw new NonReadableFieldException("could not find getter for field", ex);
}
}
private String createGetterName(Field field) {
String fieldName = field.getName();
String prefix = "get";
if (field.getType() == Boolean.TYPE) {
prefix = "is";
}
return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1);
}
private static class TypeConverter {
private final List<FieldConverter> fieldConverters;
private TypeConverter(List<FieldConverter> fieldConverters) {
this.fieldConverters = fieldConverters;
}
public Document convert(Object object) throws IllegalAccessException, InvocationTargetException {
Document document = new Document();
for (FieldConverter fieldConverter : fieldConverters) {
for (IndexableField field : fieldConverter.convert(object)) {
document.add(field);
}
}
return document;
}
}
private static class FieldConverter {
private final Method getter;
private final IndexableFieldFactory fieldFactory;
private final String name;
private FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) {
this.getter = getter;
this.fieldFactory = fieldFactory;
this.name = createName(field, indexed);
}
private String createName(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (Strings.isNullOrEmpty(nameFromAnnotation)) {
return field.getName();
}
return nameFromAnnotation;
}
Iterable<IndexableField> convert(Object object) throws IllegalAccessException, InvocationTargetException {
Object value = getter.invoke(object);
if (value != null) {
return fieldFactory.create(name, value);
}
return emptySet();
}
}
}

View File

@@ -0,0 +1,35 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
final class FieldNames {
private FieldNames(){}
static final String UID = "_uid";
static final String ID = "_id";
static final String TYPE = "_type";
static final String REPOSITORY = "_repository";
static final String PERMISSION = "_permission";
}

View File

@@ -0,0 +1,65 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import sonia.scm.SCMContextProvider;
import javax.inject.Inject;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class IndexOpener {
private final Path directory;
private final AnalyzerFactory analyzerFactory;
@Inject
public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) {
directory = context.resolve(Paths.get("index"));
this.analyzerFactory = analyzerFactory;
}
public IndexReader openForRead(String name) throws IOException {
return DirectoryReader.open(directory(name));
}
public IndexWriter openForWrite(String name, IndexOptions options) throws IOException {
IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options));
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(name), config);
}
private Directory directory(String name) throws IOException {
return FSDirectory.open(directory.resolve(name));
}
}

View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
@FunctionalInterface
public interface IndexQueueTask {
void updateIndex(Index index);
}

View File

@@ -0,0 +1,56 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class IndexQueueTaskWrapper implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class);
private final SearchEngine searchEngine;
private final String indexName;
private final IndexOptions options;
private final Iterable<IndexQueueTask> tasks;
IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable<IndexQueueTask> tasks) {
this.searchEngine = searchEngine;
this.indexName = indexName;
this.options = options;
this.tasks = tasks;
}
@Override
public void run() {
try (Index index = searchEngine.getOrCreate(this.indexName, options)) {
for (IndexQueueTask task : tasks) {
task.updateIndex(index);
}
} catch (Exception e) {
LOG.warn("failure during execution of index task for index {}", indexName, e);
}
}
}

View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.IndexableField;
@FunctionalInterface
interface IndexableFieldFactory {
Iterable<IndexableField> create(String name, Object value);
}

View File

@@ -0,0 +1,172 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
import java.text.DecimalFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static java.util.Collections.singleton;
import static sonia.scm.search.TypeCheck.isBoolean;
import static sonia.scm.search.TypeCheck.isInstant;
import static sonia.scm.search.TypeCheck.isInteger;
import static sonia.scm.search.TypeCheck.isLong;
class IndexableFields {
private IndexableFields() {
}
static PointsConfig pointConfig(Field field) {
Class<?> type = field.getType();
if (isLong(type) || isInstant(type)) {
return new PointsConfig(new DecimalFormat(), Long.class);
} else if (isInteger(type)) {
return new PointsConfig(new DecimalFormat(), Integer.class);
}
return null;
}
static IndexableFieldFactory create(Field field, Indexed indexed) {
Class<?> fieldType = field.getType();
Indexed.Type indexType = indexed.type();
if (fieldType == String.class) {
return new StringFieldFactory(indexType);
} else if (isLong(fieldType)) {
return new LongFieldFactory(indexType);
} else if (isInteger(fieldType)) {
return new IntegerFieldFactory(indexType);
} else if (isBoolean(fieldType)) {
return new BooleanFieldFactory(indexType);
} else if (isInstant(fieldType)) {
return new InstantFieldFactory(indexType);
} else {
throw new UnsupportedTypeOfFieldException(fieldType, field.getName());
}
}
private static class StringFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private StringFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
String stringValue = (String) value;
if (type.isTokenized()) {
return singleton(new TextField(name, stringValue, Store.YES));
} else if (type.isSearchable()) {
return singleton(new StringField(name, stringValue, Store.YES));
} else {
return singleton(new StoredField(name, stringValue));
}
}
}
private static class LongFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private LongFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Long longValue = (Long) value;
List<IndexableField> fields = new ArrayList<>();
if (type.isSearchable()) {
fields.add(new LongPoint(name, longValue));
}
fields.add(new StoredField(name, longValue));
return Collections.unmodifiableList(fields);
}
}
private static class IntegerFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private IntegerFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Integer integerValue = (Integer) value;
List<IndexableField> fields = new ArrayList<>();
if (type.isSearchable()) {
fields.add(new IntPoint(name, integerValue));
}
fields.add(new StoredField(name, integerValue));
return Collections.unmodifiableList(fields);
}
}
private static class BooleanFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private BooleanFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Boolean booleanValue = (Boolean) value;
if (type.isSearchable()) {
return singleton(new StringField(name, booleanValue.toString(), Store.YES));
} else {
return singleton(new StoredField(name, booleanValue.toString()));
}
}
}
private static class InstantFieldFactory extends LongFieldFactory {
private InstantFieldFactory(Indexed.Type type) {
super(type);
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Instant instant = (Instant) value;
return super.create(name, instant.toEpochMilli());
}
}
}

View File

@@ -0,0 +1,109 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import java.io.IOException;
import static sonia.scm.search.FieldNames.*;
public class LuceneIndex implements Index {
private final DocumentConverter converter;
private final IndexWriter writer;
LuceneIndex(DocumentConverter converter, IndexWriter writer) {
this.converter = converter;
this.writer = writer;
}
@Override
public void store(Id id, String permission, Object object) {
String uid = createUid(id, object.getClass());
Document document = converter.convert(object);
try {
field(document, UID, uid);
field(document, ID, id.getValue());
id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository));
field(document, TYPE, object.getClass().getName());
if (!Strings.isNullOrEmpty(permission)) {
field(document, PERMISSION, permission);
}
writer.updateDocument(new Term(UID, uid), document);
} catch (IOException e) {
throw new SearchEngineException("failed to add document to index", e);
}
}
private String createUid(Id id, Class<?> type) {
return id.asString() + "/" + type.getName();
}
private void field(Document document, String type, String name) {
document.add(new StringField(type, name, Field.Store.YES));
}
@Override
public void delete(Id id, Class<?> type) {
try {
writer.deleteDocuments(new Term(UID, createUid(id, type)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void deleteByRepository(String repository) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repository));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex);
}
}
@Override
public void deleteByType(Class<?> type) {
try {
writer.deleteDocuments(new Term(TYPE, type.getName()));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + type + " from index", ex);
}
}
@Override
public void close() {
try {
writer.close();
} catch (IOException e) {
throw new SearchEngineException("failed to close index writer", e);
}
}
}

View File

@@ -0,0 +1,149 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class LuceneQueryBuilder extends QueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class);
private static final Map<Class<?>, SearchableType> CACHE = new ConcurrentHashMap<>();
private final IndexOpener opener;
private final String indexName;
private final Analyzer analyzer;
LuceneQueryBuilder(IndexOpener opener, String indexName, Analyzer analyzer) {
this.opener = opener;
this.indexName = indexName;
this.analyzer = analyzer;
}
@Override
protected QueryResult execute(QueryParams queryParams) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString());
SearchableType searchableType = CACHE.computeIfAbsent(queryParams.getType(), SearchableTypes::create);
Query query = Queries.filter(createQuery(searchableType, queryParams, queryString), queryParams);
if (LOG.isDebugEnabled()) {
LOG.debug("execute lucene query: {}", query);
}
try (IndexReader reader = opener.openForRead(indexName)) {
IndexSearcher searcher = new IndexSearcher(reader);
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
Collector collector = new PermissionAwareCollector(reader, topScoreCollector);
searcher.search(query, collector);
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
} catch (IOException e) {
throw new SearchEngineException("failed to search index", e);
} catch (InvalidTokenOffsetsException e) {
throw new SearchEngineException("failed to highlight results", e);
}
}
@Nonnull
private TopScoreDocCollector createTopScoreCollector(QueryParams queryParams) {
return TopScoreDocCollector.create(queryParams.getStart() + queryParams.getLimit(), Integer.MAX_VALUE);
}
private TopDocs getTopDocs(QueryParams queryParams, TopScoreDocCollector topScoreCollector) {
return topScoreCollector.topDocs(queryParams.getStart(), queryParams.getLimit());
}
private Query createQuery(SearchableType searchableType, QueryParams queryParams, String queryString) {
try {
if (queryString.contains(":")) {
return createExpertQuery(searchableType, queryParams);
}
return createBestGuessQuery(searchableType, queryParams);
} catch (QueryNodeException ex) {
throw new QueryParseException(queryString, "failed to parse query", ex);
}
}
private Query createExpertQuery(SearchableType searchableType, QueryParams queryParams) throws QueryNodeException {
StandardQueryParser parser = new StandardQueryParser(analyzer);
parser.setPointsConfigMap(searchableType.getPointsConfig());
return parser.parse(queryParams.getQueryString(), "");
}
public Query createBestGuessQuery(SearchableType searchableType, QueryBuilder.QueryParams queryParams) {
String[] fieldNames = searchableType.getFieldNames();
if (fieldNames == null || fieldNames.length == 0) {
throw new NoDefaultQueryFieldsFoundException(searchableType.getType());
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (String fieldName : fieldNames) {
Term term = new Term(fieldName, appendWildcardIfNotAlreadyUsed(queryParams));
WildcardQuery query = new WildcardQuery(term);
Float boost = searchableType.getBoosts().get(fieldName);
if (boost != null) {
builder.add(new BoostQuery(query, boost), BooleanClause.Occur.SHOULD);
} else {
builder.add(query, BooleanClause.Occur.SHOULD);
}
}
return builder.build();
}
@Nonnull
private String appendWildcardIfNotAlreadyUsed(QueryParams queryParams) {
String queryString = queryParams.getQueryString().toLowerCase(Locale.ENGLISH);
if (!queryString.contains("?") && !queryString.contains("*")) {
queryString += "*";
}
return queryString;
}
}

View File

@@ -0,0 +1,44 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import javax.inject.Inject;
public class LuceneQueryBuilderFactory {
private final IndexOpener indexOpener;
private final AnalyzerFactory analyzerFactory;
@Inject
public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) {
this.indexOpener = indexOpener;
this.analyzerFactory = analyzerFactory;
}
public LuceneQueryBuilder create(String name, IndexOptions options) {
return new LuceneQueryBuilder(indexOpener, name, analyzerFactory.create(options));
}
}

View File

@@ -0,0 +1,57 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import javax.inject.Inject;
import java.io.IOException;
public class LuceneSearchEngine implements SearchEngine {
private final IndexOpener indexOpener;
private final DocumentConverter converter;
private final LuceneQueryBuilderFactory queryBuilderFactory;
@Inject
public LuceneSearchEngine(IndexOpener indexOpener, DocumentConverter converter, LuceneQueryBuilderFactory queryBuilderFactory) {
this.indexOpener = indexOpener;
this.converter = converter;
this.queryBuilderFactory = queryBuilderFactory;
}
@Override
public Index getOrCreate(String name, IndexOptions options) {
try {
return new LuceneIndex(converter, indexOpener.openForWrite(name, options));
} catch (IOException ex) {
throw new SearchEngineException("failed to open index", ex);
}
}
@Override
public QueryBuilder search(String name, IndexOptions options) {
return queryBuilderFactory.create(name, options);
}
}

View File

@@ -0,0 +1,31 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
public class NonReadableFieldException extends SearchEngineException {
public NonReadableFieldException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@@ -0,0 +1,88 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
import org.apache.shiro.SecurityUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
public class PermissionAwareCollector implements Collector {
private static final String FIELD_PERMISSION = "_permission";
private static final Set<String> FIELDS = Collections.singleton(FIELD_PERMISSION);
private final IndexReader reader;
private final Collector delegate;
public PermissionAwareCollector(IndexReader reader, Collector delegate) {
this.reader = reader;
this.delegate = delegate;
}
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
return new PermissionAwareLeafCollector(delegate.getLeafCollector(context), context.docBase);
}
@Override
public ScoreMode scoreMode() {
return delegate.scoreMode();
}
private class PermissionAwareLeafCollector implements LeafCollector {
private final LeafCollector delegate;
private final int docBase;
private PermissionAwareLeafCollector(LeafCollector delegate, int docBase) {
this.delegate = delegate;
this.docBase = docBase;
}
@Override
public void setScorer(Scorable scorer) throws IOException {
this.delegate.setScorer(scorer);
}
@Override
public void collect(int doc) throws IOException {
Document document = reader.document(docBase + doc, FIELDS);
String permission = document.get(FIELD_PERMISSION);
if (Strings.isNullOrEmpty(permission) || SecurityUtils.getSubject().isPermitted(permission)) {
this.delegate.collect(doc);
}
}
}
}

View File

@@ -0,0 +1,54 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
final class Queries {
private Queries() {
}
private static Query typeQuery(Class<?> type) {
return new TermQuery(new Term(FieldNames.TYPE, type.getName()));
}
private static Query repositoryQuery(String repositoryId) {
return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId));
}
static Query filter(Query query, QueryBuilder.QueryParams params) {
BooleanQuery.Builder builder = new BooleanQuery.Builder()
.add(query, MUST)
.add(typeQuery(params.getType()), MUST);
params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST));
return builder.build();
}
}

View File

@@ -0,0 +1,103 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static java.util.Optional.empty;
import static java.util.Optional.of;
public class QueryResultFactory {
private final Analyzer analyzer;
private final Highlighter highlighter;
private final IndexSearcher searcher;
private final SearchableType searchableType;
public QueryResultFactory(Analyzer analyzer, IndexSearcher searcher, SearchableType searchableType, Query query) {
this.analyzer = analyzer;
this.searcher = searcher;
this.searchableType = searchableType;
this.highlighter = createHighlighter(query);
}
private Highlighter createHighlighter(Query query) {
return new Highlighter(
new SimpleHTMLFormatter("**", "**"),
new QueryScorer(query)
);
}
public QueryResult create(TopDocs topDocs) throws IOException, InvalidTokenOffsetsException {
List<Hit> hits = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
hits.add(createHit(scoreDoc));
}
return new QueryResult(topDocs.totalHits.value, searchableType.getType(), hits);
}
private Hit createHit(ScoreDoc scoreDoc) throws IOException, InvalidTokenOffsetsException {
Document document = searcher.doc(scoreDoc.doc);
Map<String, Hit.Field> fields = new HashMap<>();
for (SearchableField field : searchableType.getFields()) {
field(document, field).ifPresent(f -> fields.put(field.getName(), f));
}
return new Hit(document.get(FieldNames.ID), scoreDoc.score, fields);
}
private Optional<Hit.Field> field(Document document, SearchableField field) throws IOException, InvalidTokenOffsetsException {
Object value = field.value(document);
if (value != null) {
if (field.isHighlighted()) {
String[] fragments = createFragments(field, value.toString());
if (fragments.length > 0) {
return of(new Hit.HighlightedField(fragments));
}
}
return of(new Hit.ValueField(value));
}
return empty();
}
private String[] createFragments(SearchableField field, String value) throws InvalidTokenOffsetsException, IOException {
return highlighter.getBestFragments(analyzer, field.getName(), value, 5);
}
}

View File

@@ -0,0 +1,71 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import java.util.ArrayList;
import java.util.List;
public class QueuedIndex implements Index {
private final DefaultIndexQueue queue;
private final String indexName;
private final IndexOptions indexOptions;
private final List<IndexQueueTask> tasks = new ArrayList<>();
QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) {
this.queue = queue;
this.indexName = indexName;
this.indexOptions = indexOptions;
}
@Override
public void store(Id id, String permission, Object object) {
tasks.add(index -> index.store(id, permission, object));
}
@Override
public void delete(Id id, Class<?> type) {
tasks.add(index -> index.delete(id, type));
}
@Override
public void deleteByRepository(String repository) {
tasks.add(index -> index.deleteByRepository(repository));
}
@Override
public void deleteByType(Class<?> type) {
tasks.add(index -> index.deleteByType(type));
}
@Override
public void close() {
IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper(
queue.getSearchEngine(), indexName, indexOptions, tasks
);
queue.enqueue(wrappedTask);
}
}

View File

@@ -0,0 +1,66 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import lombok.Getter;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
@Getter
class SearchableField {
private final String name;
private final Class<?> type;
private final ValueExtractor valueExtractor;
private final float boost;
private final boolean defaultQuery;
private final boolean highlighted;
private final PointsConfig pointsConfig;
SearchableField(Field field, Indexed indexed) {
this.name = name(field, indexed);
this.type = field.getType();
this.valueExtractor = ValueExtractors.create(name, type);
this.boost = indexed.boost();
this.defaultQuery = indexed.defaultQuery();
this.highlighted = indexed.highlighted();
this.pointsConfig = IndexableFields.pointConfig(field);
}
Object value(Document document) {
return valueExtractor.extract(document);
}
private String name(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (!Strings.isNullOrEmpty(nameFromAnnotation)) {
return nameFromAnnotation;
}
return field.getName();
}
}

View File

@@ -0,0 +1,51 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import lombok.Value;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Value
public class SearchableType {
Class<?> type;
String[] fieldNames;
Map<String,Float> boosts;
Map<String, PointsConfig> pointsConfig;
List<SearchableField> fields;
SearchableType(Class<?> type, String[] fieldNames, Map<String, Float> boosts, Map<String, PointsConfig> pointsConfig, List<SearchableField> fields) {
this.type = type;
this.fieldNames = fieldNames;
this.boosts = Collections.unmodifiableMap(boosts);
this.pointsConfig = Collections.unmodifiableMap(pointsConfig);
this.fields = Collections.unmodifiableList(fields);
}
}

View File

@@ -0,0 +1,82 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
final class SearchableTypes {
private static final float DEFAULT_BOOST = 1f;
private SearchableTypes() {
}
static SearchableType create(Class<?> type) {
List<SearchableField> fields = new ArrayList<>();
collectFields(type, fields);
return createSearchableType(type, fields);
}
private static SearchableType createSearchableType(Class<?> type, List<SearchableField> fields) {
String[] fieldsNames = fields.stream()
.filter(SearchableField::isDefaultQuery)
.map(SearchableField::getName)
.toArray(String[]::new);
Map<String, Float> boosts = new HashMap<>();
Map<String, PointsConfig> pointsConfig = new HashMap<>();
for (SearchableField field : fields) {
if (field.isDefaultQuery() && field.getBoost() != DEFAULT_BOOST) {
boosts.put(field.getName(), field.getBoost());
}
PointsConfig config = field.getPointsConfig();
if (config != null) {
pointsConfig.put(field.getName(), config);
}
}
return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields);
}
private static void collectFields(Class<?> type, List<SearchableField> fields) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(parent, fields);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
fields.add(new SearchableField(field, indexed));
}
}
}
}

View File

@@ -0,0 +1,53 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import java.time.Instant;
final class TypeCheck {
private TypeCheck() {
}
public static boolean isLong(Class<?> type) {
return type == Long.TYPE || type == Long.class;
}
public static boolean isInteger(Class<?> type) {
return type == Integer.TYPE || type == Integer.class;
}
public static boolean isBoolean(Class<?> type) {
return type == Boolean.TYPE || type == Boolean.class;
}
public static boolean isInstant(Class<?> type) {
return type == Instant.class;
}
public static boolean isString(Class<?> type) {
return type == String.class;
}
}

View File

@@ -0,0 +1,31 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
public class UnsupportedTypeOfFieldException extends SearchEngineException {
public UnsupportedTypeOfFieldException(Class<?> type, String field) {
super("type " + type + " of " + field + " is unsupported.");
}
}

View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.document.Document;
@FunctionalInterface
public interface ValueExtractor {
Object extract(Document document);
}

View File

@@ -0,0 +1,95 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.IndexableField;
import javax.annotation.Nonnull;
import java.time.Instant;
final class ValueExtractors {
private ValueExtractors() {
}
static ValueExtractor create(String name, Class<?> type) {
if (TypeCheck.isString(type)) {
return stringExtractor(name);
} else if (TypeCheck.isLong(type)) {
return longExtractor(name);
} else if (TypeCheck.isInteger(type)) {
return integerExtractor(name);
} else if (TypeCheck.isBoolean(type)) {
return booleanExtractor(name);
} else if (TypeCheck.isInstant(type)) {
return instantExtractor(name);
} else {
throw new UnsupportedTypeOfFieldException(type, name);
}
}
@Nonnull
private static ValueExtractor stringExtractor(String name) {
return doc -> doc.get(name);
}
@Nonnull
private static ValueExtractor instantExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return Instant.ofEpochMilli(field.numericValue().longValue());
}
return null;
};
}
@Nonnull
private static ValueExtractor booleanExtractor(String name) {
return doc -> Boolean.parseBoolean(doc.get(name));
}
@Nonnull
private static ValueExtractor integerExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return field.numericValue().intValue();
}
return null;
};
}
@Nonnull
private static ValueExtractor longExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return field.numericValue().longValue();
}
return null;
};
}
}

View File

@@ -195,5 +195,6 @@ class IndexDtoGeneratorTest {
when(resourceLinks.namespaceCollection()).thenReturn(new ResourceLinks.NamespaceCollectionLinks(scmPathInfo));
when(resourceLinks.me()).thenReturn(new ResourceLinks.MeLinks(scmPathInfo, new ResourceLinks.UserLinks(scmPathInfo)));
when(resourceLinks.repository()).thenReturn(new ResourceLinks.RepositoryLinks(scmPathInfo));
when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(scmPathInfo));
}
}

View File

@@ -82,6 +82,7 @@ public class ResourceLinksMock {
lenient().when(resourceLinks.adminInfo()).thenReturn(new ResourceLinks.AdminInfoLinks(pathInfo));
lenient().when(resourceLinks.apiKeyCollection()).thenReturn(new ResourceLinks.ApiKeyCollectionLinks(pathInfo));
lenient().when(resourceLinks.apiKey()).thenReturn(new ResourceLinks.ApiKeyLinks(pathInfo));
lenient().when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(pathInfo));
return resourceLinks;
}

View File

@@ -0,0 +1,266 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import com.fasterxml.jackson.databind.JsonNode;
import de.otto.edison.hal.HalRepresentation;
import lombok.Getter;
import lombok.Setter;
import org.jboss.resteasy.mock.MockHttpRequest;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mapstruct.factory.Mappers;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.Repository;
import sonia.scm.search.Hit;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
import sonia.scm.web.JsonMockHttpResponse;
import sonia.scm.web.RestDispatcher;
import sonia.scm.web.VndMediaType;
import javax.annotation.Nonnull;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class SearchResourceTest {
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private SearchEngine searchEngine;
private RestDispatcher dispatcher;
@Mock
private HalEnricherRegistry enricherRegistry;
@BeforeEach
void setUpDispatcher() {
QueryResultMapper mapper = Mappers.getMapper(QueryResultMapper.class);
mapper.setRegistry(enricherRegistry);
SearchResource resource = new SearchResource(
searchEngine, mapper
);
dispatcher = new RestDispatcher();
dispatcher.addSingletonResource(resource);
}
@Test
void shouldEnrichQueryResult() throws IOException, URISyntaxException {
when(enricherRegistry.allByType(QueryResult.class))
.thenReturn(Collections.singleton(new SampleEnricher()));
mockQueryResult("Hello", result(0L));
JsonMockHttpResponse response = search("Hello");
JsonNode sample = response.getContentAsJson().get("_embedded").get("sample");
assertThat(sample.get("type").asText()).isEqualTo("java.lang.String");
}
@Test
void shouldEnrichHitResult() throws IOException, URISyntaxException {
when(enricherRegistry.allByType(QueryResult.class))
.thenReturn(Collections.emptySet());
when(enricherRegistry.allByType(Hit.class))
.thenReturn(Collections.singleton(new SampleEnricher()));
mockQueryResult("Hello", result(1L, "Hello"));
JsonMockHttpResponse response = search("Hello");
JsonNode sample = response.getContentAsJson()
.get("_embedded").get("hits").get(0)
.get("_embedded").get("sample");
assertThat(sample.get("type").asText()).isEqualTo("java.lang.String");
}
@Nested
class WithoutEnricher {
@BeforeEach
void setUpEnricherRegistry() {
when(enricherRegistry.allByType(QueryResult.class)).thenReturn(Collections.emptySet());
lenient().when(enricherRegistry.allByType(Hit.class)).thenReturn(Collections.emptySet());
}
@Test
void shouldReturnVndContentType() throws UnsupportedEncodingException, URISyntaxException {
mockQueryResult("Hello", result(0L));
JsonMockHttpResponse response = search("Hello");
assertThat(response.getStatus()).isEqualTo(HttpServletResponse.SC_OK);
assertHeader(response, "Content-Type", VndMediaType.QUERY_RESULT);
}
@Test
void shouldReturnPagingLinks() throws IOException, URISyntaxException {
mockQueryResult(20, 20, "paging", result(100));
JsonMockHttpResponse response = search("paging", 1, 20);
JsonNode links = response.getContentAsJson().get("_links");
assertLink(links, "self", "/v2/search?q=paging&page=1&pageSize=20");
assertLink(links, "first", "/v2/search?q=paging&page=0&pageSize=20");
assertLink(links, "prev", "/v2/search?q=paging&page=0&pageSize=20");
assertLink(links, "next", "/v2/search?q=paging&page=2&pageSize=20");
assertLink(links, "last", "/v2/search?q=paging&page=4&pageSize=20");
}
@Test
void shouldPagingFields() throws IOException, URISyntaxException {
mockQueryResult(20, 20, "pagingFields", result(100));
JsonMockHttpResponse response = search("pagingFields", 1, 20);
JsonNode root = response.getContentAsJson();
assertThat(root.get("page").asInt()).isOne();
assertThat(root.get("pageTotal").asInt()).isEqualTo(5);
}
@Test
void shouldReturnType() throws IOException, URISyntaxException {
mockQueryResult("Hello", result(0L));
JsonMockHttpResponse response = search("Hello");
JsonNode root = response.getContentAsJson();
assertThat(root.get("type").asText()).isEqualTo("java.lang.String");
}
@Test
void shouldReturnHitsAsEmbedded() throws IOException, URISyntaxException {
mockQueryResult("Hello", result(2L, "Hello", "Hello Again"));
JsonMockHttpResponse response = search("Hello");
JsonNode hits = response.getContentAsJson().get("_embedded").get("hits");
assertThat(hits.size()).isEqualTo(2);
JsonNode first = hits.get(0);
assertThat(first.get("score").asDouble()).isEqualTo(2d);
JsonNode fields = first.get("fields");
JsonNode valueField = fields.get("value");
assertThat(valueField.get("highlighted").asBoolean()).isFalse();
assertThat(valueField.get("value").asText()).isEqualTo("Hello");
JsonNode highlightedField = fields.get("highlighted");
assertThat(highlightedField.get("highlighted").asBoolean()).isTrue();
assertThat(highlightedField.get("fragments").get(0).asText()).isEqualTo("Hello");
}
}
private void assertLink(JsonNode links, String self, String s) {
assertThat(links.get(self).get("href").asText()).isEqualTo(s);
}
private QueryResult result(long totalHits, Object... values) {
List<Hit> hits = new ArrayList<>();
for (int i = 0; i < values.length; i++) {
hits.add(hit(i, values));
}
return new QueryResult(totalHits, String.class, hits);
}
@Nonnull
private Hit hit(int i, Object[] values) {
Map<String, Hit.Field> fields = fields(values[i]);
return new Hit("" + i, values.length - i, fields);
}
@Nonnull
private Map<String, Hit.Field> fields(Object value) {
Map<String, Hit.Field> fields = new HashMap<>();
fields.put("value", new Hit.ValueField(value));
fields.put("highlighted", new Hit.HighlightedField(new String[]{value.toString()}));
return fields;
}
private void mockQueryResult(String query, QueryResult result) {
mockQueryResult(0, 10, query, result);
}
private void mockQueryResult(int start, int limit, String query, QueryResult result) {
when(
searchEngine.search(IndexNames.DEFAULT)
.start(start)
.limit(limit)
.execute(Repository.class, query)
).thenReturn(result);
}
private void assertHeader(JsonMockHttpResponse response, String header, String expectedValue) {
assertThat(response.getOutputHeaders().getFirst(header)).hasToString(expectedValue);
}
private JsonMockHttpResponse search(String query) throws URISyntaxException, UnsupportedEncodingException {
return search(query, null, null);
}
private JsonMockHttpResponse search(String query, Integer page, Integer pageSize) throws URISyntaxException, UnsupportedEncodingException {
String uri = "/v2/search?q=" + URLEncoder.encode(query, "UTF-8");
if (page != null) {
uri += "&page=" + page;
}
if (pageSize != null) {
uri += "&pageSize=" + pageSize;
}
MockHttpRequest request = MockHttpRequest.get(uri);
JsonMockHttpResponse response = new JsonMockHttpResponse();
dispatcher.invoke(request, response);
return response;
}
@Getter
@Setter
public static class SampleEmbedded extends HalRepresentation {
private Class<?> type;
}
private static class SampleEnricher implements HalEnricher {
@Override
public void enrich(HalEnricherContext context, HalAppender appender) {
QueryResult result = context.oneRequireByType(QueryResult.class);
SampleEmbedded embedded = new SampleEmbedded();
embedded.setType(result.getType());
appender.appendEmbedded("sample", embedded);
}
}
}

View File

@@ -0,0 +1,147 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository;
import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.HandlerEventType;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexLog;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.web.security.AdministrationContext;
import java.util.List;
import java.util.Optional;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class IndexUpdateListenerTest {
@Mock
private RepositoryManager repositoryManager;
@Mock
private AdministrationContext administrationContext;
@Mock
private IndexQueue indexQueue;
@Mock
private Index index;
@Mock
private IndexLogStore indexLogStore;
@InjectMocks
private IndexUpdateListener updateListener;
@Test
@SuppressWarnings("java:S6068")
void shouldIndexAllRepositories() {
when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.empty());
doAnswer(ic -> {
IndexUpdateListener.ReIndexAll reIndexAll = new IndexUpdateListener.ReIndexAll(repositoryManager, indexQueue);
reIndexAll.run();
return null;
})
.when(administrationContext)
.runAsAdmin(IndexUpdateListener.ReIndexAll.class);
Repository heartOfGold = RepositoryTestData.createHeartOfGold();
Repository puzzle42 = RepositoryTestData.create42Puzzle();
List<Repository> repositories = ImmutableList.of(heartOfGold, puzzle42);
when(repositoryManager.getAll()).thenReturn(repositories);
when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index);
updateListener.contextInitialized(null);
verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold);
verify(index).store(Id.of(puzzle42), RepositoryPermissions.read(puzzle42).asShiroString(), puzzle42);
verify(index).close();
verify(indexLogStore).log(IndexNames.DEFAULT, Repository.class, IndexUpdateListener.INDEX_VERSION);
}
@Test
void shouldSkipReIndex() {
IndexLog log = new IndexLog(1);
when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.of(log));
updateListener.contextInitialized(null);
verifyNoInteractions(indexQueue);
}
@ParameterizedTest
@EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.MATCH_ANY, names = "BEFORE_.*")
void shouldIgnoreBeforeEvents(HandlerEventType type) {
RepositoryEvent event = new RepositoryEvent(type, RepositoryTestData.create42Puzzle());
updateListener.handleEvent(event);
verifyNoInteractions(indexQueue);
}
@ParameterizedTest
@EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.INCLUDE, names = {"CREATE", "MODIFY"})
void shouldStore(HandlerEventType type) {
when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index);
Repository puzzle = RepositoryTestData.create42Puzzle();
RepositoryEvent event = new RepositoryEvent(type, puzzle);
updateListener.handleEvent(event);
verify(index).store(Id.of(puzzle), RepositoryPermissions.read(puzzle).asShiroString(), puzzle);
verify(index).close();
}
@Test
void shouldDelete() {
when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index);
Repository puzzle = RepositoryTestData.create42Puzzle();
RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, puzzle);
updateListener.handleEvent(event);
verify(index).deleteByRepository(puzzle.getId());
verify(index).close();
}
}

View File

@@ -0,0 +1,78 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.junit.jupiter.api.Test;
import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat;
class AnalyzerFactoryTest {
private final AnalyzerFactory analyzerFactory = new AnalyzerFactory();
@Test
void shouldReturnStandardAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults());
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnStandardAnalyzerForUnknownLocale() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE));
assertThat(analyzer).isInstanceOf(StandardAnalyzer.class);
}
@Test
void shouldReturnEnglishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH));
assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnGermanAnalyzerForLocaleGermany() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY));
assertThat(analyzer).isInstanceOf(GermanAnalyzer.class);
}
@Test
void shouldReturnSpanishAnalyzer() {
Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES")));
assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class);
}
}

View File

@@ -0,0 +1,61 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import sonia.scm.store.InMemoryByteDataStoreFactory;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
class DefaultIndexLogStoreTest {
private IndexLogStore indexLogStore;
@BeforeEach
void setUpIndexLogStore() {
InMemoryByteDataStoreFactory dataStoreFactory = new InMemoryByteDataStoreFactory();
indexLogStore = new DefaultIndexLogStore(dataStoreFactory);
}
@Test
void shouldReturnEmptyOptional() {
Optional<IndexLog> indexLog = indexLogStore.get("index", String.class);
assertThat(indexLog).isEmpty();
}
@Test
void shouldStoreLog() {
indexLogStore.log("index", String.class, 42);
Optional<IndexLog> index = indexLogStore.get("index", String.class);
assertThat(index).hasValueSatisfying(log -> {
assertThat(log.getVersion()).isEqualTo(42);
assertThat(log.getDate()).isNotNull();
});
}
}

View File

@@ -0,0 +1,145 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import lombok.Value;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class DefaultIndexQueueTest {
private Directory directory;
private DefaultIndexQueue queue;
@Mock
private LuceneQueryBuilderFactory queryBuilderFactory;
@BeforeEach
void createQueue() throws IOException {
directory = new ByteBuffersDirectory();
IndexOpener factory = mock(IndexOpener.class);
when(factory.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, config);
});
SearchEngine engine = new LuceneSearchEngine(factory, new DocumentConverter(), queryBuilderFactory);
queue = new DefaultIndexQueue(engine);
}
@AfterEach
void closeQueue() throws IOException {
queue.close();
directory.close();
}
@Test
void shouldWriteToIndex() throws Exception {
try (Index index = queue.getQueuedIndex("default")) {
index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan"));
index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent"));
}
assertDocCount(2);
}
@Test
void shouldWriteMultiThreaded() throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(4);
for (int i = 0; i < 20; i++) {
executorService.execute(new IndexNumberTask(i));
}
executorService.execute(() -> {
try (Index index = queue.getQueuedIndex("default")) {
index.delete(Id.of(String.valueOf(12)), IndexedNumber.class);
}
});
executorService.shutdown();
assertDocCount(19);
}
private void assertDocCount(int expectedCount) throws IOException {
// wait until all tasks are finished
await().until(() -> queue.getSize() == 0);
try (DirectoryReader reader = DirectoryReader.open(directory)) {
assertThat(reader.numDocs()).isEqualTo(expectedCount);
}
}
@Value
public static class Account {
@Indexed
String username;
@Indexed
String firstName;
@Indexed
String lastName;
}
@Value
public static class IndexedNumber {
@Indexed
int value;
}
public class IndexNumberTask implements Runnable {
private final int number;
public IndexNumberTask(int number) {
this.number = number;
}
@Override
public void run() {
try (Index index = queue.getQueuedIndex("default")) {
index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number));
}
}
}
}

View File

@@ -0,0 +1,288 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
class DocumentConverterTest {
private DocumentConverter documentConverter;
@BeforeEach
void prepare() {
documentConverter = new DocumentConverter();
}
@Test
void shouldConvertPersonToDocument() {
Person person = new Person("Arthur", "Dent");
Document document = documentConverter.convert(person);
assertThat(document.getField("firstName").stringValue()).isEqualTo("Arthur");
assertThat(document.getField("lastName").stringValue()).isEqualTo("Dent");
}
@Test
void shouldUseNameFromAnnotation() {
Document document = documentConverter.convert(new ParamSample());
assertThat(document.getField("username").stringValue()).isEqualTo("dent");
}
@Test
void shouldBeIndexedAsTextFieldByDefault() {
Document document = documentConverter.convert(new ParamSample());
assertThat(document.getField("username")).isInstanceOf(TextField.class);
}
@Test
void shouldBeIndexedAsStringField() {
Document document = documentConverter.convert(new ParamSample());
assertThat(document.getField("searchable")).isInstanceOf(StringField.class);
}
@Test
void shouldBeIndexedAsStoredField() {
Document document = documentConverter.convert(new ParamSample());
assertThat(document.getField("storedOnly")).isInstanceOf(StoredField.class);
}
@Test
void shouldIgnoreNonIndexedFields() {
Document document = documentConverter.convert(new ParamSample());
assertThat(document.getField("notIndexed")).isNull();
}
@Test
void shouldSupportInheritance() {
Account account = new Account("Arthur", "Dent", "arthur@hitchhiker.com");
Document document = documentConverter.convert(account);
assertThat(document.getField("firstName")).isNotNull();
assertThat(document.getField("lastName")).isNotNull();
assertThat(document.getField("mail")).isNotNull();
}
@Test
void shouldFailWithoutGetter() {
WithoutGetter withoutGetter = new WithoutGetter();
assertThrows(NonReadableFieldException.class, () -> documentConverter.convert(withoutGetter));
}
@Test
void shouldFailOnUnsupportedFieldType() {
UnsupportedFieldType unsupportedFieldType = new UnsupportedFieldType();
assertThrows(UnsupportedTypeOfFieldException.class, () -> documentConverter.convert(unsupportedFieldType));
}
@Test
void shouldStoreLongFieldsAsPointAndStoredByDefault() {
Document document = documentConverter.convert(new SupportedTypes());
assertPointField(document, "longType",
field -> assertThat(field.numericValue().longValue()).isEqualTo(42L)
);
}
@Test
void shouldStoreLongFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyLongType");
assertThat(field).isInstanceOf(StoredField.class);
assertThat(field.numericValue().longValue()).isEqualTo(42L);
}
@Test
void shouldStoreIntegerFieldsAsPointAndStoredByDefault() {
Document document = documentConverter.convert(new SupportedTypes());
assertPointField(document, "intType",
field -> assertThat(field.numericValue().intValue()).isEqualTo(42)
);
}
@Test
void shouldStoreIntegerFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyIntegerType");
assertThat(field).isInstanceOf(StoredField.class);
assertThat(field.numericValue().intValue()).isEqualTo(42);
}
@Test
void shouldStoreBooleanFieldsAsStringField() {
Document document = documentConverter.convert(new SupportedTypes());
IndexableField field = document.getField("boolType");
assertThat(field).isInstanceOf(StringField.class);
assertThat(field.stringValue()).isEqualTo("true");
assertThat(field.fieldType().stored()).isTrue();
}
@Test
void shouldStoreBooleanFieldAsStored() {
Document document = documentConverter.convert(new SupportedTypes());
IndexableField field = document.getField("storedOnlyBoolType");
assertThat(field).isInstanceOf(StoredField.class);
assertThat(field.stringValue()).isEqualTo("true");
}
@Test
void shouldStoreInstantFieldsAsPointAndStoredByDefault() {
Instant now = Instant.now();
Document document = documentConverter.convert(new DateTypes(now));
assertPointField(document, "instant",
field -> assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli())
);
}
@Test
void shouldStoreInstantFieldAsStored() {
Instant now = Instant.now();
Document document = documentConverter.convert(new DateTypes(now));
IndexableField field = document.getField("storedOnlyInstant");
assertThat(field).isInstanceOf(StoredField.class);
assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli());
}
@Test
void shouldCreateNoFieldForNullValues() {
Document document = documentConverter.convert(new Person("Trillian", null));
assertThat(document.getField("firstName")).isNotNull();
assertThat(document.getField("lastName")).isNull();
}
private void assertPointField(Document document, String name, Consumer<IndexableField> consumer) {
IndexableField[] fields = document.getFields(name);
assertThat(fields)
.allSatisfy(consumer)
.anySatisfy(field -> assertThat(field.fieldType().stored()).isFalse())
.anySatisfy(field -> assertThat(field.fieldType().stored()).isTrue());
}
@Getter
@AllArgsConstructor
public static class Person {
@Indexed
private String firstName;
@Indexed
private String lastName;
}
@Getter
public static class Account extends Person {
@Indexed
private String mail;
public Account(String firstName, String lastName, String mail) {
super(firstName, lastName);
this.mail = mail;
}
}
@Getter
public static class ParamSample {
@Indexed(name = "username")
private final String name = "dent";
@Indexed(type = Indexed.Type.SEARCHABLE)
private final String searchable = "--";
@Indexed(type = Indexed.Type.STORED_ONLY)
private final String storedOnly = "--";
private final String notIndexed = "--";
}
public static class WithoutGetter {
@Indexed
private final String value = "one";
}
@Getter
public static class UnsupportedFieldType {
@Indexed
private final Object value = "one";
}
@Getter
public static class SupportedTypes {
@Indexed
private final Long longType = 42L;
@Indexed(type = Indexed.Type.STORED_ONLY)
private final long storedOnlyLongType = 42L;
@Indexed
private final int intType = 42;
@Indexed(type = Indexed.Type.STORED_ONLY)
private final Integer storedOnlyIntegerType = 42;
@Indexed
private final boolean boolType = true;
@Indexed(type = Indexed.Type.STORED_ONLY)
private final boolean storedOnlyBoolType = true;
}
@Getter
private static class DateTypes {
@Indexed
private final Instant instant;
@Indexed(type = Indexed.Type.STORED_ONLY)
private final Instant storedOnlyInstant;
private DateTypes(Instant instant) {
this.instant = instant;
this.storedOnlyInstant = instant;
}
}
}

View File

@@ -0,0 +1,99 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.SCMContextProvider;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class IndexOpenerTest {
private Path directory;
@Mock
private AnalyzerFactory analyzerFactory;
private IndexOpener indexOpener;
@BeforeEach
void createIndexWriterFactory(@TempDir Path tempDirectory) {
this.directory = tempDirectory;
SCMContextProvider context = mock(SCMContextProvider.class);
when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory);
when(analyzerFactory.create(any(IndexOptions.class))).thenReturn(new SimpleAnalyzer());
indexOpener = new IndexOpener(context, analyzerFactory);
}
@Test
void shouldCreateNewIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) {
addDoc(writer, "Trillian");
}
assertThat(directory.resolve("new-index")).exists();
}
@Test
void shouldOpenExistingIndex() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) {
addDoc(writer, "Dent");
}
try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) {
assertThat(writer.getFieldNames()).contains("hitchhiker");
}
}
@Test
void shouldUseAnalyzerFromFactory() throws IOException {
try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) {
assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class);
}
}
private void addDoc(IndexWriter writer, String name) throws IOException {
Document doc = new Document();
doc.add(new TextField("hitchhiker", name, Field.Store.YES));
writer.addDocument(doc);
}
}

View File

@@ -0,0 +1,238 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import lombok.Value;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import static org.assertj.core.api.Assertions.assertThat;
import static sonia.scm.search.FieldNames.*;
class LuceneIndexTest {
private static final Id ONE = Id.of("one");
private Directory directory;
@BeforeEach
void createDirectory() {
directory = new ByteBuffersDirectory();
}
@Test
void shouldStoreObject() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
}
assertHits("value", "content", 1);
}
@Test
void shouldUpdateObject() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
index.store(ONE, null, new Storable("Awesome content"));
}
assertHits("value", "content", 1);
}
@Test
void shouldStoreUidOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Awesome content which should be indexed"));
}
assertHits(UID, "one/" + Storable.class.getName(), 1);
}
@Test
void shouldStoreIdOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Some text"));
}
assertHits(ID, "one", 1);
}
@Test
void shouldStoreRepositoryOfId() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE.withRepository("4211"), null, new Storable("Some text"));
}
assertHits(REPOSITORY, "4211", 1);
}
@Test
void shouldStoreTypeOfObject() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Some other text"));
}
assertHits(TYPE, Storable.class.getName(), 1);
}
@Test
void shouldDeleteById() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Some other text"));
}
try (LuceneIndex index = createIndex()) {
index.delete(ONE, Storable.class);
}
assertHits(ID, "one", 0);
}
@Test
void shouldDeleteAllByType() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("content"));
index.store(Id.of("two"), null, new Storable("content"));
index.store(Id.of("three"), null, new OtherStorable("content"));
}
try (LuceneIndex index = createIndex()) {
index.deleteByType(Storable.class);
}
assertHits("value", "content", 1);
}
@Test
void shouldDeleteByIdAnyType() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Some text"));
index.store(ONE, null, new OtherStorable("Some other text"));
}
try (LuceneIndex index = createIndex()) {
index.delete(ONE, Storable.class);
}
assertHits(ID, "one", 1);
ScoreDoc[] docs = assertHits(ID, "one", 1);
Document doc = doc(docs[0].doc);
assertThat(doc.get("value")).isEqualTo("Some other text");
}
@Test
void shouldDeleteByIdAndRepository() throws IOException {
Id withRepository = ONE.withRepository("4211");
try (LuceneIndex index = createIndex()) {
index.store(ONE, null, new Storable("Some other text"));
index.store(withRepository, null, new Storable("New stuff"));
}
try (LuceneIndex index = createIndex()) {
index.delete(withRepository, Storable.class);
}
ScoreDoc[] docs = assertHits(ID, "one", 1);
Document doc = doc(docs[0].doc);
assertThat(doc.get("value")).isEqualTo("Some other text");
}
@Test
void shouldDeleteByRepository() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE.withRepository("4211"), null, new Storable("Some other text"));
index.store(ONE.withRepository("4212"), null, new Storable("New stuff"));
}
try (LuceneIndex index = createIndex()) {
index.deleteByRepository("4212");
}
assertHits(ID, "one", 1);
}
@Test
void shouldStorePermission() throws IOException {
try (LuceneIndex index = createIndex()) {
index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text"));
}
assertHits(PERMISSION, "repo:4211:read", 1);
}
private Document doc(int doc) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
return reader.document(doc);
}
}
@CanIgnoreReturnValue
private ScoreDoc[] assertHits(String field, String value, int expectedHits) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs docs = searcher.search(new TermQuery(new Term(field, value)), 10);
assertThat(docs.totalHits.value).isEqualTo(expectedHits);
return docs.scoreDocs;
}
}
private LuceneIndex createIndex() throws IOException {
return new LuceneIndex(new DocumentConverter(), createWriter());
}
private IndexWriter createWriter() throws IOException {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory, config);
}
@Value
private static class Storable {
@Indexed
String value;
}
@Value
private static class OtherStorable {
@Indexed
String value;
}
}

View File

@@ -0,0 +1,592 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.github.sdorra.jse.ShiroExtension;
import org.github.sdorra.jse.SubjectAware;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
@SubjectAware(value = "trillian", permissions = "abc")
@ExtendWith({MockitoExtension.class, ShiroExtension.class})
class LuceneQueryBuilderTest {
private Directory directory;
@Mock
private IndexOpener opener;
@BeforeEach
void setUpDirectory() {
directory = new ByteBuffersDirectory();
}
@Test
void shouldReturnHitsForBestGuessQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211"));
}
QueryResult result = query(InetOrgPerson.class, "Arthur");
assertThat(result.getTotalHits()).isOne();
}
@Test
void shouldMatchPartial() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(personDoc("Trillian"));
}
QueryResult result = query(Person.class, "Trill");
assertThat(result.getTotalHits()).isOne();
}
@Test
@SuppressWarnings("java:S5976")
void shouldNotAppendWildcardIfStarIsUsed() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("Trillian"));
}
QueryResult result = query(Simple.class, "Tr*ll");
assertThat(result.getTotalHits()).isZero();
}
@Test
@SuppressWarnings("java:S5976")
void shouldNotAppendWildcardIfQuestionMarkIsUsed() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("Trillian"));
}
QueryResult result = query(Simple.class, "Tr?ll");
assertThat(result.getTotalHits()).isZero();
}
@Test
@SuppressWarnings("java:S5976")
void shouldNotAppendWildcardIfExpertQueryIsUsed() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("Trillian"));
}
QueryResult result = query(Simple.class, "lastName:Trill");
assertThat(result.getTotalHits()).isZero();
}
@Test
void shouldSupportFieldsFromParentClass() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211"));
}
QueryResult result = query(InetOrgPerson.class, "Dent");
assertThat(result.getTotalHits()).isOne();
}
@Test
void shouldIgnoreHitsOfOtherType() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211"));
writer.addDocument(personDoc("Dent"));
}
QueryResult result = query(InetOrgPerson.class, "Dent");
assertThat(result.getTotalHits()).isOne();
}
@Test
void shouldThrowQueryParseExceptionOnInvalidQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(personDoc("Dent"));
}
assertThrows(QueryParseException.class, () -> query(String.class, ":~:~"));
}
@Test
void shouldIgnoreNonDefaultFieldsForBestGuessQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "car"));
}
QueryResult result = query(InetOrgPerson.class, "car");
assertThat(result.getTotalHits()).isZero();
}
@Test
void shouldUseBoostFromAnnotationForBestGuessQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arti", "car"));
writer.addDocument(inetOrgPersonDoc("Fake", "Dent", "Arthur, Arthur, Arthur", "mycar"));
}
QueryResult result = query(InetOrgPerson.class, "Arthur");
assertThat(result.getTotalHits()).isEqualTo(2);
List<Hit> hits = result.getHits();
Hit arthur = hits.get(0);
assertValueField(arthur, "firstName", "Arthur");
Hit fake = hits.get(1);
assertValueField(fake, "firstName", "Fake");
assertThat(arthur.getScore()).isGreaterThan(fake.getScore());
}
private void assertValueField(Hit hit, String fieldName, Object value) {
assertThat(hit.getFields().get(fieldName))
.isInstanceOfSatisfying(Hit.ValueField.class, (field) -> {
assertThat(field.isHighlighted()).isFalse();
assertThat(field.getValue()).isEqualTo(value);
});
}
@Test
void shouldReturnHitsForExpertQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("Awesome content one"));
writer.addDocument(simpleDoc("Awesome content two"));
writer.addDocument(simpleDoc("Awesome content three"));
}
QueryResult result = query(Simple.class, "content:awesome");
assertThat(result.getTotalHits()).isEqualTo(3L);
assertThat(result.getHits()).hasSize(3);
}
@Test
void shouldReturnOnlyHitsOfTypeForExpertQuery() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Ford", "Prefect", "Ford Prefect", "4211"));
writer.addDocument(personDoc("Prefect"));
}
QueryResult result = query(InetOrgPerson.class, "lastName:prefect");
assertThat(result.getTotalHits()).isEqualTo(1L);
}
@Test
void shouldReturnOnlyPermittedHits() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(permissionDoc("Awesome content one", "abc"));
writer.addDocument(permissionDoc("Awesome content two", "cde"));
writer.addDocument(permissionDoc("Awesome content three", "fgh"));
}
QueryResult result = query(Simple.class, "content:awesome");
assertThat(result.getTotalHits()).isOne();
List<Hit> hits = result.getHits();
assertThat(hits).hasSize(1).allSatisfy(hit -> {
assertValueField(hit, "content", "Awesome content one");
assertThat(hit.getScore()).isGreaterThan(0f);
});
}
@Test
void shouldFilterByRepository() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(repositoryDoc("Awesome content one", "abc"));
writer.addDocument(repositoryDoc("Awesome content two", "cde"));
writer.addDocument(repositoryDoc("Awesome content three", "fgh"));
}
QueryResult result;
try (DirectoryReader reader = DirectoryReader.open(directory)) {
when(opener.openForRead("default")).thenReturn(reader);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, "default", new StandardAnalyzer()
);
result = builder.repository("cde").execute(Simple.class, "content:awesome");
}
assertThat(result.getTotalHits()).isOne();
List<Hit> hits = result.getHits();
assertThat(hits).hasSize(1).allSatisfy(hit -> {
assertValueField(hit, "content", "Awesome content two");
assertThat(hit.getScore()).isGreaterThan(0f);
});
}
@Test
void shouldReturnStringFields() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("Awesome"));
}
QueryResult result = query(Simple.class, "content:awesome");
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertValueField(hit, "content", "Awesome")
);
}
@Test
void shouldReturnIdOfHit() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Slarti", "Bartfass", "Slartibartfass", "-"));
}
QueryResult result = query(InetOrgPerson.class, "lastName:Bartfass");
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertThat(hit.getId()).isEqualTo("Bartfass")
);
}
@Test
void shouldReturnTypeOfHits() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(simpleDoc("We need the type"));
}
QueryResult result = query(Simple.class, "content:type");
assertThat(result.getType()).isEqualTo(Simple.class);
}
@Test
void shouldSupportIntRangeQueries() throws IOException {
Instant now = Instant.now();
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(42, 21L, false, now));
}
QueryResult result = query(Types.class, "intValue:[0 TO 100]");
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertValueField(hit, "intValue", 42)
);
}
@Test
void shouldSupportLongRangeQueries() throws IOException {
Instant now = Instant.now();
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(42, 21L, false, now));
}
QueryResult result = query(Types.class, "longValue:[0 TO 100]");
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertValueField(hit, "longValue", 21L)
);
}
@Test
void shouldSupportInstantRangeQueries() throws IOException {
Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli());
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(42, 21L, false, now));
}
long before = now.minus(1, ChronoUnit.MINUTES).toEpochMilli();
long after = now.plus(1, ChronoUnit.MINUTES).toEpochMilli();
String queryString = String.format("instantValue:[%d TO %d]", before, after);
QueryResult result = query(Types.class, queryString);
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertValueField(hit, "instantValue", now)
);
}
@Test
void shouldSupportQueryForBooleanFields() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(21, 42L, true, Instant.now()));
}
QueryResult result = query(Types.class, "boolValue:true");
assertThat(result.getTotalHits()).isOne();
assertThat(result.getHits()).allSatisfy(
hit -> assertValueField(hit, "boolValue", Boolean.TRUE)
);
}
@Test
void shouldReturnValueFieldForHighlightedFieldWithoutFragment() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Marvin", "HoG", "Paranoid Android", "4211"));
}
QueryResult result = query(InetOrgPerson.class, "Marvin");
Hit hit = result.getHits().get(0);
assertValueField(hit, "displayName", "Paranoid Android");
}
@Test
void shouldFailBestGuessQueryWithoutDefaultQueryFields() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(1, 2L, false, Instant.now()));
}
assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something"));
}
@Test
void shouldLimitHitsByDefaultSize() throws IOException {
try (IndexWriter writer = writer()) {
for (int i = 0; i < 20; i++)
writer.addDocument(simpleDoc("counter " + i));
}
QueryResult result = query(Simple.class, "content:counter");
assertThat(result.getTotalHits()).isEqualTo(20L);
assertThat(result.getHits()).hasSize(10);
}
@Test
void shouldLimitHitsByConfiguredSize() throws IOException {
try (IndexWriter writer = writer()) {
for (int i = 0; i < 20; i++)
writer.addDocument(simpleDoc("counter " + (i + 1)));
}
QueryResult result = query(Simple.class, "content:counter", null, 2);
assertThat(result.getTotalHits()).isEqualTo(20L);
assertThat(result.getHits()).hasSize(2);
assertContainsValues(
result, "content", "counter 1", "counter 2"
);
}
@Test
void shouldRespectStartValue() throws IOException {
try (IndexWriter writer = writer()) {
for (int i = 0; i < 20; i++)
writer.addDocument(simpleDoc("counter " + (i + 1)));
}
QueryResult result = query(Simple.class, "content:counter", 10, 3);
assertThat(result.getTotalHits()).isEqualTo(20L);
assertThat(result.getHits()).hasSize(3);
assertContainsValues(
result, "content", "counter 11", "counter 12", "counter 13"
);
}
private void assertContainsValues(QueryResult result, String fieldName, Object... expectedValues) {
List<Object> values = result.getHits().stream().map(hit -> {
Hit.ValueField content = (Hit.ValueField) hit.getFields().get(fieldName);
return content.getValue();
}).collect(Collectors.toList());
assertThat(values).containsExactly(expectedValues);
}
@Test
void shouldBeAbleToMarshalQueryResultToJson() throws IOException {
try (IndexWriter writer = writer()) {
writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211"));
}
QueryResult result = query(InetOrgPerson.class, "Arthur");
ObjectMapper mapper = new ObjectMapper();
JsonNode root = mapper.valueToTree(result);
assertThat(root.get("totalHits").asInt()).isOne();
JsonNode hit = root.get("hits").get(0);
assertThat(hit.get("score").asDouble()).isGreaterThan(0d);
JsonNode fields = hit.get("fields");
JsonNode firstName = fields.get("firstName");
assertThat(firstName.get("highlighted").asBoolean()).isFalse();
assertThat(firstName.get("value").asText()).isEqualTo("Arthur");
JsonNode displayName = fields.get("displayName");
assertThat(displayName.get("highlighted").asBoolean()).isTrue();
assertThat(displayName.get("fragments").get(0).asText()).contains("**Arthur**");
}
@Test
void shouldBeAbleToMarshalDifferentTypesOfQueryResultToJson() throws IOException {
Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli());
try (IndexWriter writer = writer()) {
writer.addDocument(typesDoc(21, 42L, true, now));
}
QueryResult result = query(Types.class, "intValue:21");
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
mapper.registerModule(new JavaTimeModule());
JsonNode root = mapper.valueToTree(result);
JsonNode fields = root.get("hits").get(0).get("fields");
assertThat(fields.get("intValue").get("value").asInt()).isEqualTo(21);
assertThat(fields.get("longValue").get("value").asLong()).isEqualTo(42L);
assertThat(fields.get("boolValue").get("value").asBoolean()).isTrue();
assertThat(fields.get("instantValue").get("value").asText()).isEqualTo(now.toString());
}
private QueryResult query(Class<?> type, String queryString) throws IOException {
return query(type, queryString, null, null);
}
private QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
try (DirectoryReader reader = DirectoryReader.open(directory)) {
lenient().when(opener.openForRead("default")).thenReturn(reader);
LuceneQueryBuilder builder = new LuceneQueryBuilder(
opener, "default", new StandardAnalyzer()
);
if (start != null) {
builder.start(start);
}
if (limit != null) {
builder.limit(limit);
}
return builder.execute(type, queryString);
}
}
private IndexWriter writer() throws IOException {
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
return new IndexWriter(directory, config);
}
private Document simpleDoc(String content) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
return document;
}
private Document permissionDoc(String content, String permission) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES));
return document;
}
private Document repositoryDoc(String content, String repository) {
Document document = new Document();
document.add(new TextField("content", content, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES));
return document;
}
private Document inetOrgPersonDoc(String firstName, String lastName, String displayName, String carLicense) {
Document document = new Document();
document.add(new TextField("firstName", firstName, Field.Store.YES));
document.add(new TextField("lastName", lastName, Field.Store.YES));
document.add(new TextField("displayName", displayName, Field.Store.YES));
document.add(new TextField("carLicense", carLicense, Field.Store.YES));
document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, InetOrgPerson.class.getName(), Field.Store.YES));
return document;
}
private Document personDoc(String lastName) {
Document document = new Document();
document.add(new TextField("lastName", lastName, Field.Store.YES));
document.add(new StringField(FieldNames.TYPE, Person.class.getName(), Field.Store.YES));
return document;
}
private Document typesDoc(int intValue, long longValue, boolean boolValue, Instant instantValue) {
Document document = new Document();
document.add(new IntPoint("intValue", intValue));
document.add(new StoredField("intValue", intValue));
document.add(new LongPoint("longValue", longValue));
document.add(new StoredField("longValue", longValue));
document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES));
document.add(new LongPoint("instantValue", instantValue.toEpochMilli()));
document.add(new StoredField("instantValue", instantValue.toEpochMilli()));
document.add(new StringField(FieldNames.TYPE, Types.class.getName(), Field.Store.YES));
return document;
}
static class Types {
@Indexed
private Integer intValue;
@Indexed
private long longValue;
@Indexed
private boolean boolValue;
@Indexed
private Instant instantValue;
}
static class Person {
@Indexed(defaultQuery = true)
private String lastName;
}
static class InetOrgPerson extends Person {
@Indexed(defaultQuery = true, boost = 2f)
private String firstName;
@Indexed(defaultQuery = true, highlighted = true)
private String displayName;
@Indexed
private String carLicense;
}
static class Simple {
@Indexed(defaultQuery = true)
private String content;
}
}