Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-10-26 08:06:09 +01:00
Code Issues Packages Projects Releases Activity

Allow distinct results and projections in queryable stores

Browse Source 
Squash commits of branch feature/distinct:

- Try distinct

- Split projection and distinct flag

- Log change

- Support distinct count
This commit is contained in:
Rene Pfeuffer
2025-06-25 10:45:47 +02:00
parent aa0c62f79a
commit a2d14e7167
8 changed files with 244 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/changelog/distinct_and_project.yaml Normal file
View File

@@ -0,0 +1,2 @@
- type: added
description: Possibility to use distinct and projection in queryable stores

30
scm-core/src/main/java/sonia/scm/store/QueryableStore.java
View File

@@ -22,8 +22,10 @@ import lombok.NoArgsConstructor;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
@@ -123,7 +125,11 @@ public interface QueryableStore<T> extends AutoCloseable {
* @param offset The offset to start the result list.
* @param limit The maximum number of results to return.
*/
List<T_RESULT> findAll(long offset, long limit);
default List<T_RESULT> findAll(long offset, long limit) {
List<T_RESULT> result = new ArrayList<>();
forEach(result::add, offset, limit);
return Collections.unmodifiableList(result);
}
/**
* Calls the given consumer for all objects that match the query.
@@ -143,6 +149,28 @@ public interface QueryableStore<T> extends AutoCloseable {
*/
void forEach(Consumer<T_RESULT> consumer, long offset, long limit);
/**
* Projects the found objects to a specific values. This is useful if you want to retrieve only a subset of the
* fields of the found objects.
* <br/>
* The projection will return an array of objects, where each object corresponds to a field that was specified in
* the {@code fields} parameter. The order of the fields in the array will be the same as the order of the fields in
* the parameter.
*
* @param fields The fields to project.
* @return The query object to continue building the query.
*/
Query<T, Object[], ?> project(QueryField<T, ?>... fields);
/**
* Returns the found objects as a distinct set. This is useful if you want to ensure that no duplicate values are
* returned, for example to determine the unique values of all parent ids. Most likely this is usefull only with
* #project(QueryField[]) to limit the selected "columns".
*
* @return The query object to continue building the query.
*/
Query<T, T_RESULT, ?> distinct();
/**
* Returns the found objects in combination with the parent ids they belong to. This is useful if you are using a
* queryable store that is not scoped to specific parent objects, and you therefore want to know to which parent

59
scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLCountField.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.store.sqlite;
import lombok.Getter;
import java.util.Collection;
import java.util.stream.Collectors;
import static java.util.Collections.emptyList;
/**
* Representation of a SQL COUNT field.
*
* @since 3.9.0
*/
@Getter
class SQLCountField implements SQLNode {
private final Collection<SQLNode> fields;
private final boolean distinct;
public SQLCountField() {
this(emptyList(), false);
}
SQLCountField(Collection<SQLNode> fields, boolean distinct) {
this.fields = fields;
this.distinct = distinct;
}
@Override
public String toSQL() {
StringBuilder sqlBuilder = new StringBuilder("COUNT(");
if (distinct) {
sqlBuilder.append("DISTINCT ");
}
if (fields.isEmpty()) {
sqlBuilder.append("*");
} else {
sqlBuilder.append(fields.stream().map(SQLNode::toSQL).collect(Collectors.joining(", ")));
}
return sqlBuilder.append(")").toString();
}
}

30
scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLSelectStatement.java
View File

@@ -21,17 +21,35 @@ import java.util.stream.Collectors;
class SQLSelectStatement extends ConditionalSQLStatement {
private final List<SQLField> columns;
private final List<SQLNode> columns;
private final SQLTable fromTable;
private final String orderBy;
private final long limit;
private final long offset;
private final boolean distinct;
SQLSelectStatement(List<SQLField> columns, SQLTable fromTable, List<SQLNodeWithValue> whereCondition) {
SQLSelectStatement(List<SQLNode> columns,
SQLTable fromTable,
List<SQLNodeWithValue> whereCondition) {
this(columns, fromTable, whereCondition, null, 0, 0);
}
SQLSelectStatement(List<SQLField> columns, SQLTable fromTable, List<SQLNodeWithValue> whereCondition, String orderBy, long limit, long offset) {
SQLSelectStatement(List<SQLNode> columns,
SQLTable fromTable,
List<SQLNodeWithValue> whereCondition,
String orderBy,
long limit,
long offset) {
this(columns, fromTable, whereCondition, orderBy, limit, offset, false);
}
SQLSelectStatement(List<SQLNode> columns,
SQLTable fromTable,
List<SQLNodeWithValue> whereCondition,
String orderBy,
long limit,
long offset,
boolean distinct) {
super(whereCondition);
if (limit < 0 || offset < 0) {
throw new IllegalArgumentException("limit and offset must be non-negative");
@@ -41,6 +59,7 @@ class SQLSelectStatement extends ConditionalSQLStatement {
this.orderBy = orderBy;
this.limit = limit;
this.offset = offset;
this.distinct = distinct;
}
@Override
@@ -48,9 +67,12 @@ class SQLSelectStatement extends ConditionalSQLStatement {
StringBuilder query = new StringBuilder();
query.append("SELECT ");
if (distinct) {
query.append("DISTINCT ");
}
if (columns != null && !columns.isEmpty()) {
String columnList = columns.stream()
.map(SQLField::toSQL)
.map(SQLNode::toSQL)
.collect(Collectors.joining(", "));
query.append(columnList);
}

4
scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableMutableStore.java
View File

@@ -105,7 +105,7 @@ class SQLiteQueryableMutableStore<T> extends SQLiteQueryableStore<T> implements
@Override
public Map<String, T> getAll() {
List<SQLField> columns = List.of(
List<SQLNode> columns = List.of(
SQLField.PAYLOAD,
new SQLField("ID")
);
@@ -221,7 +221,7 @@ class SQLiteQueryableMutableStore<T> extends SQLiteQueryableStore<T> implements
@Override
public void retain(long n) {
List<SQLField> columns = new ArrayList<>();
List<SQLNode> columns = new ArrayList<>();
addParentIdSQLFields(columns);
List<SQLNodeWithValue> conditions = new ArrayList<>();

77
scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableStore.java
View File

@@ -133,7 +133,7 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
private <R> Collection<R> readAllAs(RowBuilder<R> rowBuilder) {
List<SQLNodeWithValue> parentConditions = new ArrayList<>();
evaluateParentConditions(parentConditions);
List<SQLField> fields = new ArrayList<>();
List<SQLNode> fields = new ArrayList<>();
addParentIdSQLFields(fields);
int parentIdsLength = fields.size() - 1; // addParentIdSQLFields has already added the ID field
fields.add(new SQLField("PAYLOAD"));
@@ -201,7 +201,7 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
@Override
public MaintenanceIterator<T> iterateAll() {
List<SQLField> columns = new LinkedList<>();
List<SQLNode> columns = new LinkedList<>();
columns.add(new SQLField("payload"));
addParentIdSQLFields(columns);
@@ -292,7 +292,7 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
}
}
void addParentIdSQLFields(List<SQLField> fields) {
void addParentIdSQLFields(List<SQLNode> fields) {
for (int i = 0; i < queryableTypeDescriptor.getTypes().length; i++) {
fields.add(new SQLField(computeColumnIdentifier(queryableTypeDescriptor.getTypes()[i])));
}
@@ -343,18 +343,21 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
private final Class<T_RESULT> resultType;
private final Class<T> entityType;
private final Condition<T> condition;
private final QueryField<T, ?>[] projection;
private List<OrderBy<T>> orderBy;
private boolean distinct = false;
SQLiteQuery(Class<T_RESULT> resultType, Condition<T>[] conditions) {
this(resultType, resultType, conjunct(conditions), Collections.emptyList());
this(resultType, resultType, conjunct(conditions), Collections.emptyList(), null);
}
@SuppressWarnings({"rawtypes", "unchecked"})
private SQLiteQuery(Class<T_RESULT> resultType, Class entityType, Condition<T> condition, List<OrderBy<T>> orderBy) {
private SQLiteQuery(Class<T_RESULT> resultType, Class entityType, Condition<T> condition, List<OrderBy<T>> orderBy, QueryField<T, ?>[] projection) {
this.resultType = resultType;
this.entityType = entityType;
this.condition = condition;
this.orderBy = orderBy;
this.projection = projection;
}
private static <T> Condition<T> conjunct(Condition<T>[] conditions) {
@@ -384,13 +387,6 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
}
}
@Override
public List<T_RESULT> findAll(long offset, long limit) {
List<T_RESULT> result = new ArrayList<>();
forEach(result::add, offset, limit);
return Collections.unmodifiableList(result);
}
@Override
public void forEach(Consumer<T_RESULT> consumer, long offset, long limit) {
String orderByString = getOrderByString();
@@ -402,7 +398,8 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
computeCondition(),
orderByString,
limit,
offset
offset,
distinct
);
executeRead(
@@ -418,6 +415,17 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
);
}
@Override
public Query<T, T_RESULT, ?> distinct() {
this.distinct = true;
return this;
}
@Override
public Query<T, Object[], ?> project(QueryField<T, ?>... fields) {
return new SQLiteQuery<>(Object[].class, resultType, condition, orderBy, fields);
}
String getOrderByString() {
StringBuilder orderByBuilder = new StringBuilder();
if (orderBy != null && !orderBy.isEmpty()) {
@@ -429,14 +437,14 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
@Override
@SuppressWarnings("unchecked")
public Query<T, Result<T_RESULT>, ?> withIds() {
return new SQLiteQuery<>((Class<Result<T_RESULT>>) (Class<?>) Result.class, resultType, condition, orderBy);
return new SQLiteQuery<>((Class<Result<T_RESULT>>) (Class<?>) Result.class, resultType, condition, orderBy, null);
}
@Override
public long count() {
SQLSelectStatement sqlStatementQuery =
new SQLSelectStatement(
List.of(new SQLField("COUNT(*)")),
List.of(new SQLCountField(computeFields(), distinct)),
computeFromTable(),
computeCondition()
);
@@ -505,8 +513,15 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
return newOrderBy;
}
private List<SQLField> computeFields() {
List<SQLField> fields = new ArrayList<>();
private List<SQLNode> computeFields() {
if (projection != null && projection.length > 0) {
return computeProjectedFields();
}
return computeDefaultFields();
}
private List<SQLNode> computeDefaultFields() {
List<SQLNode> fields = new ArrayList<>();
fields.add(SQLField.PAYLOAD);
if (resultType.isAssignableFrom(Result.class)) {
addParentIdSQLFields(fields);
@@ -514,6 +529,14 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
return fields;
}
private List<SQLNode> computeProjectedFields() {
return Arrays.stream(projection)
.map(SQLFieldHelper::computeSQLField)
.map(SQLField::new)
.map(field -> (SQLNode) field)
.toList();
}
List<SQLNodeWithValue> computeCondition() {
List<SQLNodeWithValue> conditions = new ArrayList<>();
@@ -556,6 +579,22 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
@SuppressWarnings("unchecked")
private T_RESULT extractResult(ResultSet resultSet) throws JsonProcessingException, SQLException {
if (projection != null && projection.length > 0) {
Object[] values = new Object[projection.length];
for (int i = 0; i < projection.length; i++) {
QueryField<T, ?> field = projection[i];
if (field instanceof QueryableStore.CollectionSizeQueryField<?>) {
values[i] = resultSet.getInt(i + 1);
} else if (field instanceof QueryableStore.MapSizeQueryField<?>) {
values[i] = resultSet.getInt(i + 1);
} else if (field.isIdField()) {
values[i] = resultSet.getString(i + 1);
} else {
values[i] = resultSet.getObject(i + 1);
}
}
return (T_RESULT) values;
}
T entity = objectMapper.readValue(resultSet.getString(1), entityType);
if (resultType.isAssignableFrom(Result.class)) {
Map<String, String> parentIdMapping = new HashMap<>(queryableTypeDescriptor.getTypes().length);
@@ -602,11 +641,11 @@ class SQLiteQueryableStore<T> implements QueryableStore<T>, QueryableMaintenance
private class TemporaryTableMaintenanceIterator implements MaintenanceIterator<T> {
private final PreparedStatement iterateStatement;
private final List<SQLField> columns;
private final List<SQLNode> columns;
private final ResultSet resultSet;
private Boolean hasNext;
public TemporaryTableMaintenanceIterator(List<SQLField> columns) {
public TemporaryTableMaintenanceIterator(List<SQLNode> columns) {
this.columns = columns;
this.hasNext = null;
SQLSelectStatement iterateQuery =

6
scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteRetainStatement.java
View File

@@ -26,12 +26,12 @@ import static java.lang.String.format;
class SQLiteRetainStatement implements SQLNodeWithValue {
private final SQLTable table;
private final List<SQLField> columns;
private final List<SQLNode> columns;
private final SQLSelectStatement selectStatement;
private final List<SQLNodeWithValue> parentConditions;
SQLiteRetainStatement(SQLTable table, List<SQLField> columns, SQLSelectStatement selectStatement, List<SQLNodeWithValue> parentConditions) {
SQLiteRetainStatement(SQLTable table, List<SQLNode> columns, SQLSelectStatement selectStatement, List<SQLNodeWithValue> parentConditions) {
this.table = table;
this.columns = columns;
this.selectStatement = selectStatement;
@@ -57,7 +57,7 @@ class SQLiteRetainStatement implements SQLNodeWithValue {
}
return format("DELETE FROM %s WHERE (%s) NOT IN (%s) %s",
table.toSQL(),
columns.stream().map(SQLField::toSQL).collect(Collectors.joining(",")),
columns.stream().map(SQLNode::toSQL).collect(Collectors.joining(",")),
selectStatement.toSQL(),
parentConditionStatement);
}

67
scm-persistence/src/test/java/sonia/scm/store/sqlite/SQLiteQueryableStoreTest.java
View File

@@ -546,7 +546,7 @@ class SQLiteQueryableStoreTest {
assertThat(all)
.extracting("name")
.containsExactly("arthur","trillian","trish");
.containsExactly("arthur", "trillian", "trish");
}
@Test
@@ -564,7 +564,7 @@ class SQLiteQueryableStoreTest {
System.out.println(all);
assertThat(all)
.extracting("displayName")
.containsExactly("Tricia","Trillian McMillan","Arthur Dent");
.containsExactly("Tricia", "Trillian McMillan", "Arthur Dent");
}
@Test
@@ -909,6 +909,69 @@ class SQLiteQueryableStoreTest {
}
}
@Nested
class Distinct {
@BeforeEach
void fillData() {
new StoreTestBuilder(connectionString, "sonia.Group")
.withIds("42")
.put("1", new User("trillian", "Trillian McMillan", ""));
new StoreTestBuilder(connectionString, "sonia.Group")
.withIds("42")
.put("2", new User("zaphod", "Zaphod Beeblebrox", ""));
new StoreTestBuilder(connectionString, "sonia.Group")
.withIds("42")
.put("3", new User("marvin", "Marvin", ""));
new StoreTestBuilder(connectionString, "sonia.Group")
.withIds("23")
.put("1", new User("dent", "Arthur Dent", ""));
new StoreTestBuilder(connectionString, "sonia.Group")
.withIds("23")
.put("2", new User("trillian", "Trillian McMillan", ""));
}
@Test
void shouldReturnDistinctValuesFromObject() {
SQLiteQueryableMutableStore<User> store = new StoreTestBuilder(connectionString, "sonia.Group")
.withIds();
List<Object[]> result = store.query().project(USER_NAME).distinct().findAll();
assertThat(result)
.containsExactlyInAnyOrder(
new String[]{"trillian"},
new String[]{"zaphod"},
new String[]{"marvin"},
new String[]{"dent"}
);
}
@Test
void shouldReturnDistinctParentIds() {
SQLiteQueryableMutableStore<User> store = new StoreTestBuilder(connectionString, "sonia.Group")
.withIds();
List<Object[]> result = store.query().project(GROUP).distinct().findAll();
assertThat(result)
.containsExactlyInAnyOrder(
new String[]{"42"},
new String[]{"23"}
);
}
@Test
void shouldReturnDistinctCount() {
SQLiteQueryableMutableStore<User> store = new StoreTestBuilder(connectionString, "sonia.Group")
.withIds();
long count = store.query().project(GROUP).distinct().count();
assertThat(count).isEqualTo(2);
}
}
@Nested
class ForMaintenance {
@Test