From 7d485d732ab3071330dfcb93cbb8de340ead8658 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Tue, 28 Feb 2023 10:58:47 +0100 Subject: [PATCH] Add support for scrolling using offset- and keyset-based strategies. We now support scrolling through large query results using ScrollPosition and Window's of data. See: #4308 Original Pull Request: #4317 --- .../data/mongodb/core/EntityOperations.java | 36 ++++- .../mongodb/core/ExecutableFindOperation.java | 26 +++- .../core/ExecutableFindOperationSupport.java | 14 +- .../data/mongodb/core/MongoOperations.java | 49 +++++- .../data/mongodb/core/MongoTemplate.java | 92 +++++++++-- .../data/mongodb/core/QueryOperations.java | 3 +- .../mongodb/core/ReactiveFindOperation.java | 22 ++- .../core/ReactiveFindOperationSupport.java | 7 + .../mongodb/core/ReactiveMongoOperations.java | 48 +++++- .../mongodb/core/ReactiveMongoTemplate.java | 90 +++++++++-- .../data/mongodb/core/ScrollUtils.java | 145 +++++++++++++++++ .../data/mongodb/core/query/Query.java | 78 ++++++++++ .../repository/query/AbstractMongoQuery.java | 3 + .../query/AbstractReactiveMongoQuery.java | 3 + .../query/ConvertingParameterAccessor.java | 8 +- .../repository/query/MongoQueryMethod.java | 7 +- .../query/ReactiveMongoQueryMethod.java | 18 ++- .../support/FetchableFluentQuerySupport.java | 34 ++-- .../QuerydslMongoPredicateExecutor.java | 19 ++- .../support/ReactiveFluentQuerySupport.java | 24 ++- ...eactiveQuerydslMongoPredicateExecutor.java | 20 ++- .../ReactiveSpringDataMongodbQuery.java | 11 +- .../support/SimpleMongoRepository.java | 19 ++- .../SimpleReactiveMongoRepository.java | 22 ++- .../support/SpringDataMongodbQuery.java | 17 +- .../data/mongodb/util/BsonUtils.java | 29 ++-- .../core/MongoTemplateScrollTests.java | 147 ++++++++++++++++++ .../data/mongodb/core/MongoTemplateTests.java | 1 - .../ReactiveMongoTemplateScrollTests.java | 144 +++++++++++++++++ ...tractPersonRepositoryIntegrationTests.java | 75 ++++++++- .../mongodb/repository/PersonRepository.java | 26 +++- .../mongodb/repository/PersonSummaryDto.java | 9 ++ .../ReactiveMongoRepositoryTests.java | 63 +++++++- .../query/StubParameterAccessor.java | 6 + src/main/asciidoc/index.adoc | 2 + src/main/asciidoc/reference/mongodb.adoc | 1 + 36 files changed, 1196 insertions(+), 122 deletions(-) create mode 100644 spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java create mode 100644 spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java create mode 100644 spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java index db07166b5..16d084a67 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java @@ -17,6 +17,7 @@ package org.springframework.data.mongodb.core; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -44,6 +45,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.timeseries.Granularity; import org.springframework.data.mongodb.core.validation.Validator; +import org.springframework.data.mongodb.util.BsonUtils; import org.springframework.data.projection.EntityProjection; import org.springframework.data.projection.EntityProjectionIntrospector; import org.springframework.data.projection.ProjectionFactory; @@ -454,6 +456,9 @@ class EntityOperations { * @since 2.1.2 */ boolean isNew(); + + Map extractKeys(Document sortObject); + } /** @@ -475,7 +480,7 @@ class EntityOperations { T populateIdIfNecessary(@Nullable Object id); /** - * Initializes the version property of the of the current entity if available. + * Initializes the version property of the current entity if available. * * @return the entity with the version property updated if available. */ @@ -567,6 +572,19 @@ class EntityOperations { public boolean isNew() { return map.get(ID_FIELD) != null; } + + @Override + public Map extractKeys(Document sortObject) { + + Map keyset = new LinkedHashMap<>(); + keyset.put(ID_FIELD, getId()); + + for (String key : sortObject.keySet()) { + keyset.put(key, BsonUtils.resolveValue(map, key)); + } + + return keyset; + } } private static class SimpleMappedEntity> extends UnmappedEntity { @@ -701,6 +719,22 @@ class EntityOperations { public boolean isNew() { return entity.isNew(propertyAccessor.getBean()); } + + @Override + public Map extractKeys(Document sortObject) { + + Map keyset = new LinkedHashMap<>(); + keyset.put(entity.getRequiredIdProperty().getName(), getId()); + + for (String key : sortObject.keySet()) { + + // TODO: make this work for nested properties + MongoPersistentProperty persistentProperty = entity.getRequiredPersistentProperty(key); + keyset.put(key, propertyAccessor.getProperty(persistentProperty)); + } + + return keyset; + } } private static class AdaptibleMappedEntity extends MappedEntity implements AdaptibleEntity { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java index 0b7616fa3..e8b38c8e8 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java @@ -20,6 +20,8 @@ import java.util.Optional; import java.util.stream.Stream; import org.springframework.dao.DataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -124,12 +126,24 @@ public interface ExecutableFindOperation { Stream stream(); /** - * Get the number of matching elements. - *
- * This method uses an {@link com.mongodb.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) aggregation - * execution} even for empty {@link Query queries} which may have an impact on performance, but guarantees shard, - * session and transaction compliance. In case an inaccurate count satisfies the applications needs use - * {@link MongoOperations#estimatedCount(String)} for empty queries instead. + * Return a scroll of elements either starting or resuming at + * {@link org.springframework.data.domain.ScrollPosition}. + * + * @param scrollPosition the scroll position. + * @return a scroll of the resulting elements. + * @since 4.1 + * @see org.springframework.data.domain.OffsetScrollPosition + * @see org.springframework.data.domain.KeysetScrollPosition + */ + Scroll scroll(ScrollPosition scrollPosition); + + /** + * Get the number of matching elements.
+ * This method uses an + * {@link com.mongodb.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) + * aggregation execution} even for empty {@link Query queries} which may have an impact on performance, but + * guarantees shard, session and transaction compliance. In case an inaccurate count satisfies the applications + * needs use {@link MongoOperations#estimatedCount(String)} for empty queries instead. * * @return total number of matching elements. */ diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java index 1038ee232..81d7557e7 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java @@ -21,6 +21,8 @@ import java.util.stream.Stream; import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.query.NearQuery; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.SerializationUtils; @@ -71,8 +73,8 @@ class ExecutableFindOperationSupport implements ExecutableFindOperation { private final @Nullable String collection; private final Query query; - ExecutableFindSupport(MongoTemplate template, Class domainType, Class returnType, - @Nullable String collection, Query query) { + ExecutableFindSupport(MongoTemplate template, Class domainType, Class returnType, @Nullable String collection, + Query query) { this.template = template; this.domainType = domainType; this.returnType = returnType; @@ -138,6 +140,11 @@ class ExecutableFindOperationSupport implements ExecutableFindOperation { return doStream(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); + } + @Override public TerminatingFindNear near(NearQuery nearQuery) { return () -> template.geoNear(nearQuery, domainType, getCollectionName(), returnType); @@ -168,8 +175,7 @@ class ExecutableFindOperationSupport implements ExecutableFindOperation { Document fieldsObject = query.getFieldsObject(); return template.doFind(template.createDelegate(query), getCollectionName(), queryObject, fieldsObject, domainType, - returnType, - getCursorPreparer(query, preparer)); + returnType, getCursorPreparer(query, preparer)); } private List doFindDistinct(String field) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java index 4727c0b8d..cd019aae6 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java @@ -23,6 +23,8 @@ import java.util.function.Supplier; import java.util.stream.Stream; import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.BulkOperations.BulkMode; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -319,7 +321,8 @@ public interface MongoOperations extends FluentMongoOperations { * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - MongoCollection createView(String name, Class source, AggregationPipeline pipeline, @Nullable ViewOptions options); + MongoCollection createView(String name, Class source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * Create a view with the provided name. The view content is defined by the {@link AggregationPipeline pipeline} on @@ -331,7 +334,8 @@ public interface MongoOperations extends FluentMongoOperations { * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - MongoCollection createView(String name, String source, AggregationPipeline pipeline, @Nullable ViewOptions options); + MongoCollection createView(String name, String source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * A set of collection names. @@ -802,6 +806,45 @@ public interface MongoOperations extends FluentMongoOperations { */ List find(Query query, Class entityClass, String collectionName); + /** + * Query for a scroll window of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @return the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Scroll scroll(Query query, Class entityType); + + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @param collectionName name of the collection to retrieve the objects from. + * @return the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Scroll scroll(Query query, Class entityType, String collectionName); + /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be * derived from the given target class as well. @@ -1175,7 +1218,7 @@ public interface MongoOperations extends FluentMongoOperations { * @param entityClass class that determines the collection to use. Must not be {@literal null}. * @return the count of matching documents. * @throws org.springframework.data.mapping.MappingException if the collection name cannot be - * {@link #getCollectionName(Class) derived} from the given type. + * {@link #getCollectionName(Class) derived} from the given type. * @see #exactCount(Query, Class) * @see #estimatedCount(Class) */ diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index 2d0ff734a..007c20f10 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -44,6 +44,8 @@ import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.GeoResults; @@ -64,6 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext; import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -847,6 +850,48 @@ public class MongoTemplate new QueryCursorPreparer(query, entityClass)); } + @Override + public Scroll scroll(Query query, Class entityType) { + + Assert.notNull(entityType, "Entity type must not be null"); + + return scroll(query, entityType, getCollectionName(entityType)); + } + + @Override + public Scroll scroll(Query query, Class entityType, String collectionName) { + return doScroll(query, entityType, entityType, collectionName); + } + + Scroll doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + + Assert.notNull(query, "Query must not be null"); + Assert.notNull(collectionName, "CollectionName must not be null"); + Assert.notNull(sourceClass, "Entity type must not be null"); + Assert.notNull(targetClass, "Target type must not be null"); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(mongoConverter, targetClass, collectionName); + int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; + + if (query.hasKeyset()) { + + KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + operations.getIdPropertyName(sourceClass)); + + List result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(), + keysetPaginationQuery.fields(), sourceClass, + new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback); + + return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, operations); + } + + List result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(), + sourceClass, new QueryCursorPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass), + callback); + + return ScrollUtils.createWindow(result, query.getLimit(), OffsetScrollPosition.positionFunction(query.getSkip())); + } + @Nullable @Override public T findById(Object id, Class entityClass) { @@ -953,7 +998,7 @@ public class MongoTemplate optionsBuilder.readPreference(near.getReadPreference()); } - if(near.hasReadConcern()) { + if (near.hasReadConcern()) { optionsBuilder.readConcern(near.getReadConcern()); } @@ -2837,13 +2882,24 @@ public class MongoTemplate return converter; } - private Document getMappedSortObject(Query query, Class type) { + @Nullable + private Document getMappedSortObject(@Nullable Query query, Class type) { - if (query == null || ObjectUtils.isEmpty(query.getSortObject())) { + if (query == null) { return null; } - return queryMapper.getMappedSort(query.getSortObject(), mappingContext.getPersistentEntity(type)); + return getMappedSortObject(query.getSortObject(), type); + } + + @Nullable + private Document getMappedSortObject(Document sortObject, Class type) { + + if (ObjectUtils.isEmpty(sortObject)) { + return null; + } + + return queryMapper.getMappedSort(sortObject, mappingContext.getPersistentEntity(type)); } /** @@ -3199,11 +3255,23 @@ public class MongoTemplate class QueryCursorPreparer implements CursorPreparer { private final Query query; + + private final Document sortObject; + + private final int limit; + + private final long skip; private final @Nullable Class type; QueryCursorPreparer(Query query, @Nullable Class type) { + this(query, query.getSortObject(), query.getLimit(), query.getSkip(), type); + } + QueryCursorPreparer(Query query, Document sortObject, int limit, long skip, @Nullable Class type) { this.query = query; + this.sortObject = sortObject; + this.limit = limit; + this.skip = skip; this.type = type; } @@ -3218,20 +3286,20 @@ public class MongoTemplate Meta meta = query.getMeta(); HintFunction hintFunction = HintFunction.from(query.getHint()); - if (query.getSkip() <= 0 && query.getLimit() <= 0 && ObjectUtils.isEmpty(query.getSortObject()) - && hintFunction.isEmpty() && !meta.hasValues() && query.getCollation().isEmpty()) { + if (skip <= 0 && limit <= 0 && ObjectUtils.isEmpty(sortObject) && hintFunction.isEmpty() && !meta.hasValues() + && query.getCollation().isEmpty()) { return cursorToUse; } try { - if (query.getSkip() > 0) { - cursorToUse = cursorToUse.skip((int) query.getSkip()); + if (skip > 0) { + cursorToUse = cursorToUse.skip((int) skip); } - if (query.getLimit() > 0) { - cursorToUse = cursorToUse.limit(query.getLimit()); + if (limit > 0) { + cursorToUse = cursorToUse.limit(limit); } - if (!ObjectUtils.isEmpty(query.getSortObject())) { - Document sort = type != null ? getMappedSortObject(query, type) : query.getSortObject(); + if (!ObjectUtils.isEmpty(sortObject)) { + Document sort = type != null ? getMappedSortObject(sortObject, type) : sortObject; cursorToUse = cursorToUse.sort(sort); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java index 05aeda069..4e8c5f63d 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.bson.BsonValue; import org.bson.Document; import org.bson.codecs.Codec; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.springframework.data.mapping.PropertyPath; import org.springframework.data.mapping.PropertyReferenceException; @@ -776,7 +777,7 @@ class QueryOperations { Document filterWithShardKey = new Document(filter); getMappedShardKeyFields(domainType) - .forEach(key -> filterWithShardKey.putIfAbsent(key, BsonUtils.resolveValue(shardKeySource, key))); + .forEach(key -> filterWithShardKey.putIfAbsent(key, BsonUtils.resolveValue((Bson) shardKeySource, key))); return filterWithShardKey; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java index 31de934ec..1a81f92a9 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java @@ -18,6 +18,8 @@ package org.springframework.data.mongodb.core; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -87,14 +89,23 @@ public interface ReactiveFindOperation { */ Flux all(); + /** + * Return a scroll of elements either starting or resuming at {@link ScrollPosition}. + * + * @param scrollPosition the scroll position. + * @return a scroll of the resulting elements. + * @since 4.1 + * @see org.springframework.data.domain.OffsetScrollPosition + * @see org.springframework.data.domain.KeysetScrollPosition + */ + Mono> scroll(ScrollPosition scrollPosition); + /** * Get all matching elements using a {@link com.mongodb.CursorType#TailableAwait tailable cursor}. The stream will * not be completed unless the {@link org.reactivestreams.Subscription} is - * {@link org.reactivestreams.Subscription#cancel() canceled}. - *
+ * {@link org.reactivestreams.Subscription#cancel() canceled}.
* However, the stream may become dead, or invalid, if either the query returns no match or the cursor returns the - * document at the "end" of the collection and then the application deletes that document. - *
+ * document at the "end" of the collection and then the application deletes that document.
* A stream that is no longer in use must be {@link reactor.core.Disposable#dispose()} disposed} otherwise the * streams will linger and exhaust resources.
* NOTE: Requires a capped collection. @@ -105,8 +116,7 @@ public interface ReactiveFindOperation { Flux tail(); /** - * Get the number of matching elements. - *
+ * Get the number of matching elements.
* This method uses an * {@link com.mongodb.reactivestreams.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) * aggregation execution} even for empty {@link Query queries} which may have an impact on performance, but diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java index 4dcf62aac..13894c896 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java @@ -20,6 +20,8 @@ import reactor.core.publisher.Mono; import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.CollectionPreparerSupport.ReactiveCollectionPreparerDelegate; import org.springframework.data.mongodb.core.query.NearQuery; import org.springframework.data.mongodb.core.query.Query; @@ -137,6 +139,11 @@ class ReactiveFindOperationSupport implements ReactiveFindOperation { return doFind(null); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); + } + @Override public Flux tail() { return doFind(template.new TailingQueryFindPublisherPreparer(query, domainType)); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java index 323ca9dd9..d252cb4a4 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java @@ -25,7 +25,8 @@ import java.util.function.Supplier; import org.bson.Document; import org.reactivestreams.Publisher; import org.reactivestreams.Subscription; - +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.ReactiveMongoDatabaseFactory; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -279,7 +280,8 @@ public interface ReactiveMongoOperations extends ReactiveFluentMongoOperations { * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - Mono> createView(String name, Class source, AggregationPipeline pipeline, @Nullable ViewOptions options); + Mono> createView(String name, Class source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * Create a view with the provided name. The view content is defined by the {@link AggregationPipeline pipeline} on @@ -291,7 +293,8 @@ public interface ReactiveMongoOperations extends ReactiveFluentMongoOperations { * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - Mono> createView(String name, String source, AggregationPipeline pipeline, @Nullable ViewOptions options); + Mono> createView(String name, String source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * A set of collection names. @@ -462,6 +465,45 @@ public interface ReactiveMongoOperations extends ReactiveFluentMongoOperations { */ Flux find(Query query, Class entityClass, String collectionName); + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @return {@link Mono} emitting the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Mono> scroll(Query query, Class entityType); + + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @param collectionName name of the collection to retrieve the objects from. + * @return {@link Mono} emitting the converted scroll window. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Mono> scroll(Query query, Class entityType, String collectionName); + /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be * derived from the given target class as well. diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java index ee3d40ef3..6b4266a57 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java @@ -58,6 +58,8 @@ import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.Metric; @@ -78,6 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext; import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -826,6 +829,49 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati query.getFieldsObject(), entityClass, new QueryFindPublisherPreparer(query, entityClass)); } + @Override + public Mono> scroll(Query query, Class entityType) { + + Assert.notNull(entityType, "Entity type must not be null"); + + return scroll(query, entityType, getCollectionName(entityType)); + } + + @Override + public Mono> scroll(Query query, Class entityType, String collectionName) { + return doScroll(query, entityType, entityType, collectionName); + } + + Mono> doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + + Assert.notNull(query, "Query must not be null"); + Assert.notNull(collectionName, "CollectionName must not be null"); + Assert.notNull(sourceClass, "Entity type must not be null"); + Assert.notNull(targetClass, "Target type must not be null"); + + int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; + + if (query.hasKeyset()) { + + KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + operations.getIdPropertyName(sourceClass)); + + Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), + keysetPaginationQuery.query(), keysetPaginationQuery.fields(), targetClass, + new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass)).collectList(); + + return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, operations)); + } + + Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(), + query.getFieldsObject(), targetClass, + new QueryFindPublisherPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass)) + .collectList(); + + return result.map( + it -> ScrollUtils.createWindow(it, query.getLimit(), OffsetScrollPosition.positionFunction(query.getSkip()))); + } + @Override public Mono findById(Object id, Class entityClass) { return findById(id, entityClass, getCollectionName(entityClass)); @@ -1004,7 +1050,7 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati optionsBuilder.readPreference(near.getReadPreference()); } - if(near.hasReadConcern()) { + if (near.hasReadConcern()) { optionsBuilder.readConcern(near.getReadConcern()); } @@ -2652,13 +2698,24 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati return converter; } + @Nullable private Document getMappedSortObject(Query query, Class type) { if (query == null) { return null; } - return queryMapper.getMappedSort(query.getSortObject(), mappingContext.getPersistentEntity(type)); + return getMappedSortObject(query.getSortObject(), type); + } + + @Nullable + private Document getMappedSortObject(Document sortObject, Class type) { + + if (ObjectUtils.isEmpty(sortObject)) { + return null; + } + + return queryMapper.getMappedSort(sortObject, mappingContext.getPersistentEntity(type)); } // Callback implementations @@ -3081,11 +3138,24 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati class QueryFindPublisherPreparer implements FindPublisherPreparer { private final Query query; + + private final Document sortObject; + + private final int limit; + + private final long skip; private final @Nullable Class type; QueryFindPublisherPreparer(Query query, @Nullable Class type) { + this(query, query.getSortObject(), query.getLimit(), query.getSkip(), type); + } + + QueryFindPublisherPreparer(Query query, Document sortObject, int limit, long skip, @Nullable Class type) { this.query = query; + this.sortObject = sortObject; + this.limit = limit; + this.skip = skip; this.type = type; } @@ -3100,23 +3170,23 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati HintFunction hintFunction = HintFunction.from(query.getHint()); Meta meta = query.getMeta(); - if (query.getSkip() <= 0 && query.getLimit() <= 0 && ObjectUtils.isEmpty(query.getSortObject()) - && hintFunction.isEmpty() && !meta.hasValues()) { + if (skip <= 0 && limit <= 0 && ObjectUtils.isEmpty(sortObject) && hintFunction.isEmpty() + && !meta.hasValues()) { return findPublisherToUse; } try { - if (query.getSkip() > 0) { - findPublisherToUse = findPublisherToUse.skip((int) query.getSkip()); + if (skip > 0) { + findPublisherToUse = findPublisherToUse.skip((int) skip); } - if (query.getLimit() > 0) { - findPublisherToUse = findPublisherToUse.limit(query.getLimit()); + if (limit > 0) { + findPublisherToUse = findPublisherToUse.limit(limit); } - if (!ObjectUtils.isEmpty(query.getSortObject())) { - Document sort = type != null ? getMappedSortObject(query, type) : query.getSortObject(); + if (!ObjectUtils.isEmpty(sortObject)) { + Document sort = type != null ? getMappedSortObject(sortObject, type) : sortObject; findPublisherToUse = findPublisherToUse.sort(sort); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java new file mode 100644 index 000000000..6458fa9d2 --- /dev/null +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -0,0 +1,145 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.IntFunction; + +import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.mongodb.core.EntityOperations.Entity; +import org.springframework.data.mongodb.core.query.Query; + +/** + * Utilities to run scroll queries and create {@link Scroll} results. + * + * @author Mark Paluch + * @since 4.1 + */ +class ScrollUtils { + + /** + * Create the actual query to run keyset-based pagination. Affects projection, sorting, and the criteria. + * + * @param query + * @param idPropertyName + * @return + */ + static KeySetCursorQuery createKeysetPaginationQuery(Query query, String idPropertyName) { + + Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); + sortObject.put(idPropertyName, 1); + + // make sure we can extract the keyset + Document fieldsObject = query.getFieldsObject(); + if (!fieldsObject.isEmpty()) { + for (String field : sortObject.keySet()) { + fieldsObject.put(field, 1); + } + } + + Document queryObject = query.getQueryObject(); + + List or = (List) queryObject.getOrDefault("$or", new ArrayList<>()); + + // TODO: reverse scrolling + Map keysetValues = query.getKeyset().getKeys(); + Document keysetSort = new Document(); + List sortKeys = new ArrayList<>(sortObject.keySet()); + + if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) { + throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values"); + } + + // first query doesn't come with a keyset + if (!keysetValues.isEmpty()) { + + // build matrix query for keyset paging that contains sort^2 queries + // reflecting a query that follows sort order semantics starting from the last returned keyset + for (int i = 0; i < sortKeys.size(); i++) { + + Document sortConstraint = new Document(); + + for (int j = 0; j < sortKeys.size(); j++) { + + String sortSegment = sortKeys.get(j); + int sortOrder = sortObject.getInteger(sortSegment); + Object o = keysetValues.get(sortSegment); + + if (j >= i) { // tail segment + sortConstraint.put(sortSegment, new Document(sortOrder == 1 ? "$gt" : "$lt", o)); + break; + } + + sortConstraint.put(sortSegment, o); + } + + if (!sortConstraint.isEmpty()) { + or.add(sortConstraint); + } + } + } + + if (!keysetSort.isEmpty()) { + or.add(keysetSort); + } + if (!or.isEmpty()) { + queryObject.put("$or", or); + } + + return new KeySetCursorQuery(queryObject, fieldsObject, sortObject); + } + + static Scroll createWindow(Document sortObject, int limit, List result, EntityOperations operations) { + + IntFunction positionFunction = value -> { + + T last = result.get(value); + Entity entity = operations.forEntity(last); + + Map keys = entity.extractKeys(sortObject); + return KeysetScrollPosition.of(keys); + }; + + return createWindow(result, limit, positionFunction); + } + + static Scroll createWindow(List result, int limit, IntFunction positionFunction) { + return Scroll.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit)); + } + + static boolean hasMoreElements(List result, int limit) { + return !result.isEmpty() && result.size() > limit; + } + + static List getSubList(List result, int limit) { + + if (limit > 0 && result.size() > limit) { + return result.subList(0, limit); + } + + return result; + } + + record KeySetCursorQuery(Document query, Document fields, Document sort) { + + } + +} diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java index 912e7d5ce..e631852a4 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java @@ -30,7 +30,10 @@ import java.util.Optional; import java.util.Set; import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; import org.springframework.data.mongodb.InvalidMongoDbApiUsageException; @@ -64,6 +67,8 @@ public class Query implements ReadConcernAware, ReadPreferenceAware { private Sort sort = Sort.unsorted(); private long skip; private int limit; + + private KeysetScrollPosition keysetScrollPosition; private @Nullable ReadConcern readConcern; private @Nullable ReadPreference readPreference; @@ -255,6 +260,67 @@ public class Query implements ReadConcernAware, ReadPreferenceAware { return with(pageable.getSort()); } + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently set {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(ScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + if (position instanceof OffsetScrollPosition offset) { + return with(offset); + } + + if (position instanceof KeysetScrollPosition keyset) { + return with(keyset); + } + + throw new IllegalArgumentException(String.format("ScrollPosition %s not supported", position)); + } + + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently set {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(OffsetScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + this.skip = position.getOffset(); + this.keysetScrollPosition = null; + return this; + } + + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently reset {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(KeysetScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + this.skip = 0; + this.keysetScrollPosition = position; + + return this; + } + + public boolean hasKeyset() { + return keysetScrollPosition != null; + } + + @Nullable + public KeysetScrollPosition getKeyset() { + return keysetScrollPosition; + } + /** * Adds a {@link Sort} to the {@link Query} instance. * @@ -384,11 +450,22 @@ public class Query implements ReadConcernAware, ReadPreferenceAware { return this.skip; } + /** + * Returns whether the query is {@link #limit(int) limited}. + * + * @return {@code true} if the query is limited; {@code false} otherwise. + * @since 4.1 + */ + public boolean isLimited() { + return this.limit > 0; + } + /** * Get the maximum number of documents to be return. {@literal Zero} or a {@literal negative} value indicates no * limit. * * @return number of documents to return. + * @see #isLimited() */ public int getLimit() { return this.limit; @@ -688,4 +765,5 @@ public class Query implements ReadConcernAware, ReadPreferenceAware { public static boolean isRestrictedTypeKey(String key) { return RESTRICTED_TYPES_KEY.equals(key); } + } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java index 823b64d32..930a73331 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java @@ -167,6 +167,9 @@ public abstract class AbstractMongoQuery implements RepositoryQuery { return q -> operation.matching(q).stream(); } else if (method.isCollectionQuery()) { return q -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())).all(); + } else if (method.isScrollQuery()) { + return q -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())) + .scroll(accessor.getScrollPosition()); } else if (method.isPageQuery()) { return new PagedExecution(operation, accessor.getPageable()); } else if (isCountQuery()) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java index ed1fee68c..fbb078b43 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java @@ -203,6 +203,9 @@ public abstract class AbstractReactiveMongoQuery implements RepositoryQuery { return (q, t, c) -> operation.matching(q.with(accessor.getPageable())).tail(); } else if (method.isCollectionQuery()) { return (q, t, c) -> operation.matching(q.with(accessor.getPageable())).all(); + } else if (method.isScrollQuery()) { + return (q, t, c) -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())) + .scroll(accessor.getScrollPosition()); } else if (isCountQuery()) { return (q, t, c) -> operation.matching(q).count(); } else if (isExistsQuery()) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java index 21513efbf..b3ecef985 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java @@ -23,6 +23,7 @@ import java.util.List; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Distance; import org.springframework.data.geo.Point; @@ -71,6 +72,11 @@ public class ConvertingParameterAccessor implements MongoParameterAccessor { return new ConvertingIterator(delegate.iterator()); } + @Override + public ScrollPosition getScrollPosition() { + return delegate.getScrollPosition(); + } + public Pageable getPageable() { return delegate.getPageable(); } @@ -197,7 +203,7 @@ public class ConvertingParameterAccessor implements MongoParameterAccessor { if (source instanceof Iterable) { - if(source instanceof Collection) { + if (source instanceof Collection) { return new ArrayList<>((Collection) source); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java index 451e74338..1b8f6b6a5 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java @@ -335,8 +335,8 @@ public class MongoQueryMethod extends QueryMethod { public String getAnnotatedCollation() { return doFindAnnotation(Collation.class).map(Collation::value) // - .orElseThrow(() -> new IllegalStateException( - "Expected to find @Collation annotation but did not; Make sure to check hasAnnotatedCollation() before.")); + .orElseThrow(() -> new IllegalStateException( + "Expected to find @Collation annotation but did not; Make sure to check hasAnnotatedCollation() before.")); } /** @@ -420,7 +420,8 @@ public class MongoQueryMethod extends QueryMethod { if (isModifyingQuery()) { - if (isCollectionQuery() || isSliceQuery() || isPageQuery() || isGeoNearQuery() || !isNumericOrVoidReturnValue()) { // + if (isCollectionQuery() || isScrollQuery() || isSliceQuery() || isPageQuery() || isGeoNearQuery() + || !isNumericOrVoidReturnValue()) { // throw new IllegalStateException( String.format("Update method may be void or return a numeric value (the number of updated documents)." + "Offending method: %s", method)); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java index a64822a26..904c184a8 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java @@ -66,7 +66,7 @@ public class ReactiveMongoQueryMethod extends MongoQueryMethod { super(method, metadata, projectionFactory, mappingContext); this.method = method; - this.isCollectionQuery = Lazy.of(() -> (!(isPageQuery() || isSliceQuery()) + this.isCollectionQuery = Lazy.of(() -> (!(isPageQuery() || isSliceQuery() || isScrollQuery()) && ReactiveWrappers.isMultiValueType(metadata.getReturnType(method).getType()) || super.isCollectionQuery())); } @@ -136,7 +136,16 @@ public class ReactiveMongoQueryMethod extends MongoQueryMethod { boolean multiWrapper = ReactiveWrappers.isMultiValueType(returnType.getType()); boolean singleWrapperWithWrappedPageableResult = ReactiveWrappers.isSingleValueType(returnType.getType()) && (PAGE_TYPE.isAssignableFrom(returnType.getRequiredComponentType()) - || SLICE_TYPE.isAssignableFrom(returnType.getRequiredComponentType())); + || SLICE_TYPE.isAssignableFrom(returnType.getRequiredComponentType())); + + if (hasParameterOfType(method, Sort.class)) { + throw new IllegalStateException(String.format("Method must not have Pageable *and* Sort parameter;" + + " Use sorting capabilities on Pageable instead; Offending method: %s", method)); + } + + if (isScrollQuery()) { + return; + } if (singleWrapperWithWrappedPageableResult) { throw new InvalidDataAccessApiUsageException( @@ -149,11 +158,6 @@ public class ReactiveMongoQueryMethod extends MongoQueryMethod { "Method has to use a either multi-item reactive wrapper return type or a wrapped Page/Slice type; Offending method: %s", method.toString())); } - - if (hasParameterOfType(method, Sort.class)) { - throw new IllegalStateException(String.format("Method must not have Pageable *and* Sort parameter;" - + " Use sorting capabilities on Pageable instead; Offending method: %s", method)); - } } super.verify(); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java index 74abf97ab..79d26a524 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java @@ -33,17 +33,21 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl private final P predicate; private final Sort sort; + + private final int limit; + private final Class resultType; private final List fieldsToInclude; - FetchableFluentQuerySupport(P predicate, Sort sort, Class resultType, List fieldsToInclude) { + FetchableFluentQuerySupport(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { this.predicate = predicate; this.sort = sort; + this.limit = limit; this.resultType = resultType; this.fieldsToInclude = fieldsToInclude; } - /* + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#sortBy(org.springframework.data.domain.Sort) */ @@ -52,10 +56,18 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl Assert.notNull(sort, "Sort must not be null"); - return create(predicate, sort, resultType, fieldsToInclude); + return create(predicate, sort, limit, resultType, fieldsToInclude); } - /* + @Override + public FluentQuery.FetchableFluentQuery limit(int limit) { + + Assert.isTrue(limit > 0, "Limit must be greater zero"); + + return create(predicate, sort, limit, resultType, fieldsToInclude); + } + + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#as(java.lang.Class) */ @@ -64,10 +76,10 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl Assert.notNull(projection, "Projection target type must not be null"); - return create(predicate, sort, projection, fieldsToInclude); + return create(predicate, sort, limit, projection, fieldsToInclude); } - /* + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#project(java.util.Collection) */ @@ -76,11 +88,11 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl Assert.notNull(properties, "Projection properties must not be null"); - return create(predicate, sort, resultType, new ArrayList<>(properties)); + return create(predicate, sort, limit, resultType, new ArrayList<>(properties)); } - protected abstract FetchableFluentQuerySupport create(P predicate, Sort sort, Class resultType, - List fieldsToInclude); + protected abstract FetchableFluentQuerySupport create(P predicate, Sort sort, int limit, + Class resultType, List fieldsToInclude); P getPredicate() { return predicate; @@ -90,6 +102,10 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl return sort; } + int getLimit() { + return limit; + } + Class getResultType() { return resultType; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java index 0399dfd5c..4cf3ab3fe 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java @@ -25,6 +25,8 @@ import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.query.BasicQuery; @@ -228,17 +230,17 @@ public class QuerydslMongoPredicateExecutor extends QuerydslPredicateExecutor class FluentQuerydsl extends FetchableFluentQuerySupport { FluentQuerydsl(Predicate predicate, Class resultType) { - this(predicate, Sort.unsorted(), resultType, Collections.emptyList()); + this(predicate, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - FluentQuerydsl(Predicate predicate, Sort sort, Class resultType, List fieldsToInclude) { - super(predicate, sort, resultType, fieldsToInclude); + FluentQuerydsl(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { + super(predicate, sort, limit, resultType, fieldsToInclude); } @Override - protected FluentQuerydsl create(Predicate predicate, Sort sort, Class resultType, + protected FluentQuerydsl create(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new FluentQuerydsl<>(predicate, sort, resultType, fieldsToInclude); + return new FluentQuerydsl<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -256,6 +258,11 @@ public class QuerydslMongoPredicateExecutor extends QuerydslPredicateExecutor return createQuery().fetch(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Page page(Pageable pageable) { @@ -296,6 +303,8 @@ public class QuerydslMongoPredicateExecutor extends QuerydslPredicateExecutor if (getSort().isSorted()) { query.with(getSort()); } + + query.limit(getLimit()); } } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java index 505a7c0c4..914724364 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java @@ -33,12 +33,14 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF private final P predicate; private final Sort sort; + private final int limit; private final Class resultType; private final List fieldsToInclude; - ReactiveFluentQuerySupport(P predicate, Sort sort, Class resultType, List fieldsToInclude) { + ReactiveFluentQuerySupport(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { this.predicate = predicate; this.sort = sort; + this.limit = limit; this.resultType = resultType; this.fieldsToInclude = fieldsToInclude; } @@ -52,7 +54,15 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF Assert.notNull(sort, "Sort must not be null"); - return create(predicate, sort, resultType, fieldsToInclude); + return create(predicate, sort, limit, resultType, fieldsToInclude); + } + + @Override + public ReactiveFluentQuery limit(int limit) { + + Assert.isTrue(limit > 0, "Limit must be greater zero"); + + return create(predicate, sort, limit, resultType, fieldsToInclude); } /* @@ -64,7 +74,7 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF Assert.notNull(projection, "Projection target type must not be null"); - return create(predicate, sort, projection, fieldsToInclude); + return create(predicate, sort, limit, projection, fieldsToInclude); } /* @@ -76,10 +86,10 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF Assert.notNull(properties, "Projection properties must not be null"); - return create(predicate, sort, resultType, new ArrayList<>(properties)); + return create(predicate, sort, limit, resultType, new ArrayList<>(properties)); } - protected abstract ReactiveFluentQuerySupport create(P predicate, Sort sort, Class resultType, + protected abstract ReactiveFluentQuerySupport create(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude); P getPredicate() { @@ -90,6 +100,10 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF return sort; } + int getLimit() { + return limit; + } + Class getResultType() { return resultType; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java index a21bbb6c1..ff2b44293 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java @@ -26,6 +26,8 @@ import org.bson.Document; import org.reactivestreams.Publisher; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveMongoOperations; import org.springframework.data.mongodb.core.query.BasicQuery; @@ -195,17 +197,18 @@ public class ReactiveQuerydslMongoPredicateExecutor extends QuerydslPredicate class ReactiveFluentQuerydsl extends ReactiveFluentQuerySupport { ReactiveFluentQuerydsl(Predicate predicate, Class resultType) { - this(predicate, Sort.unsorted(), resultType, Collections.emptyList()); + this(predicate, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - ReactiveFluentQuerydsl(Predicate predicate, Sort sort, Class resultType, List fieldsToInclude) { - super(predicate, sort, resultType, fieldsToInclude); + ReactiveFluentQuerydsl(Predicate predicate, Sort sort, int limit, Class resultType, + List fieldsToInclude) { + super(predicate, sort, limit, resultType, fieldsToInclude); } @Override - protected ReactiveFluentQuerydsl create(Predicate predicate, Sort sort, Class resultType, + protected ReactiveFluentQuerydsl create(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new ReactiveFluentQuerydsl<>(predicate, sort, resultType, fieldsToInclude); + return new ReactiveFluentQuerydsl<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -223,6 +226,11 @@ public class ReactiveQuerydslMongoPredicateExecutor extends QuerydslPredicate return createQuery().fetch(); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Mono> page(Pageable pageable) { @@ -260,6 +268,8 @@ public class ReactiveQuerydslMongoPredicateExecutor extends QuerydslPredicate if (getSort().isSorted()) { query.with(getSort()); } + + query.limit(getLimit()); } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java index 7cc102e4d..cc27f81d6 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java @@ -24,9 +24,10 @@ import java.util.List; import java.util.function.Consumer; import org.bson.Document; - import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.ReactiveFindOperation; import org.springframework.data.mongodb.core.ReactiveMongoOperations; @@ -90,6 +91,10 @@ class ReactiveSpringDataMongodbQuery extends SpringDataMongodbQuerySupport find.matching(it).all()); } + Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().flatMap(it -> find.matching(it).scroll(scrollPosition)); + } + /** * Fetch all matching query results as page. * @@ -97,8 +102,8 @@ class ReactiveSpringDataMongodbQuery extends SpringDataMongodbQuerySupport> fetchPage(Pageable pageable) { - Mono> content = createQuery().map(it -> it.with(pageable)) - .flatMapMany(it -> find.matching(it).all()).collectList(); + Mono> content = createQuery().map(it -> it.with(pageable)).flatMapMany(it -> find.matching(it).all()) + .collectList(); return content.flatMap(it -> ReactivePageableExecutionUtils.getPage(it, pageable, fetchCount())); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java index 23a459123..750be26ae 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java @@ -32,6 +32,8 @@ import org.springframework.data.domain.Example; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ExecutableFindOperation; import org.springframework.data.mongodb.core.MongoOperations; @@ -361,17 +363,17 @@ public class SimpleMongoRepository implements MongoRepository { class FluentQueryByExample extends FetchableFluentQuerySupport, T> { FluentQueryByExample(Example example, Class resultType) { - this(example, Sort.unsorted(), resultType, Collections.emptyList()); + this(example, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - FluentQueryByExample(Example example, Sort sort, Class resultType, List fieldsToInclude) { - super(example, sort, resultType, fieldsToInclude); + FluentQueryByExample(Example example, Sort sort, int limit, Class resultType, List fieldsToInclude) { + super(example, sort, limit, resultType, fieldsToInclude); } @Override - protected FluentQueryByExample create(Example predicate, Sort sort, Class resultType, + protected FluentQueryByExample create(Example predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new FluentQueryByExample<>(predicate, sort, resultType, fieldsToInclude); + return new FluentQueryByExample<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -389,6 +391,11 @@ public class SimpleMongoRepository implements MongoRepository { return createQuery().all(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Page page(Pageable pageable) { @@ -427,6 +434,8 @@ public class SimpleMongoRepository implements MongoRepository { query.with(getSort()); } + query.limit(getLimit()); + if (!getFieldsToInclude().isEmpty()) { query.fields().include(getFieldsToInclude().toArray(new String[0])); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java index 4e1ccad65..9114f5313 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java @@ -34,6 +34,8 @@ import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.data.domain.Example; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveFindOperation; import org.springframework.data.mongodb.core.ReactiveMongoOperations; @@ -404,17 +406,18 @@ public class SimpleReactiveMongoRepository implement class ReactiveFluentQueryByExample extends ReactiveFluentQuerySupport, T> { ReactiveFluentQueryByExample(Example example, Class resultType) { - this(example, Sort.unsorted(), resultType, Collections.emptyList()); + this(example, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - ReactiveFluentQueryByExample(Example example, Sort sort, Class resultType, List fieldsToInclude) { - super(example, sort, resultType, fieldsToInclude); + ReactiveFluentQueryByExample(Example example, Sort sort, int limit, Class resultType, + List fieldsToInclude) { + super(example, sort, limit, resultType, fieldsToInclude); } @Override - protected ReactiveFluentQueryByExample create(Example predicate, Sort sort, Class resultType, - List fieldsToInclude) { - return new ReactiveFluentQueryByExample<>(predicate, sort, resultType, fieldsToInclude); + protected ReactiveFluentQueryByExample create(Example predicate, Sort sort, int limit, + Class resultType, List fieldsToInclude) { + return new ReactiveFluentQueryByExample<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -432,6 +435,11 @@ public class SimpleReactiveMongoRepository implement return createQuery().all(); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Mono> page(Pageable pageable) { @@ -465,6 +473,8 @@ public class SimpleReactiveMongoRepository implement query.with(getSort()); } + query.limit(getLimit()); + if (!getFieldsToInclude().isEmpty()) { query.fields().include(getFieldsToInclude().toArray(new String[0])); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java index e72c579ed..2c073986d 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java @@ -22,10 +22,11 @@ import java.util.function.Consumer; import java.util.stream.Stream; import org.bson.Document; - import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.ExecutableFindOperation; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.query.BasicQuery; @@ -75,8 +76,7 @@ public class SpringDataMongodbQuery extends SpringDataMongodbQuerySupport type, - String collectionName) { + public SpringDataMongodbQuery(MongoOperations operations, Class type, String collectionName) { this(operations, type, type, collectionName, it -> {}); } @@ -133,6 +133,17 @@ public class SpringDataMongodbQuery extends SpringDataMongodbQuerySupport scroll(ScrollPosition scrollPosition) { + + try { + return find.matching(createQuery()).scroll(scrollPosition); + } catch (RuntimeException e) { + return handleException(e, Scroll.from(Collections.emptyList(), value -> { + throw new UnsupportedOperationException(); + })); + } + } + @Override public Stream stream() { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java index b8d4093f7..248b03ab2 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java @@ -24,16 +24,7 @@ import java.util.StringJoiner; import java.util.function.Function; import java.util.stream.StreamSupport; -import org.bson.BSONObject; -import org.bson.BsonBinary; -import org.bson.BsonBoolean; -import org.bson.BsonDouble; -import org.bson.BsonInt32; -import org.bson.BsonInt64; -import org.bson.BsonObjectId; -import org.bson.BsonString; -import org.bson.BsonValue; -import org.bson.Document; +import org.bson.*; import org.bson.codecs.DocumentCodec; import org.bson.codecs.configuration.CodecRegistry; import org.bson.conversions.Bson; @@ -489,7 +480,7 @@ public class BsonUtils { } /** - * Resolve a the value for a given key. If the given {@link Bson} value contains the key the value is immediately + * Resolve the value for a given key. If the given {@link Bson} value contains the key the value is immediately * returned. If not and the key contains a path using the dot ({@code .}) notation it will try to resolve the path by * inspecting the individual parts. If one of the intermediate ones is {@literal null} or cannot be inspected further * (wrong) type, {@literal null} is returned. @@ -501,8 +492,22 @@ public class BsonUtils { */ @Nullable public static Object resolveValue(Bson bson, String key) { + return resolveValue(asMap(bson), key); + } - Map source = asMap(bson); + /** + * Resolve the value for a given key. If the given {@link Map} value contains the key the value is immediately + * returned. If not and the key contains a path using the dot ({@code .}) notation it will try to resolve the path by + * inspecting the individual parts. If one of the intermediate ones is {@literal null} or cannot be inspected further + * (wrong) type, {@literal null} is returned. + * + * @param source the source to inspect. Must not be {@literal null}. + * @param key the key to lookup. Must not be {@literal null}. + * @return can be {@literal null}. + * @since 4.1 + */ + @Nullable + public static Object resolveValue(Map source, String key) { if (source.containsKey(key) || !key.contains(".")) { return source.get(key); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java new file mode 100644 index 000000000..865f54f20 --- /dev/null +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -0,0 +1,147 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import static org.springframework.data.mongodb.core.query.Criteria.*; +import static org.springframework.data.mongodb.test.util.Assertions.*; + +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.auditing.IsNewAwareAuditingHandler; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Sort; +import org.springframework.data.mapping.context.PersistentEntities; +import org.springframework.data.mongodb.core.MongoTemplateTests.PersonWithIdPropertyOfTypeUUIDListener; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.test.util.Client; +import org.springframework.data.mongodb.test.util.MongoClientExtension; +import org.springframework.data.mongodb.test.util.MongoTestTemplate; + +import com.mongodb.client.MongoClient; + +/** + * Integration tests for {@link Scroll} queries. + * + * @author Mark Paluch + */ +@ExtendWith(MongoClientExtension.class) +class MongoTemplateScrollTests { + + static @Client MongoClient client; + + public static final String DB_NAME = "mongo-template-scroll-tests"; + + ConfigurableApplicationContext context = new GenericApplicationContext(); + + MongoTestTemplate template = new MongoTestTemplate(cfg -> { + + cfg.configureDatabaseFactory(it -> { + + it.client(client); + it.defaultDb(DB_NAME); + }); + + cfg.configureMappingContext(it -> { + it.autocreateIndex(false); + it.initialEntitySet(AuditablePerson.class); + }); + + cfg.configureApplicationContext(it -> { + it.applicationContext(context); + it.addEventListener(new PersonWithIdPropertyOfTypeUUIDListener()); + }); + + cfg.configureAuditing(it -> { + it.auditingHandler(ctx -> { + return new IsNewAwareAuditingHandler(PersistentEntities.of(ctx)); + }); + }); + }); + + @BeforeEach + void setUp() { + template.remove(Person.class).all(); + } + + @ParameterizedTest // GH-4308 + @MethodSource("positions") + public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + + Person john20 = new Person("John", 20); + Person john40_1 = new Person("John", 40); + Person john40_2 = new Person("John", 40); + Person jane_20 = new Person("Jane", 20); + Person jane_40 = new Person("Jane", 40); + Person jane_42 = new Person("Jane", 42); + + template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)); + Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); + q.with(scrollPosition); + + Scroll scroll = template.scroll(q, resultType, "person"); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + + scroll = template.scroll(q.with(scroll.lastPosition()).limit(3), resultType, "person"); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(3); + assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + + scroll = template.scroll(q.with(scroll.lastPosition()).limit(1), resultType, "person"); + + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + } + + static Stream positions() { + + return Stream.of(args(KeysetScrollPosition.initial(), Person.class, Function.identity()), // + args(KeysetScrollPosition.initial(), Document.class, MongoTemplateScrollTests::toDocument), // + args(OffsetScrollPosition.initial(), Person.class, Function.identity())); + } + + private static Arguments args(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + return Arguments.of(scrollPosition, resultType, assertionConverter); + } + + static Document toDocument(Person person) { + return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) + .append("firstName", person.getFirstName()).append("age", person.getAge()); + } +} diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java index 6af8f7468..9bfbe989b 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java @@ -46,7 +46,6 @@ import org.bson.types.ObjectId; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; - import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.support.GenericApplicationContext; import org.springframework.core.convert.converter.Converter; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java new file mode 100644 index 000000000..ede69c396 --- /dev/null +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java @@ -0,0 +1,144 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import static org.springframework.data.mongodb.core.query.Criteria.*; +import static org.springframework.data.mongodb.test.util.Assertions.*; + +import reactor.test.StepVerifier; + +import java.time.Duration; +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.test.util.Client; +import org.springframework.data.mongodb.test.util.MongoClientExtension; +import org.springframework.data.mongodb.test.util.ReactiveMongoTestTemplate; + +import com.mongodb.reactivestreams.client.MongoClient; + +/** + * Integration tests for {@link Scroll} queries. + * + * @author Mark Paluch + */ +@ExtendWith(MongoClientExtension.class) +class ReactiveMongoTemplateScrollTests { + + static @Client MongoClient client; + + public static final String DB_NAME = "mongo-template-scroll-tests"; + + ConfigurableApplicationContext context = new GenericApplicationContext(); + + private ReactiveMongoTestTemplate template = new ReactiveMongoTestTemplate(cfg -> { + + cfg.configureDatabaseFactory(it -> { + + it.client(client); + it.defaultDb(DB_NAME); + }); + + cfg.configureApplicationContext(it -> { + it.applicationContext(context); + }); + }); + + @BeforeEach + void setUp() { + template.remove(Person.class).all() // + .as(StepVerifier::create) // + .expectNextCount(1) // + .verifyComplete(); + } + + @ParameterizedTest // GH-4308 + @MethodSource("positions") + public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + + Person john20 = new Person("John", 20); + Person john40_1 = new Person("John", 40); + Person john40_2 = new Person("John", 40); + Person jane_20 = new Person("Jane", 20); + Person jane_40 = new Person("Jane", 40); + Person jane_42 = new Person("Jane", 42); + + template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)) // + .as(StepVerifier::create) // + .expectNextCount(6) // + .verifyComplete(); + + Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); + q.with(scrollPosition); + + Scroll scroll = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + + scroll = template.scroll(q.limit(3).with(scroll.lastPosition()), resultType, "person") + .block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(3); + assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + + scroll = template.scroll(q.limit(1).with(scroll.lastPosition()), resultType, "person") + .block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + } + + static Stream positions() { + + return Stream.of(args(KeysetScrollPosition.initial(), Person.class, Function.identity()), // + args(KeysetScrollPosition.initial(), Document.class, MongoTemplateScrollTests::toDocument), // + args(OffsetScrollPosition.initial(), Person.class, Function.identity())); + } + + private static Arguments args(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + return Arguments.of(scrollPosition, resultType, assertionConverter); + } + + static Document toDocument(Person person) { + return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) + .append("firstName", person.getFirstName()).append("age", person.getAge()); + } +} diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java index 97dec780d..697bf33b9 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java @@ -38,17 +38,11 @@ import org.bson.Document; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.IncorrectResultSizeDataAccessException; -import org.springframework.data.domain.Example; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Range; -import org.springframework.data.domain.Slice; -import org.springframework.data.domain.Sort; +import org.springframework.data.domain.*; +import org.springframework.data.domain.ExampleMatcher.GenericPropertyMatcher; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.geo.Box; import org.springframework.data.geo.Circle; @@ -208,6 +202,29 @@ public abstract class AbstractPersonRepositoryIntegrationTests implements Dirtie assertThat(result).contains(dave, stefan); } + @Test // GH-4308 + void appliesScrollPositionCorrectly() { + + Scroll page = repository.findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc("*a*", + KeysetScrollPosition.initial()); + + assertThat(page.isLast()).isFalse(); + assertThat(page.size()).isEqualTo(2); + assertThat(page).contains(carter); + } + + @Test // GH-4308 + void appliesScrollPositionWithProjectionCorrectly() { + + Scroll page = repository.findCursorProjectionByLastnameLike("*a*", + PageRequest.of(0, 2, Sort.by(Direction.ASC, "lastname", "firstname"))); + + assertThat(page.isLast()).isFalse(); + assertThat(page.size()).isEqualTo(2); + + assertThat(page).element(0).isEqualTo(new PersonSummaryDto(carter.getFirstname(), carter.getLastname())); + } + @Test void executesPagedFinderCorrectly() { @@ -936,6 +953,21 @@ public abstract class AbstractPersonRepositoryIntegrationTests implements Dirtie assertThat(repository.findAll(person.id.in(Arrays.asList(dave.id, carter.id)))).contains(dave, carter); } + @Test // DATAMONGO-969 + void shouldScrollPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { + + Scroll scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); + + assertThat(scroll).containsExactly(boyd, carter); + + ScrollPosition resumeFrom = scroll.lastPosition(); + scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(resumeFrom)); + + assertThat(scroll).containsOnly(dave); + } + @Test // DATAMONGO-1030 void executesSingleEntityQueryWithProjectionCorrectly() { @@ -1142,6 +1174,33 @@ public abstract class AbstractPersonRepositoryIntegrationTests implements Dirtie assertThat(result).hasSize(2); } + @Test // GH-4308 + void scrollByExampleShouldReturnCorrectResult() { + + Person sample = new Person(); + sample.setLastname("M"); + + // needed to tweak stuff a bit since some field are automatically set - so we need to undo this + ReflectionTestUtils.setField(sample, "id", null); + ReflectionTestUtils.setField(sample, "createdAt", null); + ReflectionTestUtils.setField(sample, "email", null); + + Scroll result = repository.findBy( + Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); + + assertThat(result).containsOnly(dave, leroi); + assertThat(result.hasNext()).isTrue(); + + ScrollPosition position = result.lastPosition(); + result = repository.findBy( + Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(position)); + + assertThat(result).containsOnly(oliver); + assertThat(result.hasNext()).isFalse(); + } + @Test // DATAMONGO-1425 void findsPersonsByFirstnameNotContains() { diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java index d668bde4c..b34b8cecd 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java @@ -23,9 +23,11 @@ import java.util.UUID; import java.util.regex.Pattern; import java.util.stream.Stream; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; +import org.springframework.data.domain.Scroll; import org.springframework.data.domain.Slice; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Box; @@ -115,7 +117,27 @@ public interface PersonRepository extends MongoRepository, Query List findByAgeLessThan(int age, Sort sort); /** - * Returns a page of {@link Person}s with a lastname mathing the given one (*-wildcards supported). + * Returns a scroll of {@link Person}s with a lastname matching the given one (*-wildcards supported). + * + * @param lastname + * @param scrollPosition + * @return + */ + Scroll findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname, + KeysetScrollPosition scrollPosition); + + /** + * Returns a scroll of {@link Person}s applying projections with a lastname matching the given one (*-wildcards + * supported). + * + * @param lastname + * @param pageable + * @return + */ + Scroll findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + + /** + * Returns a page of {@link Person}s with a lastname matching the given one (*-wildcards supported). * * @param lastname * @param pageable @@ -429,7 +451,7 @@ public interface PersonRepository extends MongoRepository, Query @Update("{ '$inc' : { 'visits' : ?1 } }") int updateAllByLastname(String lastname, int increment); - @Update( pipeline = {"{ '$set' : { 'visits' : { '$add' : [ '$visits', ?1 ] } } }"}) + @Update(pipeline = { "{ '$set' : { 'visits' : { '$add' : [ '$visits', ?1 ] } } }" }) void findAndIncrementVisitsViaPipelineByLastname(String lastname, int increment); @Update("{ '$inc' : { 'visits' : ?#{[1]} } }") diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java index 3b5d7b861..a17ba71a2 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java @@ -15,9 +15,18 @@ */ package org.springframework.data.mongodb.repository; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; + /** * @author Oliver Gierke */ +@EqualsAndHashCode +@AllArgsConstructor +@NoArgsConstructor +@ToString public class PersonSummaryDto { String firstname; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java index 2a8ea1542..a4b83c33b 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java @@ -29,24 +29,29 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.reactivestreams.Publisher; - import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.geo.Circle; @@ -285,6 +290,36 @@ class ReactiveMongoRepositoryTests implements DirtiesStateExtension.StateFunctio cappedRepository.findDtoProjectionByKey("value").as(StepVerifier::create).expectNextCount(1).thenCancel().verify(); } + @Test // GH-4308 + void appliesScrollingCorrectly() { + + Scroll scroll = repository + .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", KeysetScrollPosition.initial()).block(); + + assertThat(scroll).hasSize(2); + assertThat(scroll).containsSequence(alicia, boyd); + assertThat(scroll.isLast()).isFalse(); + + Scroll nextScroll = repository + .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", scroll.lastPosition()).block(); + + assertThat(nextScroll).hasSize(2); + assertThat(nextScroll).containsSequence(carter, dave); + assertThat(nextScroll.isLast()).isFalse(); + } + + @Test // GH-4308 + void appliesScrollingWithProjectionCorrectly() { + + repository + .findCursorProjectionByLastnameLike("*", PageRequest.of(0, 2, Sort.by(Direction.ASC, "firstname", "lastname"))) // + .flatMapIterable(Function.identity()) // + .as(StepVerifier::create) // + .expectNext(new PersonSummaryDto(alicia.getFirstname(), alicia.getLastname())) // + .expectNextCount(1) // + .verifyComplete(); + } + @Test // DATAMONGO-1444 @DirtiesState void findsPeopleByLocationWithinCircle() { @@ -436,6 +471,27 @@ class ReactiveMongoRepositoryTests implements DirtiesStateExtension.StateFunctio }).verifyComplete(); } + @Test // GH-4308 + void shouldScrollWithId() { + + List> capture = new ArrayList<>(); + repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())) // + .as(StepVerifier::create) // + .recordWith(() -> capture).assertNext(actual -> { + assertThat(actual).hasSize(2).containsExactly(boyd, carter); + }).verifyComplete(); + + Scroll scroll = capture.get(0); + + repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(scroll.lastPosition())) // + .as(StepVerifier::create) // + .recordWith(() -> capture).assertNext(actual -> { + assertThat(actual).containsOnly(dave); + }).verifyComplete(); + } + @Test // DATAMONGO-2153 void findListOfSingleValue() { @@ -712,6 +768,11 @@ class ReactiveMongoRepositoryTests implements DirtiesStateExtension.StateFunctio @Query("{ lastname: { $in: ?0 }, age: { $gt : ?1 } }") Flux findStringQuery(Flux lastname, Mono age); + Mono> findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc(String lastname, + ScrollPosition scrollPosition); + + Mono> findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + Flux findByLocationWithin(Circle circle); Flux findByLocationWithin(Circle circle, Pageable pageable); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java index 400351b1b..134d3ed5b 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java @@ -21,6 +21,7 @@ import java.util.Iterator; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; import org.springframework.data.domain.Range.Bound; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Distance; import org.springframework.data.geo.Point; @@ -72,6 +73,11 @@ class StubParameterAccessor implements MongoParameterAccessor { } } + @Override + public ScrollPosition getScrollPosition() { + return null; + } + public Pageable getPageable() { return null; } diff --git a/src/main/asciidoc/index.adoc b/src/main/asciidoc/index.adoc index 93b7896d8..b99bb0bb4 100644 --- a/src/main/asciidoc/index.adoc +++ b/src/main/asciidoc/index.adoc @@ -6,6 +6,8 @@ ifdef::backend-epub3[:front-cover-image: image:epub-cover.png[Front Cover,1050,1 :spring-data-commons-docs: ../../../../spring-data-commons/src/main/asciidoc :store: Mongo +:feature-scroll: true + (C) 2008-2022 The original authors. NOTE: Copies of this document may be made for your own use and for distribution to others, provided that you do not charge any fee for such copies and further provided that each copy contains this Copyright Notice, whether distributed in print or electronically. diff --git a/src/main/asciidoc/reference/mongodb.adoc b/src/main/asciidoc/reference/mongodb.adoc index 45f54b6ca..101cf2758 100644 --- a/src/main/asciidoc/reference/mongodb.adoc +++ b/src/main/asciidoc/reference/mongodb.adoc @@ -1289,6 +1289,7 @@ The `Query` class has some additional methods that provide options for the query * `Query` *limit* `(int limit)` used to limit the size of the returned results to the provided limit (used for paging) * `Query` *skip* `(int skip)` used to skip the provided number of documents in the results (used for paging) * `Query` *with* `(Sort sort)` used to provide sort definition for the results +* `Query` *with* `(ScrollPosition position)` used to provide a scroll position (Offset- or Keyset-based pagination) to start or resume a `Scroll` [[mongo-template.querying.field-selection]] ==== Selecting fields