DATAMONGO-2341 - Polishing.

Inline MongoPersistentEntity.idPropertyIsShardKey() into UpdateContext. Move mapped shard key cache to QueryOperations level. Simplify conditionals. Tweak documentation.

Original pull request: #833.
This commit is contained in:
Mark Paluch
2020-02-17 10:11:26 +01:00
parent 6259cd2c3b
commit 22ca597fca
12 changed files with 143 additions and 122 deletions

View File

@@ -1640,7 +1640,7 @@ public class MongoTemplate implements MongoOperations, ApplicationContextAware,
collection.find(filter, Document.class).projection(updateContext.getMappedShardKey(entity)).first());
}
}
ReplaceOptions replaceOptions = updateContext.getReplaceOptions(entityClass);
return collection.replaceOne(filter, updateObj, replaceOptions);
} else {

View File

@@ -40,6 +40,7 @@ import org.springframework.data.mongodb.core.convert.QueryMapper;
import org.springframework.data.mongodb.core.convert.UpdateMapper;
import org.springframework.data.mongodb.core.mapping.MongoPersistentEntity;
import org.springframework.data.mongodb.core.mapping.MongoPersistentProperty;
import org.springframework.data.mongodb.core.mapping.ShardKey;
import org.springframework.data.mongodb.core.query.BasicQuery;
import org.springframework.data.mongodb.core.query.Collation;
import org.springframework.data.mongodb.core.query.Query;
@@ -58,9 +59,10 @@ import com.mongodb.client.model.UpdateOptions;
/**
* {@link QueryOperations} centralizes common operations required before an operation is actually ready to be executed.
* This involves mapping {@link Query queries} into their respective MongoDB representation, computing execution options
* for {@literal count}, {@literal remove}, ... <br />
* for {@literal count}, {@literal remove}, and other methods.
*
* @author Christoph Strobl
* @author Mark Paluch
* @since 3.0
*/
class QueryOperations {
@@ -71,6 +73,7 @@ class QueryOperations {
private final CodecRegistryProvider codecRegistryProvider;
private final MappingContext<? extends MongoPersistentEntity<?>, MongoPersistentProperty> mappingContext;
private final AggregationUtil aggregationUtil;
private final Map<Class<?>, Document> mappedShardKey = new ConcurrentHashMap<>(1);
/**
* Create a new instance of {@link QueryOperations}.
@@ -503,7 +506,6 @@ class QueryOperations {
private final boolean upsert;
private final @Nullable UpdateDefinition update;
private final @Nullable MappedDocument mappedDocument;
private final Map<Class<?>, Document> mappedShardKey = new ConcurrentHashMap<>(1);
/**
* Create a new {@link UpdateContext} instance.
@@ -624,41 +626,49 @@ class QueryOperations {
return mappedQuery;
}
<T> Document applyShardKey(@Nullable MongoPersistentEntity<T> domainType, Document filter,
@Nullable Document existing) {
<T> Document applyShardKey(MongoPersistentEntity<T> domainType, Document filter, @Nullable Document existing) {
Document shardKeySource = existing != null ? existing
: mappedDocument != null ? mappedDocument.getDocument() : getMappedUpdate(domainType);
Document filterWithShardKey = new Document(filter);
for (String key : getMappedShardKeyFields(domainType)) {
if (!filterWithShardKey.containsKey(key)) {
filterWithShardKey.append(key, shardKeySource.get(key));
}
}
getMappedShardKeyFields(domainType).forEach(key -> filterWithShardKey.putIfAbsent(key, shardKeySource.get(key)));
return filterWithShardKey;
}
<T> boolean requiresShardKey(Document filter, @Nullable MongoPersistentEntity<T> domainType) {
boolean requiresShardKey(Document filter, @Nullable MongoPersistentEntity<?> domainType) {
if (multi || domainType == null || !domainType.isSharded() || domainType.idPropertyIsShardKey()) {
return false;
}
if (filter.keySet().containsAll(getMappedShardKeyFields(domainType))) {
return false;
}
return true;
return !multi && domainType != null && domainType.isSharded() && !shardedById(domainType)
&& !filter.keySet().containsAll(getMappedShardKeyFields(domainType));
}
Set<String> getMappedShardKeyFields(@Nullable MongoPersistentEntity<?> entity) {
/**
* @return {@literal true} if the {@link MongoPersistentEntity#getShardKey() shard key} is the entities
* {@literal id} property.
* @since 3.0
*/
private boolean shardedById(MongoPersistentEntity<?> domainType) {
ShardKey shardKey = domainType.getShardKey();
if (shardKey.size() != 1) {
return false;
}
String key = shardKey.getPropertyNames().iterator().next();
if ("_id".equals(key)) {
return true;
}
MongoPersistentProperty idProperty = domainType.getIdProperty();
return idProperty != null && idProperty.getName().equals(key);
}
Set<String> getMappedShardKeyFields(MongoPersistentEntity<?> entity) {
return getMappedShardKey(entity).keySet();
}
Document getMappedShardKey(@Nullable MongoPersistentEntity<?> entity) {
Document getMappedShardKey(MongoPersistentEntity<?> entity) {
return mappedShardKey.computeIfAbsent(entity.getType(),
key -> queryMapper.getMappedFields(entity.getShardKey().getDocument(), entity));
}

View File

@@ -1637,7 +1637,7 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati
? collection //
: collection.withWriteConcern(writeConcernToUse);
Publisher<?> publisher = null;
Publisher<?> publisher;
if (!mapped.hasId()) {
publisher = collectionToUse.insertOne(document);
} else {
@@ -1647,20 +1647,23 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati
Document filter = updateContext.getMappedQuery(entity);
Document replacement = updateContext.getMappedUpdate(entity);
Mono<Document> theFilter = Mono.just(filter);
Mono<Document> deferredFilter;
if(updateContext.requiresShardKey(filter, entity)) {
if (updateContext.requiresShardKey(filter, entity)) {
if (entity.getShardKey().isImmutable()) {
theFilter = Mono.just(updateContext.applyShardKey(entity, filter, null));
deferredFilter = Mono.just(updateContext.applyShardKey(entity, filter, null));
} else {
theFilter = Mono.from(
collection.find(filter, Document.class).projection(updateContext.getMappedShardKey(entity)).first())
deferredFilter = Mono
.from(
collection.find(filter, Document.class).projection(updateContext.getMappedShardKey(entity)).first())
.defaultIfEmpty(replacement).map(it -> updateContext.applyShardKey(entity, filter, it));
}
} else {
deferredFilter = Mono.just(filter);
}
publisher = theFilter.flatMap(
it -> Mono.from(collectionToUse.replaceOne(it, replacement, updateContext.getReplaceOptions(entityClass))));
publisher = deferredFilter.flatMapMany(
it -> collectionToUse.replaceOne(it, replacement, updateContext.getReplaceOptions(entityClass)));
}
return Mono.from(publisher).map(o -> mapped.getId());
@@ -1800,20 +1803,22 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati
if (!UpdateMapper.isUpdateObject(updateObj)) {
Document filter = new Document(queryObj);
Mono<Document> theFilter = Mono.just(filter);
Mono<Document> deferredFilter;
if(updateContext.requiresShardKey(filter, entity)) {
if (updateContext.requiresShardKey(filter, entity)) {
if (entity.getShardKey().isImmutable()) {
theFilter = Mono.just(updateContext.applyShardKey(entity, filter, null));
deferredFilter = Mono.just(updateContext.applyShardKey(entity, filter, null));
} else {
theFilter = Mono.from(
deferredFilter = Mono.from(
collection.find(filter, Document.class).projection(updateContext.getMappedShardKey(entity)).first())
.defaultIfEmpty(updateObj).map(it -> updateContext.applyShardKey(entity, filter, it));
}
} else {
deferredFilter = Mono.just(filter);
}
ReplaceOptions replaceOptions = updateContext.getReplaceOptions(entityClass);
return theFilter.flatMap(it -> Mono.from(collectionToUse.replaceOne(it, updateObj, replaceOptions)));
return deferredFilter.flatMap(it -> Mono.from(collectionToUse.replaceOne(it, updateObj, replaceOptions)));
}
return multi ? collectionToUse.updateMany(queryObj, updateObj, updateOptions)

View File

@@ -96,7 +96,26 @@ public class BasicMongoPersistentEntity<T> extends BasicPersistentEntity<T, Mong
this.collationExpression = null;
}
this.shardKey = detectShardKey(this);
this.shardKey = detectShardKey();
}
private ShardKey detectShardKey() {
if (!isAnnotationPresent(Sharded.class)) {
return ShardKey.none();
}
Sharded sharded = getRequiredAnnotation(Sharded.class);
String[] keyProperties = sharded.shardKey();
if (ObjectUtils.isEmpty(keyProperties)) {
keyProperties = new String[] { "_id" };
}
ShardKey shardKey = ShardingStrategy.HASH.equals(sharded.shardingStrategy()) ? ShardKey.hash(keyProperties)
: ShardKey.range(keyProperties);
return sharded.immutableKey() ? ShardKey.immutable(shardKey) : shardKey;
}
/*
@@ -307,26 +326,6 @@ public class BasicMongoPersistentEntity<T> extends BasicPersistentEntity<T, Mong
return expression instanceof LiteralExpression ? null : expression;
}
@Nullable
private static ShardKey detectShardKey(BasicMongoPersistentEntity<?> entity) {
if (!entity.isAnnotationPresent(Sharded.class)) {
return ShardKey.none();
}
Sharded sharded = entity.getRequiredAnnotation(Sharded.class);
String[] keyProperties = sharded.shardKey();
if (ObjectUtils.isEmpty(keyProperties)) {
keyProperties = new String[] { "_id" };
}
ShardKey shardKey = ShardingStrategy.HASH.equals(sharded.shardingStrategy()) ? ShardKey.hash(keyProperties)
: ShardKey.range(keyProperties);
return sharded.immutableKey() ? ShardKey.immutable(shardKey) : shardKey;
}
/**
* Handler to collect {@link MongoPersistentProperty} instances and check that each of them is mapped to a distinct
* field name.

View File

@@ -86,29 +86,11 @@ public interface MongoPersistentEntity<T> extends PersistentEntity<T, MongoPersi
ShardKey getShardKey();
/**
* @return {@literal true} if the {@link #getShardKey() shard key} does not match {@link ShardKey#none()}.
* @return {@literal true} if the {@link #getShardKey() shard key} is sharded.
* @since 3.0
*/
default boolean isSharded() {
return !ShardKey.none().equals(getShardKey());
return getShardKey().isSharded();
}
/**
* @return {@literal true} if the {@link #getShardKey() shard key} is the entities {@literal id} property.
* @since 3.0
*/
default boolean idPropertyIsShardKey() {
ShardKey shardKey = getShardKey();
if (shardKey.size() != 1) {
return false;
}
String key = shardKey.getPropertyNames().iterator().next();
if ("_id".equals(key)) {
return true;
}
MongoPersistentProperty idProperty = getIdProperty();
return idProperty != null && idProperty.getName().equals(key);
}
}

View File

@@ -28,12 +28,13 @@ import org.springframework.util.ObjectUtils;
* Value object representing an entities <a href="https://docs.mongodb.com/manual/core/sharding-shard-key/">Shard
* Key</a> used to distribute documents across a sharded MongoDB cluster.
* <p />
* {@link ShardKey#isImmutable() Immutable} shard keys indicate a fixed value that is not updated (see
* {@link ShardKey#isImmutable() Immutable} shard keys indicates a fixed value that is not updated (see
* <a href="https://docs.mongodb.com/manual/core/sharding-shard-key/#change-a-document-s-shard-key-value">MongoDB
* Reference: Change a Documents Shard Key Value</a>), which allows to skip server round trips in cases where a
* Reference: Change a Document's Shard Key Value</a>), which allows to skip server round trips in cases where a
* potential shard key change might have occurred.
*
* @author Christoph Strobl
* @author Mark Paluch
* @since 3.0
*/
public class ShardKey {
@@ -68,14 +69,23 @@ public class ShardKey {
/**
* @return {@literal true} if the shard key of an document does not change.
* @see <a href="https://docs.mongodb.com/manual/core/sharding-shard-key/#change-a-document-s-shard-key-value">MongoDB
* Reference: Change a Documents Shard Key Value</a>
* Reference: Change a Document's Shard Key Value</a>
*/
public boolean isImmutable() {
return immutable;
}
/**
* Get the unmapped MongoDB representation of the {@link ShardKey}.
* Return whether the shard key represents a sharded key. Return {@literal false} if the key is not sharded.
*
* @return {@literal true} if the key is sharded; {@literal false} otherwise.
*/
public boolean isSharded() {
return !propertyNames.isEmpty();
}
/**
* Get the raw MongoDB representation of the {@link ShardKey}.
*
* @return never {@literal null}.
*/

View File

@@ -25,26 +25,29 @@ import org.springframework.core.annotation.AliasFor;
import org.springframework.data.annotation.Persistent;
/**
* The {@link Sharded} annotation provides meta information about the actual distribution of data across multiple
* machines. The {@link #shardKey()} is used to distribute documents across shards. <br />
* Please visit the <a href="https://docs.mongodb.com/manual/sharding/">MongoDB Documentation</a> for more information
* about requirements and limitations of sharding. <br />
* Spring Data will automatically add the shard key to filter queries used for
* The {@link Sharded} annotation provides meta information about the actual distribution of data. The
* {@link #shardKey()} is used to distribute documents across shards. <br />
* Please see the <a href="https://docs.mongodb.com/manual/sharding/">MongoDB Documentation</a> for more information
* about requirements and limitations of sharding.
* <p/>
* Spring Data adds the shard key to filter queries used for
* {@link com.mongodb.client.MongoCollection#replaceOne(org.bson.conversions.Bson, Object)} operations triggered by
* {@code save} operations on {@link org.springframework.data.mongodb.core.MongoOperations} and
* {@link org.springframework.data.mongodb.core.ReactiveMongoOperations} as well as {@code update/upsert} operation
* {@link org.springframework.data.mongodb.core.ReactiveMongoOperations} as well as {@code update/upsert} operations
* replacing/upserting a single existing document as long as the given
* {@link org.springframework.data.mongodb.core.query.UpdateDefinition} holds a full copy of the entity. <br />
* {@link org.springframework.data.mongodb.core.query.UpdateDefinition} holds a full copy of the entity.
* <p/>
* All other operations that require the presence of the {@literal shard key} in the filter query need to provide the
* information via the {@link org.springframework.data.mongodb.core.query.Query} parameter when invoking the method.
*
* @author Christoph Strobl
* @author Mark Paluch
* @since 3.0
*/
@Persistent
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE })
@Target({ ElementType.TYPE, ElementType.ANNOTATION_TYPE })
public @interface Sharded {
/**
@@ -57,15 +60,15 @@ public @interface Sharded {
String[] value() default {};
/**
* The shard key determines the distribution of the collections documents among the clusters shards. The shard key
* is either a single or multiple indexed properties that exist in every document in the collection. <br />
* The shard key determines the distribution of the collection's documents among the cluster's shards. The shard key
* is either a single or multiple indexed properties that exist in every document in the collection.
* <p/>
* By default the {@literal id} property is used for sharding. <br />
* <strong>NOTE</strong> Required indexes will not be created automatically. Use
* {@link org.springframework.data.mongodb.core.index.Indexed} or
* <strong>NOTE</strong> Required indexes are not created automatically. Create these either externally, via
* {@link org.springframework.data.mongodb.core.index.IndexOperations#ensureIndex(org.springframework.data.mongodb.core.index.IndexDefinition)}
* or by annotating your domain model with {@link org.springframework.data.mongodb.core.index.Indexed}/
* {@link org.springframework.data.mongodb.core.index.CompoundIndex} along with enabled
* {@link org.springframework.data.mongodb.config.MongoConfigurationSupport#autoIndexCreation() auto index creation}
* or set up them up via
* {@link org.springframework.data.mongodb.core.index.IndexOperations#ensureIndex(org.springframework.data.mongodb.core.index.IndexDefinition)}.
* {@link org.springframework.data.mongodb.config.MongoConfigurationSupport#autoIndexCreation() auto index creation}.
*
* @return an empty key by default. Which indicates to use the entities {@literal id} property.
*/
@@ -82,10 +85,10 @@ public @interface Sharded {
/**
* As of MongoDB 4.2 it is possible to change the shard key using update. Using immutable shard keys avoids server
* round trips to obtain an entities actual shard key from the database.
*
* @return {@literal false} by default;
*
* @return {@literal false} by default.
* @see <a href="https://docs.mongodb.com/manual/core/sharding-shard-key/#change-a-document-s-shard-key-value">MongoDB
* Reference: Change a Documents Shard Key Value</a>
* Reference: Change a Document's Shard Key Value</a>
*/
boolean immutableKey() default false;

View File

@@ -1843,14 +1843,16 @@ public class MongoTemplateUnitTests extends MongoOperationsUnitTests {
@Test // DATAMONGO-2341
void saveShouldAppendNonDefaultShardKeyToVersionedEntityIfNotPresentInFilter() {
when(collection.replaceOne(any(), any(), any(ReplaceOptions.class))).thenReturn(UpdateResult.acknowledged(1, 1L, null));
when(collection.replaceOne(any(), any(), any(ReplaceOptions.class)))
.thenReturn(UpdateResult.acknowledged(1, 1L, null));
template.save(new ShardedVersionedEntityWithNonDefaultShardKey("id-1", 1L, "AT", 4230));
ArgumentCaptor<Bson> filter = ArgumentCaptor.forClass(Bson.class);
verify(collection).replaceOne(filter.capture(), any(), any());
assertThat(filter.getValue()).isEqualTo(new Document("_id", "id-1").append("version", 1L).append("country", "AT").append("userid", 4230));
assertThat(filter.getValue())
.isEqualTo(new Document("_id", "id-1").append("version", 1L).append("country", "AT").append("userid", 4230));
}
@Test // DATAMONGO-2341
@@ -1910,7 +1912,8 @@ public class MongoTemplateUnitTests extends MongoOperationsUnitTests {
@Test // DATAMONGO-2341
void saveVersionedShouldProjectOnShardKeyWhenLoadingExistingDocument() {
when(collection.replaceOne(any(), any(), any(ReplaceOptions.class))).thenReturn(UpdateResult.acknowledged(1, 1L, null));
when(collection.replaceOne(any(), any(), any(ReplaceOptions.class)))
.thenReturn(UpdateResult.acknowledged(1, 1L, null));
when(findIterable.first()).thenReturn(new Document("_id", "id-1").append("country", "US").append("userid", 4230));
template.save(new ShardedVersionedEntityWithNonDefaultShardKey("id-1", 1L, "AT", 4230));

View File

@@ -1076,7 +1076,8 @@ public class ReactiveMongoTemplateUnitTests {
@Test // DATAMONGO-2341
void saveShouldProjectOnShardKeyWhenLoadingExistingDocument() {
when(findPublisher.first()).thenReturn(Mono.just(new Document("_id", "id-1").append("country", "US").append("userid", 4230)));
when(findPublisher.first())
.thenReturn(Mono.just(new Document("_id", "id-1").append("country", "US").append("userid", 4230)));
template.save(new ShardedEntityWithNonDefaultShardKey("id-1", "AT", 4230)).subscribe();

View File

@@ -35,6 +35,8 @@ import org.springframework.lang.Nullable;
import com.mongodb.MongoClientSettings;
/**
* Unit test for {@link com.mongodb.internal.operation.UpdateOperation}.
*
* @author Christoph Strobl
*/
class UpdateOperationsUnitTests {
@@ -141,10 +143,9 @@ class UpdateOperationsUnitTests {
super(update, upsert);
}
<T> Document applyShardKey(@Nullable Class<T> domainType, Document filter, @Nullable Document existing) {
<T> Document applyShardKey(Class<T> domainType, Document filter, @Nullable Document existing) {
return applyShardKey(entityOf(domainType), filter, existing);
}
}
}
}

View File

@@ -8,6 +8,7 @@
* Support for <<mongo-template.aggregation-update,aggregation pipelines in update operations>>.
* Removal of `_id` flattening for composite Id's when using `MongoTemplate` aggregations.
* Apply pagination when using GridFS `find(Query)`.
* <<sharding,Sharding key derivation>> via `@Sharded`.
[[new-features.2-2-0]]
== What's New in Spring Data MongoDB 2.2

View File

@@ -1,12 +1,13 @@
[[sharding]]
= Sharding
MongoDB supports large data sets via sharding, a method for distributing data across multiple machines. Please refer to the https://docs.mongodb.com/manual/sharding/[MongoDB Documentation] to learn how to set up a sharded cluster, its requirements and limitations.
MongoDB supports large data sets via sharding, a method for distributing data across multiple database servers.
Please refer to the https://docs.mongodb.com/manual/sharding/[MongoDB Documentation] to learn how to set up a sharded cluster, its requirements and limitations.
Spring Data MongoDB uses the `@Sharded` annotation to identify entities stored in sharded collections as shown below.
====
[source, java]
[source,java]
----
@Document("users")
@Sharded(shardKey = { "country", "userId" }) <1>
@@ -21,23 +22,24 @@ public class User {
String country;
}
----
<1> The properties of the shard key are mapped to the actual field names. See
<1> The properties of the shard key get mapped to the actual field names.
====
[[sharding.sharded-collections]]
== Sharded Collections
Spring Data MongoDB does not auto set up sharding for collections nor indexes required for it. The snippet below shows how to do so using the MongoDB client API.
Spring Data MongoDB does not auto set up sharding for collections nor indexes required for it.
The snippet below shows how to do so using the MongoDB client API.
====
[source, java]
[source,java]
----
MongoDatabase adminDB = template.getMongoDbFactory()
.getMongoDatabase("admin"); <1>
.getMongoDatabase("admin"); <1>
adminDB.runCommand(new Document("enableSharding", "db")); <2>
adminDB.runCommand(new Document("enableSharding", "db")); <2>
Document shardCmd = new Document("shardCollection", "db.users") <3>
Document shardCmd = new Document("shardCollection", "db.users") <3>
.append("key", new Document("country", 1).append("userid", 1)); <4>
adminDB.runCommand(shardCmd);
@@ -45,24 +47,28 @@ adminDB.runCommand(shardCmd);
<1> Sharding commands need to be run against the _admin_ database.
<2> Enable sharding for a specific database if necessary.
<3> Shard a collection within the database having sharding enabled.
<4> Set the shard key (Range based sharding in this case).
<4> Specify the shard key.
This example uses range based sharding.
====
[[sharding.shard-key]]
== Shard Key Handling
The shard key consists of a single or multiple properties present in every document within the target collection, and is used to distribute them across shards.
The shard key consists of a single or multiple properties that must exist in every document in the target collection.
It is used to distribute documents across shards.
Adding the `@Sharded` annotation to an entity enables Spring Data MongoDB to do best effort optimisations required for sharded scenarios when using repositories.
This means essentially adding required shard key information, if not already present, to `replaceOne` filter queries when upserting entities. This may require an additional server round trip to determine the actual value of the current shard key.
Adding the `@Sharded` annotation to an entity enables Spring Data MongoDB to apply best effort optimisations required for sharded scenarios.
This means essentially adding required shard key information, if not already present, to `replaceOne` filter queries when upserting entities.
This may require an additional server round trip to determine the actual value of the current shard key.
TIP: By setting `@Sharded(immutableKey = true)` no attempt will be made to check if an entities shard key changed.
TIP: By setting `@Sharded(immutableKey = true)` Spring Data does not attempt to check if an entity shard key was changed.
Please see the https://docs.mongodb.com/manual/reference/method/db.collection.replaceOne/#upsert[MongoDB Documentation] for further details and the list below for which operations are eligible for auto include the shard key.
Please see the https://docs.mongodb.com/manual/reference/method/db.collection.replaceOne/#upsert[MongoDB Documentation] for further details.
The following list contains which operations are eligible for shard key auto-inclusion:
* `Reactive/CrudRepository.save(...)`
* `Reactive/CrudRepository.saveAll(...)`
* `Reactive/MongoTemplate.save(...)`
* `(Reactive)CrudRepository.save()`
* `(Reactive)CrudRepository.saveAll()`
* `(Reactive)MongoTemplate.save()`