DATAMONGO-1183 - Add support for Hashed Indexes.

We now support hashed index definitions via IndexOperations. Reading index information back allows to identify a hashed index via isHashed().

Original pull request: #750.
This commit is contained in:
Christoph Strobl
2019-05-15 15:02:47 +02:00
committed by Mark Paluch
parent e06f326de3
commit 89843a1488
10 changed files with 435 additions and 29 deletions

View File

@@ -0,0 +1,71 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.mongodb.core.index;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Marker interface for a property that should be used as key for a
* <a href="https://docs.mongodb.com/manual/core/index-hashed/">Hashed Index</a>. If used on a simple property the index
* uses a hashing function to compute the hash of the value of the index field. Added to a property of complex type the
* embedded document is collapsed and the hash computed for the entire object.
* <p />
*
* <pre>
* <code>
*
* &#64;Document
* public class DomainType {
*
* &#64;HashIndexed @Id String id;
* }
* </code>
* </pre>
*
* {@link HashIndexed} can also be used as meta {@link java.lang.annotation.Annotation} to create composed annotations.
*
* <pre>
* <code>
*
* &#64;Indexed
* &#64;HashIndexed
* &#64;Retention(RetentionPolicy.RUNTIME)
* public @interface IndexAndHash {
*
* &#64;AliasFor(annotation = Indexed.class, attribute = "name")
* String name() default "";
* }
*
* &#64;Document
* public class DomainType {
*
* &#64;ComposedHashIndexed(name = "idx-name") String value;
* }
* </code>
* </pre>
*
* @author Christoph Strobl
* @since 2.2
* @see HashedIndex
*/
@Target({ ElementType.ANNOTATION_TYPE, ElementType.FIELD })
@Retention(RetentionPolicy.RUNTIME)
public @interface HashIndexed {
}

View File

@@ -0,0 +1,66 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.mongodb.core.index;
import org.bson.Document;
import org.springframework.util.Assert;
/**
* {@link IndexDefinition} implementation for MongoDB
* <a href="https://docs.mongodb.com/manual/core/index-hashed/">Hashed Indexes</a> maintaining entries with hashes of
* the values of the indexed field.
*
* @author Christoph Strobl
* @since 2.2
*/
public class HashedIndex implements IndexDefinition {
private final String field;
private HashedIndex(String field) {
Assert.hasText(field, "Field must not be null nor empty!");
this.field = field;
}
/**
* Creates a new {@link HashedIndex} for the given field.
*
* @param field must not be {@literal null} nor empty.
* @return
*/
public static HashedIndex hashed(String field) {
return new HashedIndex(field);
}
/*
* (non-Javadoc)
* @see org.springframework.data.mongodb.core.index.IndexDefinition#getIndexKeys()
*/
@Override
public Document getIndexKeys() {
return new Document(field, "hashed");
}
/*
* (non-Javadoc)
* @see org.springframework.data.mongodb.core.index.IndexDefinition#getIndexOptions()
*/
@Override
public Document getIndexOptions() {
return new Document();
}
}

View File

@@ -30,7 +30,7 @@ import org.springframework.util.ObjectUtils;
public final class IndexField {
enum Type {
GEO, TEXT, DEFAULT;
GEO, TEXT, DEFAULT, HASH;
}
private final String key;
@@ -49,7 +49,9 @@ public final class IndexField {
if (Type.GEO.equals(type) || Type.TEXT.equals(type)) {
Assert.isNull(direction, "Geo/Text indexes must not have a direction!");
} else {
Assert.notNull(direction, "Default indexes require a direction");
if (!Type.HASH.equals(type)) {
Assert.notNull(direction, "Default indexes require a direction");
}
}
this.key = key;
@@ -65,6 +67,17 @@ public final class IndexField {
return new IndexField(key, order, Type.DEFAULT);
}
/**
* Creates a {@literal hashed} {@link IndexField} for the given key.
*
* @param key must not be {@literal null} or empty.
* @return new instance of {@link IndexField}.
* @since 2.2
*/
static IndexField hashed(String key) {
return new IndexField(key, null, Type.HASH);
}
/**
* Creates a geo {@link IndexField} for the given key.
*
@@ -120,6 +133,16 @@ public final class IndexField {
return Type.TEXT.equals(type);
}
/**
* Returns whether the {@link IndexField} is a {@literal hashed}.
*
* @return {@literal true} if {@link IndexField} is hashed.
* @since 2.2
*/
public boolean isHashed() {
return Type.HASH.equals(type);
}
/*
* (non-Javadoc)
* @see java.lang.Object#equals(java.lang.Object)

View File

@@ -96,12 +96,17 @@ public class IndexInfo {
} else {
Double keyValue = new Double(value.toString());
if (ObjectUtils.nullSafeEquals("hashed", value)) {
indexFields.add(IndexField.hashed(key));
} else {
if (ONE.equals(keyValue)) {
indexFields.add(IndexField.create(key, ASC));
} else if (MINUS_ONE.equals(keyValue)) {
indexFields.add(IndexField.create(key, DESC));
Double keyValue = new Double(value.toString());
if (ONE.equals(keyValue)) {
indexFields.add(IndexField.create(key, ASC));
} else if (MINUS_ONE.equals(keyValue)) {
indexFields.add(IndexField.create(key, DESC));
}
}
}
}
@@ -206,6 +211,14 @@ public class IndexInfo {
return Optional.ofNullable(expireAfter);
}
/**
* @return {@literal true} if a hashed index field is present.
* @since 2.2
*/
public boolean isHashed() {
return getIndexFields().stream().anyMatch(IndexField::isHashed);
}
@Override
public String toString() {

View File

@@ -33,7 +33,6 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.domain.Sort;
import org.springframework.data.mapping.Association;
@@ -140,10 +139,10 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
persistentProperty.getFieldName(), Path.of(persistentProperty), root.getCollection(), guard));
}
IndexDefinitionHolder indexDefinitionHolder = createIndexDefinitionHolderForProperty(
List<IndexDefinitionHolder> indexDefinitions = createIndexDefinitionHolderForProperty(
persistentProperty.getFieldName(), root.getCollection(), persistentProperty);
if (indexDefinitionHolder != null) {
indexes.add(indexDefinitionHolder);
if (!indexDefinitions.isEmpty()) {
indexes.addAll(indexDefinitions);
}
} catch (CyclicPropertyReferenceException e) {
LOGGER.info(e.getMessage());
@@ -170,14 +169,14 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
indexInformation.addAll(potentiallyCreateCompoundIndexDefinitions(dotPath, collection, entity));
entity.doWithProperties((PropertyHandler<MongoPersistentProperty>) property -> this
.guradAndPotentiallyAddIndexForProperty(property, dotPath, path, collection, indexInformation, guard));
.guardAndPotentiallyAddIndexForProperty(property, dotPath, path, collection, indexInformation, guard));
indexInformation.addAll(resolveIndexesForDbrefs(dotPath, collection, entity));
return indexInformation;
}
private void guradAndPotentiallyAddIndexForProperty(MongoPersistentProperty persistentProperty, String dotPath,
private void guardAndPotentiallyAddIndexForProperty(MongoPersistentProperty persistentProperty, String dotPath,
Path path, String collection, List<IndexDefinitionHolder> indexes, CycleGuard guard) {
String propertyDotPath = (StringUtils.hasText(dotPath) ? dotPath + "." : "") + persistentProperty.getFieldName();
@@ -194,25 +193,31 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
}
}
IndexDefinitionHolder indexDefinitionHolder = createIndexDefinitionHolderForProperty(propertyDotPath, collection,
List<IndexDefinitionHolder> indexDefinitions = createIndexDefinitionHolderForProperty(propertyDotPath, collection,
persistentProperty);
if (indexDefinitionHolder != null) {
indexes.add(indexDefinitionHolder);
if (!indexDefinitions.isEmpty()) {
indexes.addAll(indexDefinitions);
}
}
@Nullable
private IndexDefinitionHolder createIndexDefinitionHolderForProperty(String dotPath, String collection,
private List<IndexDefinitionHolder> createIndexDefinitionHolderForProperty(String dotPath, String collection,
MongoPersistentProperty persistentProperty) {
List<IndexDefinitionHolder> indices = new ArrayList<>(2);
if (persistentProperty.isAnnotationPresent(Indexed.class)) {
return createIndexDefinition(dotPath, collection, persistentProperty);
indices.add(createIndexDefinition(dotPath, collection, persistentProperty));
} else if (persistentProperty.isAnnotationPresent(GeoSpatialIndexed.class)) {
return createGeoSpatialIndexDefinition(dotPath, collection, persistentProperty);
indices.add(createGeoSpatialIndexDefinition(dotPath, collection, persistentProperty));
}
return null;
if (persistentProperty.isAnnotationPresent(HashIndexed.class)) {
indices.add(createHashedIndexDefinition(dotPath, collection, persistentProperty));
}
return indices;
}
private List<IndexDefinitionHolder> potentiallyCreateCompoundIndexDefinitions(String dotPath, String collection,
@@ -327,7 +332,7 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
*
* @param dotPath The properties {@literal "dot"} path representation from its document root.
* @param fallbackCollection
* @param type
* @param entity
* @return
*/
protected List<IndexDefinitionHolder> createCompoundIndexDefinitions(String dotPath, String fallbackCollection,
@@ -410,14 +415,14 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
*
* @param dotPath The properties {@literal "dot"} path representation from its document root.
* @param collection
* @param persitentProperty
* @param persistentProperty
* @return
*/
@Nullable
protected IndexDefinitionHolder createIndexDefinition(String dotPath, String collection,
MongoPersistentProperty persitentProperty) {
MongoPersistentProperty persistentProperty) {
Indexed index = persitentProperty.findAnnotation(Indexed.class);
Indexed index = persistentProperty.findAnnotation(Indexed.class);
if (index == null) {
return null;
@@ -427,7 +432,8 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
IndexDirection.ASCENDING.equals(index.direction()) ? Sort.Direction.ASC : Sort.Direction.DESC);
if (!index.useGeneratedName()) {
indexDefinition.named(pathAwareIndexName(index.name(), dotPath, persitentProperty.getOwner(), persitentProperty));
indexDefinition
.named(pathAwareIndexName(index.name(), dotPath, persistentProperty.getOwner(), persistentProperty));
}
if (index.unique()) {
@@ -455,7 +461,7 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
}
Duration timeout = computeIndexTimeout(index.expireAfter(),
getEvaluationContextForProperty(persitentProperty.getOwner()));
getEvaluationContextForProperty(persistentProperty.getOwner()));
if (!timeout.isZero() && !timeout.isNegative()) {
indexDefinition.expire(timeout);
}
@@ -464,6 +470,29 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
return new IndexDefinitionHolder(dotPath, indexDefinition, collection);
}
/**
* Creates {@link HashedIndex} wrapped in {@link IndexDefinitionHolder} out of {@link HashIndexed} for given
* {@link MongoPersistentProperty}.
*
* @param dotPath The properties {@literal "dot"} path representation from its document root.
* @param collection
* @param persistentProperty
* @return
* @since 2.2
*/
@Nullable
protected IndexDefinitionHolder createHashedIndexDefinition(String dotPath, String collection,
MongoPersistentProperty persistentProperty) {
HashIndexed index = persistentProperty.findAnnotation(HashIndexed.class);
if (index == null) {
return null;
}
return new IndexDefinitionHolder(dotPath, HashedIndex.hashed(dotPath), collection);
}
/**
* Get the default {@link EvaluationContext}.
*
@@ -588,11 +617,11 @@ public class MongoPersistentEntityIndexResolver implements IndexResolver {
propertyDotPath));
}
IndexDefinitionHolder indexDefinitionHolder = createIndexDefinitionHolderForProperty(propertyDotPath, collection,
List<IndexDefinitionHolder> indexDefinitions = createIndexDefinitionHolderForProperty(propertyDotPath, collection,
property);
if (indexDefinitionHolder != null) {
indexes.add(indexDefinitionHolder);
if (!indexDefinitions.isEmpty()) {
indexes.addAll(indexDefinitions);
}
}

View File

@@ -31,6 +31,7 @@ import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.index.HashedIndex;
import org.springframework.data.mongodb.core.index.Index;
import org.springframework.data.mongodb.core.mapping.Field;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
@@ -112,6 +113,14 @@ public class DefaultIndexOperationsUnitTests {
.isEqualTo(com.mongodb.client.model.Collation.builder().locale("en_US").build());
}
@Test // DATAMONGO-1183
public void shouldCreateHashedIndexCorrectly() {
indexOpsFor(Jedi.class).ensureIndex(HashedIndex.hashed("name"));
verify(collection).createIndex(eq(new Document("firstname", "hashed")), any());
}
private DefaultIndexOperations indexOpsFor(Class<?> type) {
return new DefaultIndexOperations(template, template.getCollectionName(type), type);
}

View File

@@ -25,6 +25,7 @@ import org.springframework.data.domain.Sort.Direction;
* Unit tests for {@link IndexField}.
*
* @author Oliver Gierke
* @author Christoph Strobl
*/
@SuppressWarnings("deprecation")
public class IndexFieldUnitTests {
@@ -68,4 +69,19 @@ public class IndexFieldUnitTests {
assertThat(first, is(second));
assertThat(second, is(first));
}
@Test // DATAMONGO-1183
public void correctTypeForHashedFields() {
assertThat(IndexField.hashed("key").isHashed(), is(true));
}
@Test // DATAMONGO-1183
public void correctEqualsForHashedFields() {
IndexField first = IndexField.hashed("bar");
IndexField second = IndexField.hashed("bar");
assertThat(first, is(second));
assertThat(second, is(first));
}
}

View File

@@ -35,6 +35,7 @@ public class IndexInfoUnitTests {
static final String ID_INDEX = "{ \"v\" : 2, \"key\" : { \"_id\" : 1 }, \"name\" : \"_id_\", \"ns\" : \"db.collection\" }";
static final String INDEX_WITH_PARTIAL_FILTER = "{ \"v\" : 2, \"key\" : { \"k3y\" : 1 }, \"name\" : \"partial-filter-index\", \"ns\" : \"db.collection\", \"partialFilterExpression\" : { \"quantity\" : { \"$gte\" : 10 } } }";
static final String INDEX_WITH_EXPIRATION_TIME = "{ \"v\" : 2, \"key\" : { \"lastModifiedDate\" : 1 },\"name\" : \"expire-after-last-modified\", \"ns\" : \"db.collectio\", \"expireAfterSeconds\" : 3600 }";
static final String HASHED_INDEX = "{ \"v\" : 2, \"key\" : { \"score\" : \"hashed\" }, \"name\" : \"score_hashed\", \"ns\" : \"db.collection\" }";
@Test
public void isIndexForFieldsCorrectly() {
@@ -68,6 +69,16 @@ public class IndexInfoUnitTests {
assertThat(getIndexInfo(ID_INDEX).getExpireAfter()).isEmpty();
}
@Test // DATAMONGO-1183
public void readsHashedIndexCorrectly() {
assertThat(getIndexInfo(HASHED_INDEX).getIndexFields()).containsExactly(IndexField.hashed("score"));
}
@Test // DATAMONGO-1183
public void hashedIndexIsMarkedAsSuch() {
assertThat(getIndexInfo(HASHED_INDEX).isHashed()).isTrue();
}
private static IndexInfo getIndexInfo(String documentJson) {
return IndexInfo.indexInfoOf(Document.parse(documentJson));
}

View File

@@ -1174,6 +1174,60 @@ public class MongoPersistentEntityIndexResolverUnitTests {
"listWithGeneircTypeElement.entity.property_index");
}
@Test // DATAMONGO-1183
public void hashedIndexOnId() {
List<IndexDefinitionHolder> indexDefinitions = prepareMappingContextAndResolveIndexForType(
WithHashedIndexOnId.class);
assertThat(indexDefinitions).hasSize(1);
assertThat(indexDefinitions.get(0)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("_id", "hashed"));
});
}
@Test // DATAMONGO-1183
public void hashedIndex() {
List<IndexDefinitionHolder> indexDefinitions = prepareMappingContextAndResolveIndexForType(WithHashedIndex.class);
assertThat(indexDefinitions).hasSize(1);
assertThat(indexDefinitions.get(0)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("value", "hashed"));
});
}
@Test // DATAMONGO-1183
public void hashedIndexAndIndex() {
List<IndexDefinitionHolder> indexDefinitions = prepareMappingContextAndResolveIndexForType(
WithHashedIndexAndIndex.class);
assertThat(indexDefinitions).hasSize(2);
assertThat(indexDefinitions.get(0)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("value", 1));
});
assertThat(indexDefinitions.get(1)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("value", "hashed"));
});
}
@Test // DATAMONGO-1183
public void hashedIndexAndIndexViaComposedAnnotation() {
List<IndexDefinitionHolder> indexDefinitions = prepareMappingContextAndResolveIndexForType(
WithComposedHashedIndexAndIndex.class);
assertThat(indexDefinitions).hasSize(2);
assertThat(indexDefinitions.get(0)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("value", 1));
assertThat(it.getIndexOptions()).containsEntry("name", "idx-name");
});
assertThat(indexDefinitions.get(1)).satisfies(it -> {
assertThat(it.getIndexKeys()).isEqualTo(new org.bson.Document("value", "hashed"));
});
}
@Document
static class MixedIndexRoot {
@@ -1376,6 +1430,41 @@ public class MongoPersistentEntityIndexResolverUnitTests {
static class EntityWithGenericTypeWrapperAsElement {
List<GenericEntityWrapper<DocumentWithNamedIndex>> listWithGeneircTypeElement;
}
@Document
static class WithHashedIndexOnId {
@HashIndexed @Id String id;
}
@Document
static class WithHashedIndex {
@HashIndexed String value;
}
@Document
static class WithHashedIndexAndIndex {
@Indexed //
@HashIndexed //
String value;
}
@Document
static class WithComposedHashedIndexAndIndex {
@ComposedHashIndexed(name = "idx-name") String value;
}
@HashIndexed
@Indexed
@Retention(RetentionPolicy.RUNTIME)
@interface ComposedHashIndexed {
@AliasFor(annotation = Indexed.class, attribute = "name")
String name() default "";
}
}
private static List<IndexDefinitionHolder> prepareMappingContextAndResolveIndexForType(Class<?> type) {

View File

@@ -414,6 +414,7 @@ The MappingMongoConverter can use metadata to drive the mapping of objects to do
* `@CompoundIndex` (repeatable): Applied at the type level to declare Compound Indexes.
* `@GeoSpatialIndexed`: Applied at the field level to describe how to geoindex the field.
* `@TextIndexed`: Applied at the field level to mark the field to be included in the text index.
* `@HashIndexed`: Applied at the field level for usage within a hashed index to partition data across a sharded cluster.
* `@Language`: Applied at the field level to set the language override property for text index.
* `@Transient`: By default all private fields are mapped to the document, this annotation excludes the field where it is applied from being stored in the database
* `@PersistenceConstructor`: Marks a given constructor - even a package protected one - to use when instantiating the object from the database. Constructor arguments are mapped by name to the key values in the retrieved Document.
@@ -602,6 +603,84 @@ public class Person {
----
====
[[mapping-usage-indexes.hashed-index]]
=== Hashed Indexes
Hashed indexes allow hash based sharding within a sharded cluster. The hashed field value is used to shard collection results
in a more random distribution. For details refer to the https://docs.mongodb.com/manual/core/index-hashed/[MongoDB Documentation].
Here's an example that creates a hashed index for `_id`:
.Example Hashed Index Usage
====
[source,java]
----
@Document
public class DomainType {
@HashIndexed @Id String id;
// ...
}
----
====
Hashed indexes can be created next to other index definitions like shown below, in that case both indices will be created.
.Example Hashed Index Usage togehter with simple index
====
[source,java]
----
@Document
public class DomainType {
@Indexed
@HashIndexed
String value;
// ...
}
----
====
In case the above example is too verbose, a compound annotation allows to reduce the number of annotations present.
.Example Composed Hashed Index Usage
====
[source,java]
----
@Document
public class DomainType {
@IndexAndHash(name = "idx...") <1>
String value;
// ...
}
@Indexed
@HashIndexed
@Retention(RetentionPolicy.RUNTIME)
public @interface IndexAndHash {
@AliasFor(annotation = Indexed.class, attribute = "name") <1>
String name() default "";
}
----
<1> Potentially register an alias for certain attributes of the meta annotation.
====
[NOTE]
====
Although index creation via annotations comes in handy for many scenarios please cosider taking over more control by setting up indices manually via `IndexOperations`.
[source,java]
----
mongoOperations.indexOpsFor(Jedi.class)
.ensureIndex(HashedIndex.hashed("useTheForce"));
----
====
[[mapping-usage-indexes.text-index]]
=== Text Indexes