Added support of the expression based durability levels. (#1721)

Closes #1063.
This commit is contained in:
Tigran Babloyan
2023-04-24 20:17:19 +04:00
committed by GitHub
parent e91322bd13
commit 7edc00e10c
20 changed files with 238 additions and 19 deletions

View File

@@ -78,8 +78,8 @@ This key needs to be any string with a length of maximum 250 characters.
Feel free to use whatever fits your use case, be it a UUID, an email address or anything else.
Writes to Couchbase-Server buckets can optionally be assigned durability requirements; which instruct Couchbase Server to update the specified document on multiple nodes in memory and/or disk locations across the cluster; before considering the write to be committed.
Default durability requirements can also be configured through the `@Document` annotation.
For example: `@Document(durabilityLevel = DurabilityLevel.MAJORITY)` will force mutations to be replicated to a majority of the Data Service nodes.
Default durability requirements can also be configured through the `@Document` or `@Durability` annotations.
For example: `@Document(durabilityLevel = DurabilityLevel.MAJORITY)` will force mutations to be replicated to a majority of the Data Service nodes. Both of the annotations support expression based durability level assignment via `durabilityExpression` attribute (Note SPEL is not supported).
[[datatypes]]
== Datatypes and Converters

View File

@@ -1,10 +1,10 @@
= Spring Data Couchbase - Reference Documentation
Michael Nitschinger, Oliver Gierke, Simon Basle, Michael Reiche
Michael Nitschinger, Oliver Gierke, Simon Basle, Michael Reiche, Tigran Babloyan
:revnumber: {version}
:revdate: {localdate}
:spring-data-commons-docs: ../../../../spring-data-commons/src/main/asciidoc
(C) 2014-2022 The original author(s).
(C) 2014-2023 The original author(s).
NOTE: Copies of this document may be made for your own use and for distribution to others, provided that you do not charge any fee for such copies and further provided that each copy contains this Copyright Notice, whether distributed in print or electronically.

View File

@@ -40,7 +40,7 @@ public class ExecutableInsertByIdOperationSupport implements ExecutableInsertByI
Assert.notNull(domainType, "DomainType must not be null!");
return new ExecutableInsertByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null);
}

View File

@@ -51,7 +51,7 @@ public class ExecutableMutateInByIdOperationSupport implements ExecutableMutateI
Assert.notNull(domainType, "DomainType must not be null!");
return new ExecutableMutateInByIdSupport(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
Collections.emptyList(), false);
}

View File

@@ -52,7 +52,7 @@ public class ExecutableRemoveByIdOperationSupport implements ExecutableRemoveByI
return new ExecutableRemoveByIdSupport(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null);
}

View File

@@ -40,7 +40,7 @@ public class ExecutableReplaceByIdOperationSupport implements ExecutableReplaceB
Assert.notNull(domainType, "DomainType must not be null!");
return new ExecutableReplaceByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null);
}

View File

@@ -40,7 +40,7 @@ public class ExecutableUpsertByIdOperationSupport implements ExecutableUpsertByI
Assert.notNull(domainType, "DomainType must not be null!");
return new ExecutableUpsertByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null);
}

View File

@@ -61,7 +61,7 @@ public class ReactiveInsertByIdOperationSupport implements ReactiveInsertByIdOpe
Assert.notNull(domainType, "DomainType must not be null!");
return new ReactiveInsertByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null, template.support());
}

View File

@@ -52,7 +52,7 @@ public class ReactiveMutateInByIdOperationSupport implements ReactiveMutateInByI
Assert.notNull(domainType, "DomainType must not be null!");
return new ReactiveMutateInByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null, template.support(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
Collections.emptyList(), false);
}

View File

@@ -67,7 +67,7 @@ public class ReactiveRemoveByIdOperationSupport implements ReactiveRemoveByIdOpe
public ReactiveRemoveById removeById(Class<?> domainType) {
return new ReactiveRemoveByIdSupport(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null);
}

View File

@@ -64,7 +64,7 @@ public class ReactiveReplaceByIdOperationSupport implements ReactiveReplaceByIdO
Assert.notNull(domainType, "DomainType must not be null!");
return new ReactiveReplaceByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null, template.support());
}

View File

@@ -53,7 +53,7 @@ public class ReactiveUpsertByIdOperationSupport implements ReactiveUpsertByIdOpe
Assert.notNull(domainType, "DomainType must not be null!");
return new ReactiveUpsertByIdSupport<>(template, domainType, OptionsBuilder.getScopeFrom(domainType),
OptionsBuilder.getCollectionFrom(domainType), null, OptionsBuilder.getPersistTo(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType),
OptionsBuilder.getReplicateTo(domainType), OptionsBuilder.getDurabilityLevel(domainType, template.getConverter()),
null, template.support());
}

View File

@@ -21,6 +21,7 @@ import java.util.Calendar;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import com.couchbase.client.core.msg.kv.DurabilityLevel;
import org.springframework.context.EnvironmentAware;
import org.springframework.core.annotation.AnnotatedElementUtils;
import org.springframework.core.env.Environment;
@@ -36,6 +37,7 @@ import org.springframework.util.StringUtils;
* @author Michael Nitschinger
* @author Mark Paluch
* @author Michael Reiche
* @author Tigran Babloyan
*/
public class BasicCouchbasePersistentEntity<T> extends BasicPersistentEntity<T, CouchbasePersistentProperty>
implements CouchbasePersistentEntity<T>, EnvironmentAware {
@@ -50,6 +52,7 @@ public class BasicCouchbasePersistentEntity<T> extends BasicPersistentEntity<T,
public BasicCouchbasePersistentEntity(final TypeInformation<T> typeInformation) {
super(typeInformation);
validateExpirationConfiguration();
validateDurabilityConfiguration();
}
private void validateExpirationConfiguration() {
@@ -61,6 +64,15 @@ public class BasicCouchbasePersistentEntity<T> extends BasicPersistentEntity<T,
}
}
private void validateDurabilityConfiguration() {
Document annotation = getType().getAnnotation(Document.class);
if (annotation != null && annotation.durabilityLevel() != DurabilityLevel.NONE && StringUtils.hasLength(annotation.durabilityExpression())) {
String msg = String.format("Incorrect durability configuration on class %s using %s. "
+ "You cannot use 'durabilityLevel' and 'durabilityExpression' at the same time", getType().getName(), annotation);
throw new IllegalArgumentException(msg);
}
}
@Override
public void setEnvironment(Environment environment) {
this.environment = environment;
@@ -158,6 +170,30 @@ public class BasicCouchbasePersistentEntity<T> extends BasicPersistentEntity<T,
return expiryValue;
}
@Override
public DurabilityLevel getDurabilityLevel() {
return getDurabilityLevel(AnnotatedElementUtils.findMergedAnnotation(getType(), Durability.class), environment);
}
private static DurabilityLevel getDurabilityLevel(Durability annotation, Environment environment) {
if (annotation == null) {
return DurabilityLevel.NONE;
}
DurabilityLevel durabilityLevel = annotation.durabilityLevel();
String durabilityExpressionString = annotation.durabilityExpression();
if (StringUtils.hasLength(durabilityExpressionString)) {
Assert.notNull(environment, "Environment must be set to use 'durabilityExpressionString'");
String durabilityWithReplacedPlaceholders = environment.resolveRequiredPlaceholders(durabilityExpressionString);
try {
durabilityLevel = DurabilityLevel.valueOf(durabilityWithReplacedPlaceholders);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
"Invalid value for durability expression: " + durabilityWithReplacedPlaceholders);
}
}
return durabilityLevel;
}
@Override
public boolean isTouchOnRead() {
org.springframework.data.couchbase.core.mapping.Document annotation = getType()

View File

@@ -18,6 +18,7 @@ package org.springframework.data.couchbase.core.mapping;
import java.time.Duration;
import com.couchbase.client.core.msg.kv.DurabilityLevel;
import org.springframework.data.mapping.PersistentEntity;
/**
@@ -25,6 +26,7 @@ import org.springframework.data.mapping.PersistentEntity;
*
* @author Michael Nitschinger
* @author Michael Reiche
* @author Tigran Babloyan
*/
public interface CouchbasePersistentEntity<T> extends PersistentEntity<T, CouchbasePersistentProperty> {
@@ -54,6 +56,15 @@ public interface CouchbasePersistentEntity<T> extends PersistentEntity<T, Couchb
*/
Duration getExpiryDuration();
/**
* Returns the durability level of the entity.
* <p>
* Allows the application to wait until this replication (or persistence) is successful before proceeding
*
* @return the durability level.
*/
DurabilityLevel getDurabilityLevel();
/**
* Flag for using getAndTouch operations for reads, resetting the expiration (if one was set) when the entity is
* directly read (eg. findOne, findById).

View File

@@ -47,6 +47,7 @@ import com.couchbase.client.java.query.QueryScanConsistency;
@Target({ ElementType.TYPE })
@Expiry
@ScanConsistency
@Durability
public @interface Document {
/**
@@ -105,5 +106,16 @@ public @interface Document {
* The optional durabilityLevel for all mutating operations, allows the application to wait until this replication
* (or persistence) is successful before proceeding
*/
@AliasFor(annotation = Durability.class, attribute = "durabilityLevel")
DurabilityLevel durabilityLevel() default DurabilityLevel.NONE;
/**
* Same as {@link #durabilityLevel()} but allows the actual value to be set using standard Spring property sources mechanism.
* Only one might be set at the same time: either {@link #durabilityLevel()} or {@link #durabilityExpression()}. <br />
* Syntax is the same as for {@link org.springframework.core.env.Environment#resolveRequiredPlaceholders(String)}.
* <br />
* SpEL is NOT supported.
*/
@AliasFor(annotation = Durability.class, attribute = "durabilityExpression")
String durabilityExpression() default "";
}

View File

@@ -0,0 +1,32 @@
package org.springframework.data.couchbase.core.mapping;
import com.couchbase.client.core.msg.kv.DurabilityLevel;
import org.springframework.data.annotation.Persistent;
import java.lang.annotation.*;
/**
* Durability annotation
*
* @author Tigran Babloyan
*/
@Persistent
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE, ElementType.ANNOTATION_TYPE })
public @interface Durability {
/**
* The optional durabilityLevel for all mutating operations, allows the application to wait until this replication
* (or persistence) is successful before proceeding
*/
DurabilityLevel durabilityLevel() default DurabilityLevel.NONE;
/**
* Same as {@link #durabilityLevel()} but allows the actual value to be set using standard Spring property sources mechanism.
* Only one might be set at the same time: either {@link #durabilityLevel()} or {@link #durabilityExpression()}. <br />
* Syntax is the same as for {@link org.springframework.core.env.Environment#resolveRequiredPlaceholders(String)}.
* <br />
* SpEL is NOT supported.
*/
String durabilityExpression() default "";
}

View File

@@ -35,7 +35,9 @@ import com.couchbase.client.core.error.InvalidArgumentException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.AnnotatedElementUtils;
import org.springframework.data.couchbase.core.convert.CouchbaseConverter;
import org.springframework.data.couchbase.core.mapping.CouchbaseDocument;
import org.springframework.data.couchbase.core.mapping.CouchbasePersistentEntity;
import org.springframework.data.couchbase.core.mapping.Document;
import org.springframework.data.couchbase.repository.Collection;
import org.springframework.data.couchbase.repository.ScanConsistency;
@@ -284,12 +286,14 @@ public class OptionsBuilder {
return null;
}
public static DurabilityLevel getDurabilityLevel(Class<?> domainType) {
public static DurabilityLevel getDurabilityLevel(Class<?> domainType, CouchbaseConverter converter) {
if (domainType == null) {
return DurabilityLevel.NONE;
}
Document document = AnnotatedElementUtils.findMergedAnnotation(domainType, Document.class);
return document != null ? document.durabilityLevel() : DurabilityLevel.NONE;
final CouchbasePersistentEntity<?> entity = converter.getMappingContext()
.getRequiredPersistentEntity(domainType);
return entity.getDurabilityLevel();
}
public static PersistTo getPersistTo(Class<?> domainType) {

View File

@@ -46,6 +46,7 @@ import org.springframework.data.couchbase.domain.*;
import org.springframework.data.couchbase.util.ClusterType;
import org.springframework.data.couchbase.util.IgnoreWhen;
import org.springframework.data.couchbase.util.JavaIntegrationTests;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import com.couchbase.client.core.error.CouchbaseException;
@@ -63,6 +64,7 @@ import com.couchbase.client.java.query.QueryScanConsistency;
*/
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
@SpringJUnitConfig(Config.class)
@TestPropertySource(properties = { "valid.document.durability = MAJORITY" })
class CouchbaseTemplateKeyValueIntegrationTests extends JavaIntegrationTests {
@Autowired public CouchbaseTemplate couchbaseTemplate;
@@ -290,7 +292,8 @@ class CouchbaseTemplateKeyValueIntegrationTests extends JavaIntegrationTests {
@Test
void withDurability()
throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
for (Class<?> clazz : new Class[] { User.class, UserAnnotatedDurability.class, UserAnnotatedPersistTo.class, UserAnnotatedReplicateTo.class }) {
for (Class<?> clazz : new Class[] { User.class, UserAnnotatedDurability.class, UserAnnotatedDurabilityExpression.class,
UserAnnotatedPersistTo.class, UserAnnotatedReplicateTo.class }) {
// insert, replace, upsert
for (OneAndAllEntity<User> operator : new OneAndAllEntity[]{couchbaseTemplate.insertById(clazz),
couchbaseTemplate.replaceById(clazz), couchbaseTemplate.upsertById(clazz)}) {
@@ -1132,6 +1135,31 @@ class CouchbaseTemplateKeyValueIntegrationTests extends JavaIntegrationTests {
couchbaseTemplate.removeById(UserAnnotatedDurability.class).one(user.getId());
}
@Test
void insertByIdWithAnnotatedDurabilityExpression() {
UserAnnotatedDurabilityExpression user = new UserAnnotatedDurabilityExpression(UUID.randomUUID().toString(), "firstname", "lastname");
UserAnnotatedDurabilityExpression inserted = null;
// occasionally gives "reactor.core.Exceptions$OverflowException: Could not emit value due to lack of requests"
for (int i = 1; i != 5; i++) {
try {
inserted = couchbaseTemplate.insertById(UserAnnotatedDurabilityExpression.class)
.one(user);
break;
} catch (Exception ofe) {
System.out.println("" + i + " caught: " + ofe);
couchbaseTemplate.removeByQuery(UserAnnotatedDurabilityExpression.class).withConsistency(QueryScanConsistency.REQUEST_PLUS).all();
if (i == 4) {
throw ofe;
}
sleepSecs(1);
}
}
assertEquals(user, inserted);
assertThrows(DuplicateKeyException.class, () -> couchbaseTemplate.insertById(UserAnnotatedDurabilityExpression.class).one(user));
couchbaseTemplate.removeById(UserAnnotatedDurabilityExpression.class).one(user.getId());
}
@Test
void existsById() {
String id = UUID.randomUUID().toString();

View File

@@ -23,6 +23,7 @@ import java.util.Calendar;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import com.couchbase.client.core.msg.kv.DurabilityLevel;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
@@ -33,7 +34,8 @@ import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
@SpringJUnitConfig
@TestPropertySource(properties = { "valid.document.expiry = 10", "invalid.document.expiry = abc" })
@TestPropertySource(properties = { "valid.document.expiry = 10", "invalid.document.expiry = abc",
"invalid.document.durability = some", "valid.document.durability = MAJORITY" })
public class BasicCouchbasePersistentEntityTests {
@Autowired ConfigurableEnvironment environment;
@@ -180,6 +182,34 @@ public class BasicCouchbasePersistentEntityTests {
() -> getBasicCouchbasePersistentEntity(ExpiryAndExpression.class).getExpiry());
}
@Test
void doesNotAllowUseDurabilityAndExpressionSimultaneously() {
assertThrows(IllegalArgumentException.class,
() -> getBasicCouchbasePersistentEntity(DurabilityAndExpression.class).getDurabilityLevel());
}
@Test
void failsIfDurabilityExpressionMissesRequiredProperty() {
assertThrows(IllegalArgumentException.class,
() -> getBasicCouchbasePersistentEntity(DurabilityWithMissingProperty.class).getDurabilityLevel());
}
@Test
void doesNotAllowUseDurabilityFromInvalidExpression() {
assertThrows(IllegalArgumentException.class,
() -> getBasicCouchbasePersistentEntity(DurabilityWithInvalidExpression.class).getDurabilityLevel());
}
@Test
void usesGetDurabilityExpression() {
assertThat(getBasicCouchbasePersistentEntity(ConstantDurabilityExpression.class).getDurabilityLevel()).isEqualTo(DurabilityLevel.MAJORITY);
}
@Test
void usesGetDurabilityFromValidExpression() {
assertThat(getBasicCouchbasePersistentEntity(DurabilityWithValidExpression.class).getDurabilityLevel()).isEqualTo(DurabilityLevel.MAJORITY);
}
private BasicCouchbasePersistentEntity getBasicCouchbasePersistentEntity(Class<?> clazz) {
BasicCouchbasePersistentEntity basicCouchbasePersistentEntity = new BasicCouchbasePersistentEntity(
ClassTypeInformation.from(clazz));
@@ -264,4 +294,34 @@ public class BasicCouchbasePersistentEntityTests {
@Document(expiry = 10, expiryExpression = "10")
class ExpiryAndExpression {}
/**
* Simple POJO to test that durability and durability expression cannot be used simultaneously
*/
@Document(durabilityLevel = DurabilityLevel.MAJORITY, durabilityExpression = "MAJORITY")
class DurabilityAndExpression {}
/**
* Simple POJO to test durability expression logic failure to resolve property placeholder
*/
@Document(durabilityExpression = "${missing.durability}")
class DurabilityWithMissingProperty {}
/**
* Simple POJO to test invalid durability expression
*/
@Document(durabilityExpression = "${invalid.document.durability}")
class DurabilityWithInvalidExpression {}
/**
* Simple POJO to test constant expiry expression
*/
@Document(durabilityExpression = "MAJORITY")
class ConstantDurabilityExpression {}
/**
* Simple POJO to test valid expiry expression by resolving simple property from environment
*/
@Document(durabilityExpression = "${valid.document.durability}")
class DurabilityWithValidExpression {}
}

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2020-2023 the original author or authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.couchbase.domain;
import com.couchbase.client.core.msg.kv.DurabilityLevel;
import org.springframework.data.couchbase.core.mapping.Document;
import java.io.Serializable;
/**
* Annotated User entity for tests
*
* @author Tigran Babloyan
*/
@Document(durabilityExpression = "${valid.document.durability}")
public class UserAnnotatedDurabilityExpression extends User implements Serializable {
public UserAnnotatedDurabilityExpression(String id, String firstname, String lastname) {
super(id, firstname, lastname);
}
}