Add CouchBase sample

This commit is contained in:
David Turanski
2020-12-07 09:25:02 -05:00
parent 9d9804c365
commit 15cf920c13
24 changed files with 1779 additions and 0 deletions

View File

@@ -0,0 +1,59 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.spring.example</groupId>
<artifactId>couchbase-stream-applications</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>io.spring.example</groupId>
<artifactId>couchbase-consumer</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>couchbase-consumer</name>
<description>Demo Couchbase Consumer</description>
<dependencies>
<dependency>
<groupId>com.couchbase.client</groupId>
<artifactId>java-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-function-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.cloud.fn</groupId>
<artifactId>config-common</artifactId>
<version>1.0.0-RC1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>couchbase</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,81 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.example.couchbase.consumer;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Function;
import com.couchbase.client.java.Cluster;
import com.couchbase.client.java.ReactiveBucket;
import com.couchbase.client.java.ReactiveCollection;
import com.couchbase.client.java.kv.MutationResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Flux;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.expression.Expression;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
@Configuration
@EnableConfigurationProperties(CouchbaseConsumerProperties.class)
public class CouchbaseConsumerConfiguration {
private static Logger logger = LoggerFactory.getLogger(CouchbaseConsumerConfiguration.class);
@Bean
public Consumer<Flux<Message<?>>> couchbaseConsumer(
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction) {
return message -> couchbaseConsumerFunction.apply(message)
.subscribe(mutationResult ->
logger.debug("Processed " + message));
}
@Bean
public Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction(Cluster cluster,
CouchbaseConsumerProperties consumerProperties) {
return flux -> flux.flatMap(message -> {
logger.debug("Processing message " + message);
String bucketName = bucket(message, consumerProperties.getBucketExpression());
String key = key(message, consumerProperties.getKeyExpression());
ReactiveBucket bucket = cluster.bucket(bucketName).reactive();
ReactiveCollection collection = collection(message, consumerProperties.getCollectionExpression())
.map(name -> bucket.collection(name)).orElse(bucket.defaultCollection());
return collection.upsert(key, value(message, consumerProperties.getValueExpression()));
});
}
private String bucket(Message<?> message, Expression expression) {
return expression.getValue(message, String.class);
}
private String key(Message<?> message, Expression expression) {
return expression.getValue(message, String.class);
}
private Object value(Message<?> message, Expression expression) {
return expression.getValue(message);
}
private Optional<String> collection(Message<?> message, @Nullable Expression expression) {
return expression == null ? Optional.empty() : Optional.of(expression.getValue(message, String.class));
}
}

View File

@@ -0,0 +1,88 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.example.couchbase.consumer;
import javax.validation.constraints.NotNull;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.validation.annotation.Validated;
@ConfigurationProperties("couchbase.consumer")
@Validated
public class CouchbaseConsumerProperties {
private static final String DEFAULT_VALUE_EXPRESSION = "payload";
private final SpelExpressionParser parser = new SpelExpressionParser();
/**
* A SpEL expression to specify the bucket.
*/
private Expression bucketExpression;
/**
* A SpEL expression to specify the key.
*/
private Expression keyExpression;
/**
* A SpEL expression to specify the collection.
*/
private Expression collectionExpression;
/**
* A SpEL expression to specify the value (default is payload).
*/
private Expression valueExpression = parser.parseExpression(DEFAULT_VALUE_EXPRESSION);
@NotNull(message = "'valueExpression' is required")
public Expression getValueExpression() {
return valueExpression;
}
public void setValueExpression(Expression valueExpression) {
this.valueExpression = valueExpression;
}
public Expression getCollectionExpression() {
return collectionExpression;
}
public void setCollectionExpression(Expression collectionExpression) {
this.collectionExpression = collectionExpression;
}
@NotNull(message = "'keyExpression' is required")
public Expression getKeyExpression() {
return keyExpression;
}
public void setKeyExpression(Expression keyExpression) {
this.keyExpression = keyExpression;
}
@NotNull(message = "'bucketExpression' is required")
public Expression getBucketExpression() {
return bucketExpression;
}
public void setBucketExpression(Expression bucketExpression) {
this.bucketExpression = bucketExpression;
}
}

View File

@@ -0,0 +1,292 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.example.couchbase.consumer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Function;
import javax.annotation.PreDestroy;
import com.couchbase.client.java.Bucket;
import com.couchbase.client.java.Cluster;
import com.couchbase.client.java.kv.MutationResult;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.couchbase.BucketDefinition;
import org.testcontainers.couchbase.CouchbaseContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import reactor.core.publisher.Flux;
import reactor.test.StepVerifier;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.messaging.support.MessageBuilder;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
@Testcontainers
public class CouchbaseConsumerTests {
@Container
static CouchbaseContainer container = new CouchbaseContainer("couchbase/server:6.6.0")
.withBucket(new BucketDefinition("test"));
static Map<String, Object> connectProperties = new HashMap<>();
@BeforeAll
static void initialize() {
connectProperties.put("spring.couchbase.connection-string", container.getConnectionString());
connectProperties.put("spring.couchbase.username", container.getUsername());
connectProperties.put("spring.couchbase.password", container.getPassword());
}
private SpringApplicationBuilder applicationBuilder;
@BeforeEach
void setup() {
applicationBuilder = new SpringApplicationBuilder(TestConfig.class).web(WebApplicationType.NONE)
.properties(connectProperties);
}
@Test
void keyExpressionRequired() {
assertThatExceptionOfType(RuntimeException.class).isThrownBy(
() -> applicationBuilder.run("--couchbase.consumer.bucket-expression='test'")) // faster
.havingRootCause()
.withMessageContaining("'keyExpression' is required");
}
@Test
void singleUpsert() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "'test'",
"couchbase.consumer.keyExpression", "payload.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Cluster cluster = context.getBean(Cluster.class);
String bucketName = properties.getBucketExpression().getValue(String.class);
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction = context
.getBean("couchbaseConsumerFunction", Function.class);
StepVerifier.create(couchbaseConsumerFunction
.apply(Flux.just(new GenericMessage<>(new User("David", "david@david.com")))))
.expectNextMatches(mutationResult -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.verifyComplete();
User saved = cluster.bucket(bucketName).defaultCollection().get("david@david.com").contentAs(User.class);
assertThat(saved.getName()).isEqualTo("David");
}
}
@Test
void singleUpsertConsumer() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "'test'",
"couchbase.consumer.keyExpression", "payload.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Cluster cluster = context.getBean(Cluster.class);
String bucketName = properties.getBucketExpression().getValue(String.class);
Consumer<Flux<Message<?>>> couchbaseConsumer = context
.getBean("couchbaseConsumer", Consumer.class);
couchbaseConsumer.accept(Flux.just(new GenericMessage<>(new User("David", "david@david.com"))));
User saved = cluster.bucket(bucketName).defaultCollection().get("david@david.com").contentAs(User.class);
assertThat(saved.getName()).isEqualTo("David");
}
}
@Test
void multipleUpsert() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "'test'",
"couchbase.consumer.keyExpression", "payload.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Cluster cluster = context.getBean(Cluster.class);
String bucketName = properties.getBucketExpression().getValue(String.class);
User user1 = new User("David", "david@david.com");
User user2 = new User("Nanette", "nanette@nanette.com");
User user3 = new User("Soby", "soby@soby.com");
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction = context
.getBean("couchbaseConsumerFunction", Function.class);
StepVerifier.create(couchbaseConsumerFunction
.apply(Flux.just(
new GenericMessage<>(user1),
new GenericMessage<>(user2),
new GenericMessage<>(user3))))
.expectNextMatches(mutationResult -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.expectNextMatches(mutationResult -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.expectNextMatches(mutationResult -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.verifyComplete();
List<User> users = cluster.query("SELECT name,email from test").rowsAs(User.class);
assertThat(users).containsExactlyInAnyOrder(user1, user2, user3);
}
}
@Test
void customBucketExpression() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "headers.bucketName",
"couchbase.consumer.keyExpression", "payload.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Cluster cluster = context.getBean(Cluster.class);
String bucketName = "test";
MessageBuilder.withPayload(new User("David", "david@david.com"))
.copyHeaders(Map.of("bucketName", bucketName)).build();
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction = context
.getBean("couchbaseConsumerFunction", Function.class);
Message<?> message = MessageBuilder.withPayload(new User("David", "david@david.com"))
.copyHeaders(Map.of("bucketName", bucketName)).build();
StepVerifier.create(couchbaseConsumerFunction.apply(Flux.just(message)))
.expectNextMatches(
(MutationResult mutationResult) -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.verifyComplete();
User saved = cluster.bucket(bucketName).defaultCollection().get("david@david.com").contentAs(User.class);
assertThat(saved.getName()).isEqualTo("David");
}
}
@Test
void customValueExpression() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "'test'",
"couchbase.consumer.valueExpression", "payload.user",
"couchbase.consumer.keyExpression", "payload.user.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Cluster cluster = context.getBean(Cluster.class);
String bucketName = properties.getBucketExpression().getValue(String.class);
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction = context
.getBean("couchbaseConsumerFunction", Function.class);
StepVerifier.create(couchbaseConsumerFunction
.apply(Flux.just(new GenericMessage<>(Map.of("user", new User("David", "david@david.com"))))))
.expectNextMatches(mutationResult -> mutationResult.mutationToken().get().bucketName().equals(
bucketName))
.verifyComplete();
Bucket bucket = cluster.bucket(bucketName);
User saved = cluster.bucket(bucketName).defaultCollection().get("david@david.com").contentAs(User.class);
assertThat(saved.getName()).isEqualTo("David");
}
}
@Test
void bucketDoesNotExistShouldThrowException() {
try (ConfigurableApplicationContext context = applicationBuilder
.properties(Map.of(
"couchbase.consumer.bucketExpression", "'users'",
"couchbase.consumer.keyExpression", "payload.email"))
.run()) {
CouchbaseConsumerProperties properties = context.getBean(CouchbaseConsumerProperties.class);
Function<Flux<Message<?>>, Flux<MutationResult>> couchbaseConsumerFunction = context
.getBean("couchbaseConsumerFunction", Function.class);
StepVerifier.create(couchbaseConsumerFunction
.apply(Flux.just(new GenericMessage<>(new User("David", "david@david.com")))))
.expectErrorMatches(e -> e.toString().contains("BUCKET_NOT_AVAILABLE"))
.verify();
}
}
@SpringBootApplication
static class TestConfig {
@Autowired
Cluster cluster;
@PreDestroy
public void destroy() {
cluster.disconnect();
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static class User {
private String name;
private String email;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public User() {
}
User(String name, String email) {
this.name = name;
this.email = email;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
User user = (User) o;
return Objects.equals(name, user.name) &&
Objects.equals(email, user.email);
}
@Override
public int hashCode() {
return Objects.hash(name, email);
}
}
}

View File

@@ -0,0 +1,14 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
<logger name="org.testcontainers" level="INFO"/>
<logger name="com.github.dockerjava" level="WARN"/>
</configuration>