Add Avro serialization and schema management support
- Add schema server implementation - Add schema client abstraction - Add schema client implementation for own schema registry server - Add schema client supporting Confluent schema registry - Add Avro-based message converter supporting a static schema resource - Add Avro-based message converter with schema evolution support, via schema registry client. - On serialization, the converter register writer schemas with the schema registry server and augment the content type of outbound message with schema information. On deserialization, the reading converter will fetch the schema from the server if not available locally. Use class information if schema is not specified In the case of SpecificRecord and Reflective readers/writers, the class information can be used instead Make subtype prefix configurable and shorten the subject - Subtype prefix is now configurable and subject is the lowercase schema name - Enhance/correct javadoc Refine AbstractAvroMessageConverter - distinguish between writer and reader schema when reader is created Add schema registry and schema registry client docs
This commit is contained in:
Vinicius Carvalho
committed by
Marius Bogoevici
Marius Bogoevici
parent
8dd22ebca0
commit
4422b21438
@@ -0,0 +1,252 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.cloud.stream.annotation.EnableBinding;
|
||||
import org.springframework.cloud.stream.annotation.StreamListener;
|
||||
import org.springframework.cloud.stream.messaging.Sink;
|
||||
import org.springframework.cloud.stream.messaging.Source;
|
||||
import org.springframework.cloud.stream.schema.avro.AvroSchemaMessageConverter;
|
||||
import org.springframework.cloud.stream.schema.client.SchemaRegistryClient;
|
||||
import org.springframework.cloud.stream.test.binder.MessageCollector;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.converter.MessageConverter;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
import org.springframework.util.MimeType;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class AvroSchemaMessageConverterTests {
|
||||
|
||||
static StubSchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient();
|
||||
|
||||
@Test
|
||||
public void testSendMessageWithLocation() throws Exception {
|
||||
ConfigurableApplicationContext sourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--schemaLocation=classpath:schemas/users_v1.schema",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=avro/bytes");
|
||||
Source source = sourceContext.getBean(Source.class);
|
||||
User1 firstOutboundFoo = new User1();
|
||||
firstOutboundFoo.setName("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundFoo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(firstOutboundFoo).build());
|
||||
MessageCollector sourceMessageCollector = sourceContext.getBean(MessageCollector.class);
|
||||
Message<?> outboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext barSourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--schemaLocation=classpath:schemas/users_v1.schema",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=avro/bytes");
|
||||
Source barSource = barSourceContext.getBean(Source.class);
|
||||
User2 firstOutboundUser2 = new User2();
|
||||
firstOutboundUser2.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setFavoritePlace("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setName("foo" + UUID.randomUUID().toString());
|
||||
barSource.output().send(MessageBuilder.withPayload(firstOutboundUser2).build());
|
||||
MessageCollector barSourceMessageCollector = barSourceContext.getBean(MessageCollector.class);
|
||||
Message<?> barOutboundMessage = barSourceMessageCollector.forChannel(barSource.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
assertThat(barOutboundMessage).isNotNull();
|
||||
|
||||
|
||||
User2 secondUser2OutboundPojo = new User2();
|
||||
secondUser2OutboundPojo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
secondUser2OutboundPojo.setFavoritePlace("foo" + UUID.randomUUID().toString());
|
||||
secondUser2OutboundPojo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(secondUser2OutboundPojo).build());
|
||||
Message<?> secondBarOutboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext sinkContext = SpringApplication.run(AvroSinkApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false",
|
||||
"--schemaLocation=classpath:schemas/users_v1.schema");
|
||||
Sink sink = sinkContext.getBean(Sink.class);
|
||||
sink.input().send(outboundMessage);
|
||||
sink.input().send(barOutboundMessage);
|
||||
sink.input().send(secondBarOutboundMessage);
|
||||
List<User1> receivedUsers = sinkContext.getBean(AvroSinkApplication.class).receivedUsers;
|
||||
assertThat(receivedUsers).hasSize(3);
|
||||
assertThat(receivedUsers.get(0)).isNotSameAs(firstOutboundFoo);
|
||||
assertThat(receivedUsers.get(0).getFavoriteColor()).isEqualTo(firstOutboundFoo.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(0).getName()).isEqualTo(firstOutboundFoo.getName());
|
||||
|
||||
assertThat(receivedUsers.get(1)).isNotSameAs(firstOutboundUser2);
|
||||
assertThat(receivedUsers.get(1).getFavoriteColor()).isEqualTo(firstOutboundUser2.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(1).getName()).isEqualTo(firstOutboundUser2.getName());
|
||||
|
||||
assertThat(receivedUsers.get(2)).isNotSameAs(secondUser2OutboundPojo);
|
||||
assertThat(receivedUsers.get(2).getFavoriteColor()).isEqualTo(secondUser2OutboundPojo.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(2).getName()).isEqualTo(secondUser2OutboundPojo.getName());
|
||||
|
||||
sourceContext.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSendMessageWithoutLocation() throws Exception {
|
||||
ConfigurableApplicationContext sourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=avro/bytes");
|
||||
Source source = sourceContext.getBean(Source.class);
|
||||
User1 firstOutboundFoo = new User1();
|
||||
firstOutboundFoo.setName("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundFoo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(firstOutboundFoo).build());
|
||||
MessageCollector sourceMessageCollector = sourceContext.getBean(MessageCollector.class);
|
||||
Message<?> outboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext barSourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=avro/bytes");
|
||||
Source barSource = barSourceContext.getBean(Source.class);
|
||||
User2 firstOutboundUser2 = new User2();
|
||||
firstOutboundUser2.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setFavoritePlace("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setName("foo" + UUID.randomUUID().toString());
|
||||
barSource.output().send(MessageBuilder.withPayload(firstOutboundUser2).build());
|
||||
MessageCollector barSourceMessageCollector = barSourceContext.getBean(MessageCollector.class);
|
||||
Message<?> barOutboundMessage = barSourceMessageCollector.forChannel(barSource.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
assertThat(barOutboundMessage).isNotNull();
|
||||
|
||||
|
||||
User2 secondUser2OutboundPojo = new User2();
|
||||
secondUser2OutboundPojo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
secondUser2OutboundPojo.setFavoritePlace("foo" + UUID.randomUUID().toString());
|
||||
secondUser2OutboundPojo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(secondUser2OutboundPojo).build());
|
||||
Message<?> secondBarOutboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext sinkContext = SpringApplication.run(AvroSinkApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.schemaRegistryClient.enabled=false");
|
||||
Sink sink = sinkContext.getBean(Sink.class);
|
||||
sink.input().send(outboundMessage);
|
||||
sink.input().send(barOutboundMessage);
|
||||
sink.input().send(secondBarOutboundMessage);
|
||||
List<User1> receivedUsers = sinkContext.getBean(AvroSinkApplication.class).receivedUsers;
|
||||
assertThat(receivedUsers).hasSize(3);
|
||||
assertThat(receivedUsers.get(0)).isNotSameAs(firstOutboundFoo);
|
||||
assertThat(receivedUsers.get(0).getFavoriteColor()).isEqualTo(firstOutboundFoo.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(0).getName()).isEqualTo(firstOutboundFoo.getName());
|
||||
|
||||
assertThat(receivedUsers.get(1)).isNotSameAs(firstOutboundUser2);
|
||||
assertThat(receivedUsers.get(1).getFavoriteColor()).isEqualTo(firstOutboundUser2.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(1).getName()).isEqualTo(firstOutboundUser2.getName());
|
||||
|
||||
assertThat(receivedUsers.get(2)).isNotSameAs(secondUser2OutboundPojo);
|
||||
assertThat(receivedUsers.get(2).getFavoriteColor()).isEqualTo(secondUser2OutboundPojo.getFavoriteColor());
|
||||
assertThat(receivedUsers.get(2).getName()).isEqualTo(secondUser2OutboundPojo.getName());
|
||||
|
||||
sourceContext.close();
|
||||
}
|
||||
|
||||
|
||||
@EnableBinding(Source.class)
|
||||
@EnableAutoConfiguration
|
||||
@ConfigurationProperties
|
||||
public static class AvroSourceApplication {
|
||||
|
||||
@Bean
|
||||
public SchemaRegistryClient schemaRegistryClient() {
|
||||
return stubSchemaRegistryClient;
|
||||
}
|
||||
|
||||
private Resource schemaLocation;
|
||||
|
||||
public void setSchemaLocation(Resource schemaLocation) {
|
||||
this.schemaLocation = schemaLocation;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public MessageConverter userMessageConverter() throws IOException {
|
||||
AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(
|
||||
MimeType.valueOf("avro/bytes"));
|
||||
if (schemaLocation != null) {
|
||||
avroSchemaMessageConverter.setSchemaLocation(schemaLocation);
|
||||
}
|
||||
return avroSchemaMessageConverter;
|
||||
}
|
||||
}
|
||||
|
||||
@EnableBinding(Sink.class)
|
||||
@EnableAutoConfiguration
|
||||
@ConfigurationProperties
|
||||
public static class AvroSinkApplication {
|
||||
|
||||
public List<User1> receivedUsers = new ArrayList<>();
|
||||
|
||||
@StreamListener(Sink.INPUT)
|
||||
public void listen(User1 user) {
|
||||
receivedUsers.add(user);
|
||||
}
|
||||
|
||||
private Resource schemaLocation;
|
||||
|
||||
public void setSchemaLocation(Resource schemaLocation) {
|
||||
this.schemaLocation = schemaLocation;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public MessageConverter userMessageConverter() throws IOException {
|
||||
AvroSchemaMessageConverter avroSchemaMessageConverter = new AvroSchemaMessageConverter(
|
||||
MimeType.valueOf("avro/bytes"));
|
||||
if (schemaLocation != null) {
|
||||
avroSchemaMessageConverter.setSchemaLocation(schemaLocation);
|
||||
}
|
||||
return avroSchemaMessageConverter;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.cloud.stream.annotation.EnableBinding;
|
||||
import org.springframework.cloud.stream.annotation.StreamListener;
|
||||
import org.springframework.cloud.stream.messaging.Sink;
|
||||
import org.springframework.cloud.stream.messaging.Source;
|
||||
import org.springframework.cloud.stream.schema.client.EnableSchemaRegistryClient;
|
||||
import org.springframework.cloud.stream.schema.client.SchemaRegistryClient;
|
||||
import org.springframework.cloud.stream.schema.server.SchemaRegistryServerApplication;
|
||||
import org.springframework.cloud.stream.test.binder.MessageCollector;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class AvroSchemaRegistryClientMessageConverterTests {
|
||||
|
||||
static SchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient();
|
||||
|
||||
@Test
|
||||
public void testSendMessage() throws Exception {
|
||||
|
||||
ConfigurableApplicationContext schemaRegistryServerContext = SpringApplication.run(
|
||||
SchemaRegistryServerApplication.class);
|
||||
|
||||
ConfigurableApplicationContext sourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=application/*+avro",
|
||||
"--spring.cloud.stream.schema.avro.dynamicSchemaGenerationEnabled=true");
|
||||
Source source = sourceContext.getBean(Source.class);
|
||||
User1 firstOutboundFoo = new User1();
|
||||
firstOutboundFoo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundFoo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(firstOutboundFoo).build());
|
||||
MessageCollector sourceMessageCollector = sourceContext.getBean(MessageCollector.class);
|
||||
Message<?> outboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext barSourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=application/vnd.user1.v1+avro",
|
||||
"--spring.cloud.stream.schema.avro.dynamicSchemaGenerationEnabled=true");
|
||||
Source barSource = barSourceContext.getBean(Source.class);
|
||||
User2 firstOutboundUser2 = new User2();
|
||||
firstOutboundUser2.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setName("foo" + UUID.randomUUID().toString());
|
||||
barSource.output().send(MessageBuilder.withPayload(firstOutboundUser2).build());
|
||||
MessageCollector barSourceMessageCollector = barSourceContext.getBean(MessageCollector.class);
|
||||
Message<?> barOutboundMessage = barSourceMessageCollector.forChannel(barSource.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
assertThat(barOutboundMessage).isNotNull();
|
||||
|
||||
|
||||
User2 secondBarOutboundPojo = new User2();
|
||||
secondBarOutboundPojo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
secondBarOutboundPojo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(secondBarOutboundPojo).build());
|
||||
Message<?> secondBarOutboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext sinkContext = SpringApplication.run(AvroSinkApplication.class,
|
||||
"--server.port=0", "--spring.jmx.enabled=false");
|
||||
Sink sink = sinkContext.getBean(Sink.class);
|
||||
sink.input().send(outboundMessage);
|
||||
sink.input().send(barOutboundMessage);
|
||||
sink.input().send(secondBarOutboundMessage);
|
||||
List<User2> receivedPojos = sinkContext.getBean(AvroSinkApplication.class).receivedPojos;
|
||||
assertThat(receivedPojos).hasSize(3);
|
||||
assertThat(receivedPojos.get(0)).isNotSameAs(firstOutboundFoo);
|
||||
assertThat(receivedPojos.get(0).getFavoriteColor()).isEqualTo(firstOutboundFoo.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(0).getName()).isEqualTo(firstOutboundFoo.getName());
|
||||
assertThat(receivedPojos.get(0).getFavoritePlace()).isEqualTo("NYC");
|
||||
|
||||
assertThat(receivedPojos.get(1)).isNotSameAs(firstOutboundUser2);
|
||||
assertThat(receivedPojos.get(1).getFavoriteColor()).isEqualTo(firstOutboundUser2.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(1).getName()).isEqualTo(firstOutboundUser2.getName());
|
||||
assertThat(receivedPojos.get(1).getFavoritePlace()).isEqualTo("NYC");
|
||||
|
||||
|
||||
assertThat(receivedPojos.get(2)).isNotSameAs(secondBarOutboundPojo);
|
||||
assertThat(receivedPojos.get(2).getFavoriteColor()).isEqualTo(secondBarOutboundPojo.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(2).getName()).isEqualTo(secondBarOutboundPojo.getName());
|
||||
assertThat(receivedPojos.get(2).getFavoritePlace()).isEqualTo(secondBarOutboundPojo.getFavoritePlace());
|
||||
|
||||
sinkContext.close();
|
||||
barSourceContext.close();
|
||||
sourceContext.close();
|
||||
schemaRegistryServerContext.close();
|
||||
}
|
||||
|
||||
@EnableBinding(Source.class)
|
||||
@EnableAutoConfiguration
|
||||
@EnableSchemaRegistryClient
|
||||
public static class AvroSourceApplication {
|
||||
|
||||
}
|
||||
|
||||
@EnableBinding(Sink.class)
|
||||
@EnableAutoConfiguration
|
||||
@EnableSchemaRegistryClient
|
||||
public static class AvroSinkApplication {
|
||||
|
||||
public List<User2> receivedPojos = new ArrayList<>();
|
||||
|
||||
@StreamListener(Sink.INPUT)
|
||||
public void listen(User2 fooPojo) {
|
||||
receivedPojos.add(fooPojo);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.cloud.stream.annotation.EnableBinding;
|
||||
import org.springframework.cloud.stream.annotation.StreamListener;
|
||||
import org.springframework.cloud.stream.messaging.Sink;
|
||||
import org.springframework.cloud.stream.messaging.Source;
|
||||
import org.springframework.cloud.stream.schema.client.SchemaRegistryClient;
|
||||
import org.springframework.cloud.stream.test.binder.MessageCollector;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class AvroStubSchemaRegistryClientMessageConverterTests {
|
||||
|
||||
static SchemaRegistryClient stubSchemaRegistryClient = new StubSchemaRegistryClient();
|
||||
|
||||
@Test
|
||||
public void testSendMessage() throws Exception {
|
||||
ConfigurableApplicationContext sourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=application/*+avro",
|
||||
"--spring.cloud.stream.schema.avro.dynamicSchemaGenerationEnabled=true");
|
||||
Source source = sourceContext.getBean(Source.class);
|
||||
User1 firstOutboundFoo = new User1();
|
||||
firstOutboundFoo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundFoo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(firstOutboundFoo).build());
|
||||
MessageCollector sourceMessageCollector = sourceContext.getBean(MessageCollector.class);
|
||||
Message<?> outboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext barSourceContext = SpringApplication.run(AvroSourceApplication.class,
|
||||
"--server.port=0",
|
||||
"--spring.jmx.enabled=false",
|
||||
"--spring.cloud.stream.bindings.output.contentType=application/vnd.user1.v1+avro",
|
||||
"--spring.cloud.stream.schema.avro.dynamicSchemaGenerationEnabled=true");
|
||||
Source barSource = barSourceContext.getBean(Source.class);
|
||||
User2 firstOutboundUser2 = new User2();
|
||||
firstOutboundUser2.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
firstOutboundUser2.setName("foo" + UUID.randomUUID().toString());
|
||||
barSource.output().send(MessageBuilder.withPayload(firstOutboundUser2).build());
|
||||
MessageCollector barSourceMessageCollector = barSourceContext.getBean(MessageCollector.class);
|
||||
Message<?> barOutboundMessage = barSourceMessageCollector.forChannel(barSource.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
assertThat(barOutboundMessage).isNotNull();
|
||||
|
||||
|
||||
User2 secondBarOutboundPojo = new User2();
|
||||
secondBarOutboundPojo.setFavoriteColor("foo" + UUID.randomUUID().toString());
|
||||
secondBarOutboundPojo.setName("foo" + UUID.randomUUID().toString());
|
||||
source.output().send(MessageBuilder.withPayload(secondBarOutboundPojo).build());
|
||||
Message<?> secondBarOutboundMessage = sourceMessageCollector.forChannel(source.output()).poll(1000,
|
||||
TimeUnit.MILLISECONDS);
|
||||
|
||||
|
||||
ConfigurableApplicationContext sinkContext = SpringApplication.run(AvroSinkApplication.class,
|
||||
"--server.port=0", "--spring.jmx.enabled=false");
|
||||
Sink sink = sinkContext.getBean(Sink.class);
|
||||
sink.input().send(outboundMessage);
|
||||
sink.input().send(barOutboundMessage);
|
||||
sink.input().send(secondBarOutboundMessage);
|
||||
List<User2> receivedPojos = sinkContext.getBean(AvroSinkApplication.class).receivedPojos;
|
||||
assertThat(receivedPojos).hasSize(3);
|
||||
assertThat(receivedPojos.get(0)).isNotSameAs(firstOutboundFoo);
|
||||
assertThat(receivedPojos.get(0).getFavoriteColor()).isEqualTo(firstOutboundFoo.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(0).getName()).isEqualTo(firstOutboundFoo.getName());
|
||||
assertThat(receivedPojos.get(0).getFavoritePlace()).isEqualTo("NYC");
|
||||
|
||||
assertThat(receivedPojos.get(1)).isNotSameAs(firstOutboundUser2);
|
||||
assertThat(receivedPojos.get(1).getFavoriteColor()).isEqualTo(firstOutboundUser2.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(1).getName()).isEqualTo(firstOutboundUser2.getName());
|
||||
assertThat(receivedPojos.get(1).getFavoritePlace()).isEqualTo("NYC");
|
||||
|
||||
|
||||
assertThat(receivedPojos.get(2)).isNotSameAs(secondBarOutboundPojo);
|
||||
assertThat(receivedPojos.get(2).getFavoriteColor()).isEqualTo(secondBarOutboundPojo.getFavoriteColor());
|
||||
assertThat(receivedPojos.get(2).getName()).isEqualTo(secondBarOutboundPojo.getName());
|
||||
assertThat(receivedPojos.get(2).getFavoritePlace()).isEqualTo(secondBarOutboundPojo.getFavoritePlace());
|
||||
|
||||
sourceContext.close();
|
||||
}
|
||||
|
||||
@EnableBinding(Source.class)
|
||||
@EnableAutoConfiguration
|
||||
public static class AvroSourceApplication {
|
||||
|
||||
@Bean
|
||||
public SchemaRegistryClient schemaRegistryClient() {
|
||||
return stubSchemaRegistryClient;
|
||||
}
|
||||
}
|
||||
|
||||
@EnableBinding(Sink.class)
|
||||
@EnableAutoConfiguration
|
||||
public static class AvroSinkApplication {
|
||||
|
||||
public List<User2> receivedPojos = new ArrayList<>();
|
||||
|
||||
@StreamListener(Sink.INPUT)
|
||||
public void listen(User2 fooPojo) {
|
||||
receivedPojos.add(fooPojo);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public SchemaRegistryClient schemaRegistryClient() {
|
||||
return stubSchemaRegistryClient;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.springframework.cloud.stream.schema.SchemaNotFoundException;
|
||||
import org.springframework.cloud.stream.schema.SchemaReference;
|
||||
import org.springframework.cloud.stream.schema.SchemaRegistrationResponse;
|
||||
import org.springframework.cloud.stream.schema.avro.AvroSchemaRegistryClientMessageConverter;
|
||||
import org.springframework.cloud.stream.schema.client.SchemaRegistryClient;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class StubSchemaRegistryClient implements SchemaRegistryClient {
|
||||
|
||||
private final AtomicInteger index = new AtomicInteger(0);
|
||||
|
||||
private final Map<Integer, String> schemasById = new HashMap<>();
|
||||
|
||||
private final Map<String, Map<Integer, SchemaWithId>> storedSchemas = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public SchemaRegistrationResponse register(String subject, String format, String schema) {
|
||||
if (!this.storedSchemas.containsKey(subject)) {
|
||||
this.storedSchemas.put(subject, new TreeMap<Integer, SchemaWithId>());
|
||||
}
|
||||
Map<Integer, SchemaWithId> schemaVersions = this.storedSchemas.get(subject);
|
||||
for (Map.Entry<Integer, SchemaWithId> integerSchemaEntry : schemaVersions.entrySet()) {
|
||||
|
||||
if (integerSchemaEntry.getValue().getSchema().equals(schema)) {
|
||||
SchemaRegistrationResponse schemaRegistrationResponse = new SchemaRegistrationResponse();
|
||||
schemaRegistrationResponse.setId(integerSchemaEntry.getValue().getId());
|
||||
schemaRegistrationResponse.setSchemaReference(
|
||||
new SchemaReference(subject, integerSchemaEntry.getKey(),
|
||||
AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT));
|
||||
return schemaRegistrationResponse;
|
||||
}
|
||||
}
|
||||
int nextVersion = schemaVersions.size() + 1;
|
||||
int id = this.index.incrementAndGet();
|
||||
schemaVersions.put(nextVersion, new SchemaWithId(id, schema));
|
||||
SchemaRegistrationResponse schemaRegistrationResponse = new SchemaRegistrationResponse();
|
||||
schemaRegistrationResponse.setId(this.index.getAndIncrement());
|
||||
schemaRegistrationResponse.setSchemaReference(
|
||||
new SchemaReference(subject, nextVersion, AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT));
|
||||
this.schemasById.put(id, schema);
|
||||
return schemaRegistrationResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String fetch(SchemaReference schemaReference) {
|
||||
if (!AvroSchemaRegistryClientMessageConverter.AVRO_FORMAT.equals(schemaReference.getFormat())) {
|
||||
throw new IllegalArgumentException("Only 'avro' is supported by this client");
|
||||
}
|
||||
if (!this.storedSchemas.containsKey(schemaReference.getSubject())) {
|
||||
throw new SchemaNotFoundException("Not found: " + schemaReference);
|
||||
}
|
||||
if (!this.storedSchemas.get(schemaReference.getSubject()).containsKey(schemaReference.getVersion())) {
|
||||
throw new SchemaNotFoundException("Not found: " + schemaReference);
|
||||
}
|
||||
return this.storedSchemas.get(schemaReference.getSubject()).get(schemaReference.getVersion()).getSchema();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String fetch(Integer id) {
|
||||
return this.schemasById.get(id);
|
||||
}
|
||||
|
||||
static class SchemaWithId {
|
||||
|
||||
int id;
|
||||
|
||||
String schema;
|
||||
|
||||
SchemaWithId(int id, String schema) {
|
||||
this.id = id;
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
public int getId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public String getSchema() {
|
||||
return this.schema;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import org.apache.avro.reflect.Nullable;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class User1 {
|
||||
|
||||
@Nullable
|
||||
private String name;
|
||||
|
||||
private int favoriteNumber;
|
||||
|
||||
@Nullable
|
||||
private String favoriteColor;
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public int getFavoriteNumber() {
|
||||
return this.favoriteNumber;
|
||||
}
|
||||
|
||||
public void setFavoriteNumber(int favoriteNumber) {
|
||||
this.favoriteNumber = favoriteNumber;
|
||||
}
|
||||
|
||||
public String getFavoriteColor() {
|
||||
return this.favoriteColor;
|
||||
}
|
||||
|
||||
public void setFavoriteColor(String favoriteColor) {
|
||||
this.favoriteColor = favoriteColor;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.schema.avro;
|
||||
|
||||
import org.apache.avro.reflect.AvroDefault;
|
||||
import org.apache.avro.reflect.Nullable;
|
||||
|
||||
/**
|
||||
* @author Marius Bogoevici
|
||||
*/
|
||||
public class User2 {
|
||||
|
||||
@Nullable
|
||||
private String name;
|
||||
|
||||
private int favoriteNumber;
|
||||
|
||||
@Nullable
|
||||
private String favoriteColor;
|
||||
|
||||
@AvroDefault("\"NYC\"")
|
||||
private String favoritePlace = "Boston";
|
||||
|
||||
public String getName() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public int getFavoriteNumber() {
|
||||
return this.favoriteNumber;
|
||||
}
|
||||
|
||||
public void setFavoriteNumber(int favoriteNumber) {
|
||||
this.favoriteNumber = favoriteNumber;
|
||||
}
|
||||
|
||||
public String getFavoriteColor() {
|
||||
return this.favoriteColor;
|
||||
}
|
||||
|
||||
public void setFavoriteColor(String favoriteColor) {
|
||||
this.favoriteColor = favoriteColor;
|
||||
}
|
||||
|
||||
public String getFavoritePlace() {
|
||||
return this.favoritePlace;
|
||||
}
|
||||
|
||||
public void setFavoritePlace(String favoritePlace) {
|
||||
this.favoritePlace = favoritePlace;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"namespace":"org.springframework.cloud.stream.samples",
|
||||
"name": "Status",
|
||||
"type" : "record",
|
||||
"fields": [
|
||||
{"name": "id", "type": "string"},
|
||||
{"name": "text", "type": "string"},
|
||||
{"name": "timestamp", "type": "long"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{"namespace": "example.avro",
|
||||
"type": "record",
|
||||
"name": "User",
|
||||
"fields": [
|
||||
{"name": "name", "type": "string"},
|
||||
{"name": "favoriteNumber", "type": ["int", "null"]},
|
||||
{"name": "favoriteColor", "type": ["string", "null"]}
|
||||
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{"namespace": "example.avro",
|
||||
"type": "record",
|
||||
"name": "User",
|
||||
"fields": [
|
||||
{"name": "name", "type": "string"},
|
||||
{"name": "favoriteNumber", "type": ["int", "null"]},
|
||||
{"name": "favoriteColor", "type": ["string", "null"]},
|
||||
{"name": "favoritePlace", "type": ["string","null"], "default" : "NYC"}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user