INT-3045: Add in & out ZeroMq channel adapters (#3388)
* INT-3045: Add in & out ZeroMq channel adapters JIRA: https://jira.spring.io/browse/INT-3045 * Add `ZeroMqMessageHandler` to produce messages into one-way ZeroMq sockets * Add `ZeroMqMessageProducer` to consumer messages from one-way ZeroMq sockets * Add `ConvertingBytesMessageMapper` impl for the `BytesMessageMapper` to delegate an actual conversion into the provided `MessageConverter` * Add `ZeroMqHeaders` for message headers constants representing ZeroMq message attributes * Fix `ZeroMqChannel` for the proper deferred `zeroMqProxy` evaluation * Add more JavaDocs * Fix `ZeroMqChannelTests.testPubSubBind()` to be sure that really all the subscribed channels get the same message from the `PUB` socket * * Fix typo in the `ConvertingBytesMessageMapper` * Add `this` for `doOnError()` in the `ZeroMqChannel` & `ZeroMqMessageProducer` * Change the bind logic in the `ZeroMqMessageProducer` to `port` and let it to bind to random port. The actual port is available later via `getBoundPort()` * Introduce a `ZeroMqMessageProducer.receiveRaw()` to let received `ZMsg` to be produce as a `payload` * Add a logic into `ZeroMqMessageHandler` to treat `ZMsg` in the payload of request message as is without any conversion * Fix race condition in the `ZeroMqMessageProducer` to destroy `consumerScheduler` when the main `Flux` is complete * * Add Java DSL for ZeroMq components * Extract `ReactiveMessageHandlerSpec` for `ReactiveMessageHandler` impls * Add debug message into `EmbeddedJsonHeadersMessageMapper` when cannot `decodeNativeFormat()` * Make `ReactiveMongoDbMessageHandlerSpec` extending `ReactiveMessageHandlerSpec` * Make `ZeroMqProxy` `autoStartup` by default * Add `ZeroMqDslTests` to cover all the Java DSL for ZeroMq * Introduce a `MimeTypeSerializer` to serialize a `MimeType` into JSON as a plain string; use it as extra serializer in the `JacksonJsonUtils.messagingAwareMapper()` * Fix typo for the `AllowListTypeResolverBuilder` inner class * * Add some docs * Fix Checkstyle violations * * More docs * Fix language in Docs Co-authored-by: Gary Russell <grussell@vmware.com> Co-authored-by: Gary Russell <grussell@vmware.com>
This commit is contained in:
Artem Bilan
@@ -114,8 +114,8 @@ public class ZeroMqChannelTests {
|
||||
await().until(() -> proxy.getBackendPort() > 0);
|
||||
|
||||
ZMQ.Socket captureSocket = CONTEXT.createSocket(SocketType.SUB);
|
||||
captureSocket.connect(proxy.getCaptureAddress());
|
||||
captureSocket.subscribe(ZMQ.SUBSCRIPTION_ALL);
|
||||
captureSocket.connect(proxy.getCaptureAddress());
|
||||
|
||||
ZeroMqChannel channel = new ZeroMqChannel(CONTEXT);
|
||||
channel.setConnectUrl("tcp://localhost:" + proxy.getFrontendPort() + ':' + proxy.getBackendPort());
|
||||
@@ -170,10 +170,13 @@ public class ZeroMqChannelTests {
|
||||
ZeroMqChannel channel2 = new ZeroMqChannel(CONTEXT, true);
|
||||
channel2.setConnectUrl("tcp://localhost:" + proxy.getFrontendPort() + ':' + proxy.getBackendPort());
|
||||
channel2.setBeanName("testChannel5");
|
||||
channel.setConsumeDelay(Duration.ofMillis(10));
|
||||
channel2.setConsumeDelay(Duration.ofMillis(10));
|
||||
channel2.afterPropertiesSet();
|
||||
|
||||
channel.subscribe(received::offer);
|
||||
channel2.subscribe(received::offer);
|
||||
|
||||
// Give it some time to connect and subscribe
|
||||
Thread.sleep(1000);
|
||||
|
||||
GenericMessage<String> testMessage = new GenericMessage<>("test1");
|
||||
assertThat(channel.send(testMessage)).isTrue();
|
||||
@@ -187,6 +190,7 @@ public class ZeroMqChannelTests {
|
||||
assertThat(received.poll(100, TimeUnit.MILLISECONDS)).isNull();
|
||||
|
||||
channel.destroy();
|
||||
channel2.destroy();
|
||||
proxy.stop();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* Copyright 2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.zeromq.dsl;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.zeromq.SocketType;
|
||||
import org.zeromq.ZContext;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.integration.config.EnableIntegration;
|
||||
import org.springframework.integration.dsl.IntegrationFlow;
|
||||
import org.springframework.integration.dsl.IntegrationFlows;
|
||||
import org.springframework.integration.dsl.Transformers;
|
||||
import org.springframework.integration.dsl.context.IntegrationFlowContext;
|
||||
import org.springframework.integration.zeromq.ZeroMqHeaders;
|
||||
import org.springframework.integration.zeromq.ZeroMqProxy;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
|
||||
import org.springframework.util.SocketUtils;
|
||||
|
||||
/**
|
||||
* @author Artem Bilan
|
||||
*
|
||||
* @since 5.4
|
||||
*/
|
||||
@SpringJUnitConfig
|
||||
@DirtiesContext
|
||||
public class ZeroMqDslTests {
|
||||
|
||||
private static final int PROXY_PUB_PORT = SocketUtils.findAvailableTcpPort();
|
||||
|
||||
@Autowired
|
||||
ZContext context;
|
||||
|
||||
@Autowired
|
||||
@Qualifier("publishToZeroMqPubSubFlow.input")
|
||||
MessageChannel publishToZeroMqPubSubFlowInput;
|
||||
|
||||
@Autowired
|
||||
ZeroMqProxy subPubZeroMqProxy;
|
||||
|
||||
@Autowired
|
||||
ZeroMqProxy pullPushZeroMqProxy;
|
||||
|
||||
@Autowired
|
||||
IntegrationFlowContext integrationFlowContext;
|
||||
|
||||
@Test
|
||||
void testZeroMqDslIntegration() throws InterruptedException {
|
||||
BlockingQueue<Message<?>> results = new LinkedBlockingQueue<>();
|
||||
|
||||
await().until(() -> this.subPubZeroMqProxy.getBackendPort() > 0);
|
||||
|
||||
for (int i = 0; i < 2; i++) {
|
||||
IntegrationFlow consumerFlow =
|
||||
IntegrationFlows.from(
|
||||
ZeroMq.inboundChannelAdapter(this.context, SocketType.SUB)
|
||||
.connectUrl("tcp://localhost:" + this.subPubZeroMqProxy.getBackendPort())
|
||||
.topics("someTopic")
|
||||
.consumeDelay(Duration.ofMillis(100)))
|
||||
.channel(ZeroMq.zeroMqChannel(this.context).zeroMqProxy(this.pullPushZeroMqProxy))
|
||||
.transform(Transformers.objectToString())
|
||||
.handle(results::offer)
|
||||
.get();
|
||||
|
||||
this.integrationFlowContext.registration(consumerFlow).register();
|
||||
}
|
||||
|
||||
// Give it some time to connect and subscribe
|
||||
Thread.sleep(2000);
|
||||
|
||||
this.publishToZeroMqPubSubFlowInput.send(new GenericMessage<>("test"));
|
||||
|
||||
Message<?> message = results.poll(10, TimeUnit.SECONDS);
|
||||
assertThat(message).isNotNull()
|
||||
.extracting(Message::getPayload)
|
||||
.isEqualTo("test");
|
||||
|
||||
assertThat(message.getHeaders()).containsEntry(ZeroMqHeaders.TOPIC, "someTopic");
|
||||
|
||||
message = results.poll(1, TimeUnit.SECONDS);
|
||||
assertThat(message).isNotNull()
|
||||
.extracting(Message::getPayload)
|
||||
.isEqualTo("test");
|
||||
|
||||
// With Pub/Sub channel we would have 4 messages.
|
||||
assertThat(results.poll(1, TimeUnit.SECONDS)).isNull();
|
||||
|
||||
this.integrationFlowContext.getRegistry()
|
||||
.values()
|
||||
.forEach(IntegrationFlowContext.IntegrationFlowRegistration::destroy);
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@EnableIntegration
|
||||
public static class ContextConfiguration {
|
||||
|
||||
@Bean
|
||||
ZContext context() {
|
||||
return new ZContext();
|
||||
}
|
||||
|
||||
@Bean
|
||||
ZeroMqProxy subPubZeroMqProxy() {
|
||||
ZeroMqProxy zeroMqProxy = new ZeroMqProxy(context(), ZeroMqProxy.Type.SUB_PUB);
|
||||
zeroMqProxy.setFrontendPort(PROXY_PUB_PORT);
|
||||
return zeroMqProxy;
|
||||
}
|
||||
|
||||
@Bean
|
||||
ZeroMqProxy pullPushZeroMqProxy() {
|
||||
return new ZeroMqProxy(context());
|
||||
}
|
||||
|
||||
@Bean
|
||||
IntegrationFlow publishToZeroMqPubSubFlow() {
|
||||
return flow ->
|
||||
flow.handle(ZeroMq.outboundChannelAdapter(context(), "tcp://localhost:" + PROXY_PUB_PORT,
|
||||
SocketType.PUB)
|
||||
.topic("someTopic"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
/*
|
||||
* Copyright 2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.zeromq.inbound;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.assertj.core.api.InstanceOfAssertFactories;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.zeromq.SocketType;
|
||||
import org.zeromq.ZContext;
|
||||
import org.zeromq.ZFrame;
|
||||
import org.zeromq.ZMQ;
|
||||
import org.zeromq.ZMsg;
|
||||
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.integration.channel.FluxMessageChannel;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
import reactor.test.StepVerifier;
|
||||
|
||||
/**
|
||||
* @author Artem Bilan
|
||||
*
|
||||
* @since 5.4
|
||||
*/
|
||||
public class ZeroMqMessageProducerTests {
|
||||
|
||||
private static final ZContext CONTEXT = new ZContext();
|
||||
|
||||
@AfterAll
|
||||
static void tearDown() {
|
||||
CONTEXT.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMessageProducerForPair() {
|
||||
FluxMessageChannel outputChannel = new FluxMessageChannel();
|
||||
|
||||
StepVerifier stepVerifier =
|
||||
StepVerifier.create(outputChannel)
|
||||
.assertNext((message) -> assertThat(message.getPayload()).isEqualTo("test"))
|
||||
.assertNext((message) -> assertThat(message.getPayload()).isEqualTo("test2"))
|
||||
.thenCancel()
|
||||
.verifyLater();
|
||||
|
||||
ZeroMqMessageProducer messageProducer = new ZeroMqMessageProducer(CONTEXT);
|
||||
messageProducer.setOutputChannel(outputChannel);
|
||||
messageProducer.setMessageMapper((object, headers) -> new GenericMessage<>(new String(object)));
|
||||
messageProducer.setConsumeDelay(Duration.ofMillis(10));
|
||||
messageProducer.setBeanFactory(mock(BeanFactory.class));
|
||||
messageProducer.afterPropertiesSet();
|
||||
messageProducer.start();
|
||||
|
||||
ZMQ.Socket socket = CONTEXT.createSocket(SocketType.PAIR);
|
||||
|
||||
await().until(() -> messageProducer.getBoundPort() > 0);
|
||||
|
||||
socket.connect("tcp://localhost:" + messageProducer.getBoundPort());
|
||||
|
||||
socket.send("test");
|
||||
socket.send("test2");
|
||||
|
||||
stepVerifier.verify();
|
||||
|
||||
messageProducer.destroy();
|
||||
socket.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMessageProducerForPubSubReceiveRaw() throws InterruptedException {
|
||||
String socketAddress = "inproc://messageProducer.test";
|
||||
ZMQ.Socket socket = CONTEXT.createSocket(SocketType.PUB);
|
||||
socket.bind(socketAddress);
|
||||
|
||||
FluxMessageChannel outputChannel = new FluxMessageChannel();
|
||||
|
||||
StepVerifier stepVerifier =
|
||||
StepVerifier.create(outputChannel)
|
||||
.assertNext((message) ->
|
||||
assertThat(message.getPayload())
|
||||
.asInstanceOf(InstanceOfAssertFactories.type(ZMsg.class))
|
||||
.extracting(ZMsg::unwrap)
|
||||
.isEqualTo(new ZFrame("testTopic")))
|
||||
.assertNext((message) ->
|
||||
assertThat(message.getPayload())
|
||||
.asInstanceOf(InstanceOfAssertFactories.type(ZMsg.class))
|
||||
.extracting(ZMsg::unwrap)
|
||||
.isEqualTo(new ZFrame("otherTopic")))
|
||||
.thenCancel()
|
||||
.verifyLater();
|
||||
|
||||
ZeroMqMessageProducer messageProducer = new ZeroMqMessageProducer(CONTEXT, SocketType.SUB);
|
||||
messageProducer.setOutputChannel(outputChannel);
|
||||
messageProducer.setTopics("test");
|
||||
messageProducer.setReceiveRaw(true);
|
||||
messageProducer.setConnectUrl(socketAddress);
|
||||
messageProducer.setConsumeDelay(Duration.ofMillis(10));
|
||||
messageProducer.setBeanFactory(mock(BeanFactory.class));
|
||||
messageProducer.afterPropertiesSet();
|
||||
messageProducer.start();
|
||||
|
||||
// Give it some time to connect and subscribe
|
||||
Thread.sleep(2000);
|
||||
|
||||
ZMsg msg = ZMsg.newStringMsg("test");
|
||||
msg.wrap(new ZFrame("testTopic"));
|
||||
msg.send(socket);
|
||||
|
||||
messageProducer.subscribeToTopics("other");
|
||||
|
||||
// Give it some time to connect and subscribe
|
||||
Thread.sleep(2000);
|
||||
|
||||
msg = ZMsg.newStringMsg("test");
|
||||
msg.wrap(new ZFrame("otherTopic"));
|
||||
msg.send(socket);
|
||||
|
||||
stepVerifier.verify(Duration.ofSeconds(10));
|
||||
|
||||
messageProducer.destroy();
|
||||
socket.close();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright 2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.zeromq.outbound;
|
||||
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.zeromq.SocketType;
|
||||
import org.zeromq.ZContext;
|
||||
import org.zeromq.ZMQ;
|
||||
import org.zeromq.ZMsg;
|
||||
|
||||
import org.springframework.beans.factory.BeanFactory;
|
||||
import org.springframework.integration.expression.FunctionExpression;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.integration.support.json.EmbeddedJsonHeadersMessageMapper;
|
||||
import org.springframework.integration.zeromq.ZeroMqProxy;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.converter.ByteArrayMessageConverter;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
/**
|
||||
* @author Artem Bilan
|
||||
*
|
||||
* @since 5.4
|
||||
*/
|
||||
public class ZeroMqMessageHandlerTests {
|
||||
|
||||
private static final ZContext CONTEXT = new ZContext();
|
||||
|
||||
@AfterAll
|
||||
static void tearDown() {
|
||||
CONTEXT.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMessageHandlerForPair() {
|
||||
String socketAddress = "inproc://messageHandler.test";
|
||||
ZMQ.Socket socket = CONTEXT.createSocket(SocketType.PAIR);
|
||||
socket.bind(socketAddress);
|
||||
|
||||
ZeroMqMessageHandler messageHandler = new ZeroMqMessageHandler(CONTEXT, socketAddress);
|
||||
messageHandler.setBeanFactory(mock(BeanFactory.class));
|
||||
messageHandler.afterPropertiesSet();
|
||||
|
||||
Message<?> testMessage = new GenericMessage<>("test");
|
||||
messageHandler.handleMessage(testMessage).subscribe();
|
||||
|
||||
assertThat(socket.recvStr()).isEqualTo("test");
|
||||
|
||||
messageHandler.handleMessage(new GenericMessage<>(ZMsg.newStringMsg("test2"))).subscribe();
|
||||
|
||||
assertThat(socket.recvStr()).isEqualTo("test2");
|
||||
|
||||
messageHandler.destroy();
|
||||
socket.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMessageHandlerForPubSub() throws InterruptedException {
|
||||
ZMQ.Socket subSocket = CONTEXT.createSocket(SocketType.SUB);
|
||||
subSocket.setReceiveTimeOut(10_000);
|
||||
int port = subSocket.bindToRandomPort("tcp://*");
|
||||
subSocket.subscribe("test");
|
||||
|
||||
ZeroMqMessageHandler messageHandler =
|
||||
new ZeroMqMessageHandler(CONTEXT, "tcp://localhost:" + port, SocketType.PUB);
|
||||
messageHandler.setBeanFactory(mock(BeanFactory.class));
|
||||
messageHandler.setTopicExpression(
|
||||
new FunctionExpression<Message<?>>((message) -> message.getHeaders().get("topic")));
|
||||
messageHandler.setMessageMapper(new EmbeddedJsonHeadersMessageMapper());
|
||||
messageHandler.afterPropertiesSet();
|
||||
|
||||
// Give it some time to bind and subscribe
|
||||
Thread.sleep(2000);
|
||||
|
||||
Message<?> testMessage = MessageBuilder.withPayload("test").setHeader("topic", "testTopic").build();
|
||||
messageHandler.handleMessage(testMessage).subscribe();
|
||||
|
||||
ZMsg msg = ZMsg.recvMsg(subSocket);
|
||||
assertThat(msg).isNotNull();
|
||||
assertThat(msg.unwrap().getString(ZMQ.CHARSET)).isEqualTo("testTopic");
|
||||
Message<?> capturedMessage = new EmbeddedJsonHeadersMessageMapper().toMessage(msg.getFirst().getData());
|
||||
assertThat(capturedMessage).isEqualTo(testMessage);
|
||||
|
||||
msg.destroy();
|
||||
messageHandler.destroy();
|
||||
subSocket.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMessageHandlerForPushPullOverProxy() {
|
||||
ZeroMqProxy proxy = new ZeroMqProxy(CONTEXT);
|
||||
proxy.setBeanName("pullPushProxy");
|
||||
proxy.afterPropertiesSet();
|
||||
proxy.start();
|
||||
|
||||
await().until(() -> proxy.getBackendPort() > 0);
|
||||
|
||||
ZMQ.Socket pullSocket = CONTEXT.createSocket(SocketType.PULL);
|
||||
pullSocket.setReceiveTimeOut(10_000);
|
||||
pullSocket.connect("tcp://localhost:" + proxy.getBackendPort());
|
||||
|
||||
ZeroMqMessageHandler messageHandler =
|
||||
new ZeroMqMessageHandler(CONTEXT, "tcp://localhost:" + proxy.getFrontendPort(), SocketType.PUSH);
|
||||
messageHandler.setBeanFactory(mock(BeanFactory.class));
|
||||
messageHandler.setMessageConverter(new ByteArrayMessageConverter());
|
||||
messageHandler.afterPropertiesSet();
|
||||
|
||||
Message<?> testMessage = new GenericMessage<>("test".getBytes());
|
||||
messageHandler.handleMessage(testMessage).subscribe();
|
||||
|
||||
assertThat(pullSocket.recvStr()).isEqualTo("test");
|
||||
|
||||
messageHandler.destroy();
|
||||
pullSocket.close();
|
||||
proxy.stop();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user