INT-3085 Add a Redis-backed MetadataStore
* Add tests * Add documentation INT-3085 Code review changes INT-3085 Add Twitter Integration Test JIRA: https://jira.springsource.org/browse/INT-3085
This commit is contained in:
committed by
Artem Bilan
parent
bd2cde4202
commit
1fa9b92c0b
@@ -535,6 +535,8 @@ project('spring-integration-twitter') {
|
||||
}
|
||||
compile("javax.activation:activation:$javaxActivationVersion", optional)
|
||||
testCompile project(":spring-integration-test")
|
||||
testCompile project(":spring-integration-redis")
|
||||
testCompile project(":spring-integration-redis").sourceSets.test.output
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.redis.store.metadata;
|
||||
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.core.BoundValueOperations;
|
||||
import org.springframework.data.redis.core.RedisTemplate;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.integration.store.metadata.MetadataStore;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
/**
|
||||
* Redis implementation of {@link MetadataStore}. Use this {@link MetadataStore}
|
||||
* to achieve meta-data persistence across application restarts.
|
||||
*
|
||||
* @author Gunnar Hillert
|
||||
* @since 3.0
|
||||
*/
|
||||
public class RedisMetadataStore implements MetadataStore {
|
||||
|
||||
private final StringRedisTemplate redisTemplate;
|
||||
|
||||
/**
|
||||
* Initializes the {@link RedisTemplate}.
|
||||
* A {@link StringRedisTemplate} is used with default properties.
|
||||
*
|
||||
* @param connectionFactory Must not be null
|
||||
*/
|
||||
public RedisMetadataStore(RedisConnectionFactory connectionFactory) {
|
||||
Assert.notNull(connectionFactory, "'connectionFactory' must not be null.");
|
||||
this.redisTemplate = new StringRedisTemplate(connectionFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists the provided key and value to Redis.
|
||||
*
|
||||
* @param key Must not be null
|
||||
* @param value Must not be null
|
||||
*/
|
||||
public void put(String key, String value) {
|
||||
Assert.notNull(key, "'key' must not be null.");
|
||||
Assert.notNull(value, "'value' must not be null.");
|
||||
BoundValueOperations<String, String> ops = this.redisTemplate.boundValueOps(key);
|
||||
ops.set(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the persisted value for the provided key.
|
||||
*
|
||||
* @param key Must not be null
|
||||
*/
|
||||
public String get(String key) {
|
||||
Assert.notNull(key, "'key' must not be null.");
|
||||
BoundValueOperations<String, String> ops = this.redisTemplate.boundValueOps(key);
|
||||
return ops.get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Provides support for Redis-based
|
||||
* {@link org.springframework.integration.store.metadata.MetadataStore}s.
|
||||
*/
|
||||
package org.springframework.integration.redis.store.metadata;
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.integration.redis.store.metadata;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
|
||||
import org.springframework.data.redis.core.BoundValueOperations;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.integration.redis.rules.RedisAvailable;
|
||||
import org.springframework.integration.redis.rules.RedisAvailableTests;
|
||||
|
||||
/**
|
||||
* @author Gunnar Hillert
|
||||
* @since 3.0
|
||||
*
|
||||
*/
|
||||
public class RedisMetadataStoreTests extends RedisAvailableTests {
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testGetNonExistingKeyValue(){
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
String retrievedValue = metadataStore.get("does-not-exist");
|
||||
assertNull(retrievedValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPersistKeyValue(){
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
metadataStore.put("RedisMetadataStoreTests-Spring", "Integration");
|
||||
|
||||
StringRedisTemplate redisTemplate = new StringRedisTemplate(jcf);
|
||||
BoundValueOperations<String, String> ops = redisTemplate.boundValueOps("RedisMetadataStoreTests-Spring");
|
||||
|
||||
assertEquals("Integration", ops.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testGetValueFromMetadataStore(){
|
||||
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
metadataStore.put("RedisMetadataStoreTests-GetValue", "Hello Redis");
|
||||
|
||||
String retrievedValue = metadataStore.get("RedisMetadataStoreTests-GetValue");
|
||||
assertEquals("Hello Redis", retrievedValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPersistEmptyStringToMetadataStore(){
|
||||
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
metadataStore.put("RedisMetadataStoreTests-PersistEmpty", "");
|
||||
|
||||
String retrievedValue = metadataStore.get("RedisMetadataStoreTests-PersistEmpty");
|
||||
assertEquals("", retrievedValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPersistNullStringToMetadataStore(){
|
||||
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
|
||||
try {
|
||||
metadataStore.put("RedisMetadataStoreTests-PersistEmpty", null);
|
||||
}
|
||||
catch (IllegalArgumentException e) {
|
||||
assertEquals("'value' must not be null.", e.getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
fail("Expected an IllegalArgumentException to be thrown.");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPersistWithEmptyKeyToMetadataStore(){
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
metadataStore.put("", "PersistWithEmptyKey");
|
||||
|
||||
String retrievedValue = metadataStore.get("");
|
||||
assertEquals("PersistWithEmptyKey", retrievedValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPersistWithNullKeyToMetadataStore(){
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
|
||||
try {
|
||||
metadataStore.put(null, "something");
|
||||
}
|
||||
catch (IllegalArgumentException e) {
|
||||
assertEquals("'key' must not be null.", e.getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
fail("Expected an IllegalArgumentException to be thrown.");
|
||||
}
|
||||
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testGetValueWithNullKeyFromMetadataStore(){
|
||||
JedisConnectionFactory jcf = this.getConnectionFactoryForTest();
|
||||
RedisMetadataStore metadataStore = new RedisMetadataStore(jcf);
|
||||
|
||||
try {
|
||||
metadataStore.get(null);
|
||||
}
|
||||
catch (IllegalArgumentException e) {
|
||||
assertEquals("'key' must not be null.", e.getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
fail("Expected an IllegalArgumentException to be thrown.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<beans
|
||||
xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:int="http://www.springframework.org/schema/integration"
|
||||
xmlns:int-twitter="http://www.springframework.org/schema/integration/twitter"
|
||||
xsi:schemaLocation="
|
||||
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
|
||||
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
|
||||
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
|
||||
http://www.springframework.org/schema/integration/twitter http://www.springframework.org/schema/integration/twitter/spring-integration-twitter.xsd">
|
||||
|
||||
<int:channel id="inbound_twitter"/>
|
||||
|
||||
<int-twitter:search-inbound-channel-adapter id="twitterSearchAdapter"
|
||||
query="springintegration"
|
||||
twitter-template="twitterTemplate"
|
||||
channel="inbound_twitter"
|
||||
auto-startup="false">
|
||||
<int:poller fixed-rate="5000" max-messages-per-poll="3"/>
|
||||
</int-twitter:search-inbound-channel-adapter>
|
||||
|
||||
<bean id="metadataStore" class="org.springframework.integration.redis.store.metadata.RedisMetadataStore">
|
||||
<constructor-arg name="connectionFactory" ref="redisConnectionFactory"/>
|
||||
</bean>
|
||||
|
||||
<bean id="redisConnectionFactory" class="org.springframework.data.redis.connection.jedis.JedisConnectionFactory">
|
||||
<property name="port" value="#{T(org.springframework.integration.redis.rules.RedisAvailableRule).REDIS_PORT}"/>
|
||||
</bean>
|
||||
</beans>
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright 2013 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.twitter.inbound;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.ImportResource;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.integration.Message;
|
||||
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
|
||||
import org.springframework.integration.redis.rules.RedisAvailable;
|
||||
import org.springframework.integration.redis.rules.RedisAvailableTests;
|
||||
import org.springframework.integration.redis.store.metadata.RedisMetadataStore;
|
||||
import org.springframework.integration.store.metadata.MetadataStore;
|
||||
import org.springframework.integration.test.util.TestUtils;
|
||||
import org.springframework.social.twitter.api.SearchMetadata;
|
||||
import org.springframework.social.twitter.api.SearchOperations;
|
||||
import org.springframework.social.twitter.api.SearchResults;
|
||||
import org.springframework.social.twitter.api.Tweet;
|
||||
import org.springframework.social.twitter.api.impl.SearchParameters;
|
||||
import org.springframework.social.twitter.api.impl.TwitterTemplate;
|
||||
|
||||
/**
|
||||
* @author Gunnar Hillert
|
||||
* @since 3.0
|
||||
*/
|
||||
public class SearchReceivingMessageSourceWithRedisTests extends RedisAvailableTests {
|
||||
|
||||
private SourcePollingChannelAdapter twitterSearchAdapter;
|
||||
private RedisConnectionFactory redisConnectionFactory;
|
||||
private StringRedisTemplate redisTemplate;
|
||||
|
||||
private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
context.register(SearchReceivingMessageSourceWithRedisTestsConfig.class);
|
||||
context.registerShutdownHook();
|
||||
context.refresh();
|
||||
|
||||
this.redisConnectionFactory = context.getBean(RedisConnectionFactory.class);
|
||||
this.twitterSearchAdapter = context.getBean(SourcePollingChannelAdapter.class);
|
||||
this.redisTemplate = new StringRedisTemplate(redisConnectionFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that a polling operation returns in fact 3 results.
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
@RedisAvailable
|
||||
public void testPollForTweetsThreeResultsWithRedisMetadataStore() throws Exception {
|
||||
|
||||
final MetadataStore metadataStore = TestUtils.getPropertyValue(twitterSearchAdapter, "source.metadataStore", MetadataStore.class);
|
||||
assertTrue("Exptected metadataStore to be an instance of RedisMetadataStore", metadataStore instanceof RedisMetadataStore);
|
||||
|
||||
/*
|
||||
* The metadataKey is automatically generated. To ensure that we use the
|
||||
* the correct key, we retrieve it from the adapter.
|
||||
*/
|
||||
final String metadataKey = TestUtils.getPropertyValue(twitterSearchAdapter, "source.metadataKey", String.class);
|
||||
|
||||
/*
|
||||
* As we had to retrieve the metadataKey from the adapter. The metdataStore
|
||||
* was already invoked and the id retrieved from Redis before we had a chance
|
||||
* to reset possibly pre-existing values.
|
||||
*
|
||||
* Rather than deleting the value, we have to set a value, because "null" values
|
||||
* returned from the MetadataStore are ignored by the onInit() method in
|
||||
* the AbstractTwitterMessageSource. */
|
||||
redisTemplate.opsForValue().set(metadataKey, "-1");
|
||||
assertEquals("-1", redisTemplate.opsForValue().get(metadataKey));
|
||||
|
||||
final SearchReceivingMessageSource source = TestUtils.getPropertyValue(twitterSearchAdapter, "source", SearchReceivingMessageSource.class);
|
||||
|
||||
/* We need to call onInit() in order to update the id from the metadataStore. */
|
||||
source.onInit();
|
||||
|
||||
final Message<?> message1 = source.receive();
|
||||
final Message<?> message2 = source.receive();
|
||||
final Message<?> message3 = source.receive();
|
||||
|
||||
/* We received 3 messages so far. When invoking receive() again the search
|
||||
* will return again the 3 test Tweets but as we already processed them
|
||||
* no message (null) is returned. */
|
||||
final Message<?> message4 = source.receive();
|
||||
|
||||
assertNotNull(message1);
|
||||
assertNotNull(message2);
|
||||
assertNotNull(message3);
|
||||
assertNull(message4);
|
||||
|
||||
final String persistedMetadataStoreValue = redisTemplate.opsForValue().get(metadataKey);
|
||||
assertNotNull(persistedMetadataStoreValue);
|
||||
assertEquals("3", redisTemplate.opsForValue().get(metadataKey));
|
||||
|
||||
redisTemplate.delete(metadataKey);
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@ImportResource("classpath:org/springframework/integration/twitter/inbound/SearchReceivingMessageSourceWithRedisTests-context.xml")
|
||||
static class SearchReceivingMessageSourceWithRedisTestsConfig {
|
||||
|
||||
@Bean(name="twitterTemplate")
|
||||
public TwitterTemplate twitterTemplate() {
|
||||
final TwitterTemplate twitterTemplate = mock(TwitterTemplate.class);
|
||||
|
||||
final SearchOperations so = mock(SearchOperations.class);
|
||||
|
||||
final Tweet tweet3 = new Tweet(3L, "first", new GregorianCalendar(2013, 2, 20).getTime(), "fromUser", "profileImageUrl", 888L, 999L, "languageCode", "source");
|
||||
final Tweet tweet1 = new Tweet(1L, "first", new GregorianCalendar(2013, 0, 20).getTime(), "fromUser", "profileImageUrl", 888L, 999L, "languageCode", "source");
|
||||
final Tweet tweet2 = new Tweet(2L, "first", new GregorianCalendar(2013, 1, 20).getTime(), "fromUser", "profileImageUrl", 888L, 999L, "languageCode", "source");
|
||||
|
||||
final List<Tweet> tweets = new ArrayList<Tweet>();
|
||||
|
||||
tweets.add(tweet3);
|
||||
tweets.add(tweet1);
|
||||
tweets.add(tweet2);
|
||||
|
||||
final SearchResults results = new SearchResults(tweets, new SearchMetadata(111, 111));
|
||||
|
||||
when(twitterTemplate.searchOperations()).thenReturn(so);
|
||||
when(twitterTemplate.searchOperations().search(any(SearchParameters.class))).thenReturn(results);
|
||||
|
||||
return twitterTemplate;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,33 +3,33 @@
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<title>Feed Adapter</title>
|
||||
<para>
|
||||
Spring Integration provides support for Syndication via Feed Adapters
|
||||
Spring Integration provides support for Syndication via Feed Adapters
|
||||
</para>
|
||||
<section id="feed-intro">
|
||||
<title>Introduction</title>
|
||||
<para>
|
||||
Web syndication is a form of publishing material such as news stories, press releases, blog posts, and
|
||||
Web syndication is a form of publishing material such as news stories, press releases, blog posts, and
|
||||
other items typically available on a website but also made available in a feed format such as RSS or ATOM.
|
||||
</para>
|
||||
<para>
|
||||
Spring integration provides support for Web Syndication via its 'feed' adapter and provides convenient
|
||||
namespace-based configuration for it.
|
||||
Spring integration provides support for Web Syndication via its 'feed' adapter and provides convenient
|
||||
namespace-based configuration for it.
|
||||
To configure the 'feed' namespace, include the following elements within the headers of your XML configuration file:
|
||||
|
||||
|
||||
<programlisting language="xml"><![CDATA[xmlns:int-feed="http://www.springframework.org/schema/integration/feed"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/integration/feed
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/integration/feed
|
||||
http://www.springframework.org/schema/integration/feed/spring-integration-feed.xsd"]]></programlisting>
|
||||
|
||||
</para>
|
||||
</section>
|
||||
<section>
|
||||
<section id="feed-inbound-channel-adapter">
|
||||
<title>Feed Inbound Channel Adapter</title>
|
||||
<para>
|
||||
The only adapter that is really needed to provide support for retrieving feeds is an <emphasis>inbound channel adapter</emphasis>.
|
||||
This allows you to subscribe to a particular URL. Below is an example configuration:
|
||||
|
||||
<programlisting language="xml"><![CDATA[<int-feed:inbound-channel-adapter id="feedAdapter"
|
||||
channel="feedChannel"
|
||||
|
||||
<programlisting language="xml"><![CDATA[<int-feed:inbound-channel-adapter id="feedAdapter"
|
||||
channel="feedChannel"
|
||||
url="http://feeds.bbci.co.uk/news/rss.xml">
|
||||
<int:poller fixed-rate="10000" max-messages-per-poll="100" />
|
||||
</int-feed:inbound-channel-adapter>]]></programlisting>
|
||||
@@ -38,46 +38,74 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/feed
|
||||
</para>
|
||||
<para>
|
||||
As news items are retrieved they will be converted to Messages and sent to a channel identified by the <code>channel</code> attribute.
|
||||
The payload of each message will be a <classname>com.sun.syndication.feed.synd.SyndEntry</classname> instance. That encapsulates
|
||||
The payload of each message will be a <classname>com.sun.syndication.feed.synd.SyndEntry</classname> instance. That encapsulates
|
||||
various data about a news item (content, dates, authors, etc.).
|
||||
</para>
|
||||
<para>
|
||||
You can also see that the <emphasis>Inbound Feed Channel Adapter</emphasis> is a Polling Consumer. That means you have to
|
||||
You can also see that the <emphasis>Inbound Feed Channel Adapter</emphasis> is a Polling Consumer. That means you have to
|
||||
provide a poller configuration. However, one important thing you must understand with regard to Feeds is that its inner-workings
|
||||
are slightly different then most other poling consumers. When an Inbound Feed adapter is started, it does the first poll and
|
||||
receives a <classname>com.sun.syndication.feed.synd.SyndEntryFeed</classname> instance. That is an object that contains multiple
|
||||
<classname>SyndEntry</classname> objects. Each entry is stored in the local entry queue and is released based on
|
||||
the value in the <code>max-messages-per-poll</code> attribute such that each Message will contain a single entry.
|
||||
If during retrieval of the entries from the entry queue the queue had become empty, the adapter will attempt to update
|
||||
are slightly different then most other poling consumers. When an Inbound Feed adapter is started, it does the first poll and
|
||||
receives a <classname>com.sun.syndication.feed.synd.SyndEntryFeed</classname> instance. That is an object that contains multiple
|
||||
<classname>SyndEntry</classname> objects. Each entry is stored in the local entry queue and is released based on
|
||||
the value in the <code>max-messages-per-poll</code> attribute such that each Message will contain a single entry.
|
||||
If during retrieval of the entries from the entry queue the queue had become empty, the adapter will attempt to update
|
||||
the Feed thereby populating the queue with more entries (SyndEntry instances) if available. Otherwise the next attempt to
|
||||
poll for a feed will be determined by the trigger of the poller (e.g., every 10 seconds in the above configuration).
|
||||
</para>
|
||||
|
||||
|
||||
<para>
|
||||
<emphasis>Duplicate Entries</emphasis>
|
||||
</para>
|
||||
<para>
|
||||
Polling for a Feed might result in entries that have already been processed
|
||||
("I already read that news item, why are you showing it to me again?").
|
||||
("I already read that news item, why are you showing it to me again?").
|
||||
Spring Integration provides a convenient mechanism to eliminate the need to worry about duplicate entries.
|
||||
Each feed entry will have a <emphasis>published date</emphasis> field. Every time a new Message is generated and sent,
|
||||
Each feed entry will have a <emphasis>published date</emphasis> field. Every time a new Message is generated and sent,
|
||||
Spring Integration will store the value of the latest <emphasis>published date</emphasis> in an instance of the
|
||||
<classname>org.springframework.integration.store.MetadataStore</classname> strategy. The MetadataStore interface is
|
||||
designed to store various types of generic meta-data (e.g., published date of the last feed entry that has been processed)
|
||||
to help components such as this Feed adapter deal with duplicates.
|
||||
to help components such as this Feed adapter deal with duplicates.
|
||||
</para>
|
||||
<para>
|
||||
The default rule for locating this metadata store is as follows: Spring Integration will look for a bean of type
|
||||
<classname>org.springframework.integration.store.MetadataStore</classname> in the ApplicationContext. If one is found then it will be used,
|
||||
otherwise it will create a new instance of <classname>SimpleMetadataStore</classname> which is an in-memory implementation that
|
||||
will only persist metadata within the lifecycle of the currently running Application Context. This means that upon restart you may
|
||||
end up with duplicate entries. If you need to persist metadata between Application Context restarts, you may use the
|
||||
<classname>PropertiesPersistingMetadataStore</classname> which is backed by a properties file and a properties-persister.
|
||||
Alternatively, you could provide your own implementation of the <classname>MetadataStore</classname> interface
|
||||
(e.g. JdbcMetadataStore) and configure it as bean in the Application Context.
|
||||
|
||||
<programlisting language="xml"><![CDATA[<bean id="metadataStore"
|
||||
<para>
|
||||
The default rule for locating this metadata store is as follows:
|
||||
<emphasis>Spring Integration</emphasis> will look for a bean of type
|
||||
<classname>org.springframework.integration.store.MetadataStore</classname> in
|
||||
the ApplicationContext. If one is found then it will be used, otherwise
|
||||
it will create a new instance of <classname>SimpleMetadataStore</classname>
|
||||
which is an in-memory implementation that will only persist metadata within
|
||||
the lifecycle of the currently running Application Context. This means
|
||||
that upon restart you may end up with duplicate entries.
|
||||
</para>
|
||||
<para>
|
||||
If you need to persist metadata between Application Context restarts, two
|
||||
persistent <interfacename>MetadataStores</interfacename> are available:
|
||||
</para>
|
||||
<itemizedlist>
|
||||
<listitem>PropertiesPersistingMetadataStore</listitem>
|
||||
<listitem>RedisMetadataStore</listitem>
|
||||
</itemizedlist>
|
||||
<para>
|
||||
The <classname>PropertiesPersistingMetadataStore</classname> is backed by
|
||||
a properties file and a
|
||||
<interfacename><ulink url="http://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/util/PropertiesPersister.html">PropertiesPersister</ulink></interfacename>.
|
||||
</para>
|
||||
<programlisting language="xml"><![CDATA[<bean id="metadataStore"
|
||||
class="org.springframework.integration.store.PropertiesPersistingMetadataStore"/>]]></programlisting>
|
||||
</para>
|
||||
</section>
|
||||
<para>
|
||||
As of <emphasis>Spring Integration 3.0</emphasis> a Redis-based
|
||||
<interfacename>MetadataStore</interfacename> is also available. For
|
||||
more information regarding the <classname>RedisMetadataStore</classname>
|
||||
see <xref linkend="redis-metadata-store" />.
|
||||
</para>
|
||||
<warning>
|
||||
Be careful when using the same Redis instancce across multiple application
|
||||
contexts as separate Feed adapters may accidentally use the same persisted
|
||||
key.
|
||||
</warning>
|
||||
<para>
|
||||
Alternatively, you could provide your own implementation of the
|
||||
<interfacename>MetadataStore</interfacename> interface (e.g. JdbcMetadataStore)
|
||||
and configure it as bean in the Application Context.
|
||||
</para>
|
||||
</section>
|
||||
</chapter>
|
||||
|
||||
@@ -223,7 +223,36 @@ rt.setConnectionFactory(redisConnectionFactory);]]></programlisting>
|
||||
the <code>valueSerializer</code> property of the <classname>RedisMessageStore</classname>.
|
||||
</para>
|
||||
</section>
|
||||
|
||||
<section id="redis-metadata-store">
|
||||
<title>Redis Metadata Store</title>
|
||||
<para>
|
||||
As of <emphasis>Spring Integration 3.0</emphasis> a new Redis-based
|
||||
<interfacename><ulink url="http://docs.spring.io/spring-integration/docs/latest-ga/api/org/springframework/integration/store/MetadataStore.html">MetadataStore</ulink></interfacename>
|
||||
implementation is available. The <classname>RedisMetadataStore</classname> can
|
||||
be used to maintain state of a <interfacename>MetadataStore</interfacename>
|
||||
across application restarts. This new <interfacename>MetadataStore</interfacename>
|
||||
implementation can be used with adapters such as:
|
||||
</para>
|
||||
<itemizedlist>
|
||||
<listitem>Twitter Inbound Adapters</listitem>
|
||||
<listitem>Feed Inbound Channel Adapter</listitem>
|
||||
</itemizedlist>
|
||||
<para>
|
||||
In order to instruct these adapters to use the new <classname>RedisMetadataStore</classname>
|
||||
simply declare a Spring bean using the bean name <emphasis role="bold">metadataStore</emphasis>.
|
||||
The <emphasis>Twitter Inbound Channel Adapter</emphasis> and the
|
||||
<emphasis>Feed Inbound Channel Adapter</emphasis> will both automatically
|
||||
pick up and use the declared <classname>RedisMetadataStore</classname>.
|
||||
</para>
|
||||
<programlisting language="xml"><![CDATA[<bean name="metadataStore" class="o.s.i.redis.store.metadata.RedisMetadataStore">
|
||||
<constructor-arg name="connectionFactory" ref="redisConnectionFactory"/>
|
||||
</bean>]]></programlisting>
|
||||
<warning>
|
||||
Be careful when using the same Redis instancce across multiple application
|
||||
contexts as separate adapters may accidentally use the same persisted
|
||||
key.
|
||||
</warning>
|
||||
</section>
|
||||
<section id="redis-store-inbound-channel-adapter">
|
||||
<title>RedisStore Inbound Channel Adapter</title>
|
||||
|
||||
|
||||
@@ -151,11 +151,22 @@ twitter.oauth.accessTokenSecret=AbRxUAvyNCtqQtxFK8w5ZMtMj20KFhB6o]]></programlis
|
||||
restarts, you may use the <classname>PropertiesPersistingMetadataStore</classname> (which is backed by a properties file, and a persister
|
||||
strategy), or you may create your own custom implementation of the <classname>MetadataStore</classname> interface (e.g., JdbcMetadatStore)
|
||||
and configure it as a bean named 'metadataStore' within the Application Context.
|
||||
</para>
|
||||
<para>
|
||||
As of <emphasis>Spring Integration 3.0</emphasis> a Redis-based
|
||||
<interfacename>MetadataStore</interfacename> is available. The
|
||||
<classname>RedisMetadataStore</classname> allows you to maintain persisted
|
||||
metadata across Application Context restarts. For more information see <xref linkend="redis-metadata-store" />.
|
||||
</para>
|
||||
<warning>
|
||||
Be careful when using the same Redis instance across multiple application
|
||||
contexts as separate Twitter adapters may accidentally use the same persisted
|
||||
key.
|
||||
</warning>
|
||||
<programlisting language="xml"><![CDATA[<bean id="metadataStore" class="o.s.i.store.PropertiesPersistingMetadataStore"/>
|
||||
]]></programlisting>
|
||||
The Poller that is configured as part of any Inbound Twitter Adapter (see below) will simply poll from this MetadataStore to determine the latest tweet
|
||||
received.
|
||||
</para>
|
||||
<section id="inbound-twitter-update">
|
||||
<title>Inbound Message Channel Adapter</title>
|
||||
<para>
|
||||
|
||||
@@ -132,6 +132,24 @@
|
||||
For more information see <xref linkend="http-namespace"/>.
|
||||
</para>
|
||||
</section>
|
||||
<section id="3.0-redis-meta-data-store">
|
||||
<title>Redis Metadata Store</title>
|
||||
<para>
|
||||
A new Redis-based
|
||||
<interfacename><ulink url="http://docs.spring.io/spring-integration/docs/latest-ga/api/org/springframework/integration/store/MetadataStore.html">MetadataStore</ulink></interfacename>
|
||||
implementation was added. The <classname>RedisMetadataStore</classname> can
|
||||
be used to maintain state of a <interfacename>MetadataStore</interfacename>
|
||||
across application restarts. This new <interfacename>MetadataStore</interfacename>
|
||||
implementation can be used with adapters such as:
|
||||
</para>
|
||||
<itemizedlist>
|
||||
<listitem>Twitter Inbound Adapters</listitem>
|
||||
<listitem>Feed Inbound Channel Adapter</listitem>
|
||||
</itemizedlist>
|
||||
<para>
|
||||
For more information see <xref linkend="redis-metadata-store" />.
|
||||
</para>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
<section id="3.0-general">
|
||||
|
||||
Reference in New Issue
Block a user