Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-3147: (3167,3173,1941) Improve MetadataStore

Browse Source 
Previously, `MetadataStore` couldn't be configured for Twitter Adapters
- only a global one could be used.
The `metadataKey` was generated automatically with a 'difficult' value.

* Register all `MessageSource` for `SourcePollingChannelAdapter`
as beans with id based on adapter id and prefix '.source' (INT-3147)
* Polishing parser to get rid of explicit `MessageSource` beans. (INT-3147)
* Make Feed and Twitter adapters `id` attribute as required -
now it presents a `metadataKey` for `MetadataStore` (INT-3147)
* Add to Twitter adapters a reference attribute for `MetadataStore` (INT-3173)
* Add Twitter adapters `poll-skip-period` attribute (INT-3167)
* Add and implement `MetadataStore#remove` (INT-1941)
* Make `MetadataStore` as `@ManagedResource` (INT-1941)
* Polishing tests

JIRAs:
https://jira.springsource.org/browse/INT-3147
https://jira.springsource.org/browse/INT-3167
https://jira.springsource.org/browse/INT-3173
https://jira.springsource.org/browse/INT-1941

INT-3147: Polishing and fixes

* add domain suffix to `metadataKey`
* change contract of `MetadataStore.remove`
* remove timeout window from `AbstractTwitterMessageSource`
* polishing and fix `SearchReceivingMessageSourceWithRedisTests`

INT-3147: Rebasing and polishing

INT-3147: fix 'metadata' package tangle

INT-3147 Doc Polishing
This commit is contained in:
Artem Bilan
2013-10-03 11:27:36 -04:00
committed by Gary Russell
parent 1fb838dd1a
commit 5be8ef3fd8
40 changed files with 393 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
spring-integration-feed/src/main/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.feed.inbound.FeedEntryMessageSource;
import org.springframework.util.StringUtils;
/**
@@ -31,20 +32,23 @@ import org.springframework.util.StringUtils;
* @author Josh Long
* @author Oleg Zhurakousky
* @author Mark Fisher
* @author Gunnar Hillert
* @author Artem Bilan
* @since 2.0
*/
public class FeedInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@Override
protected BeanMetadataElement parseSource(final Element element, final ParserContext parserContext) {
BeanDefinitionBuilder sourceBuilder = BeanDefinitionBuilder.genericBeanDefinition(
"org.springframework.integration.feed.inbound.FeedEntryMessageSource");
BeanDefinitionBuilder sourceBuilder = BeanDefinitionBuilder.genericBeanDefinition(FeedEntryMessageSource.class);
sourceBuilder.addConstructorArgValue(element.getAttribute("url"));
sourceBuilder.addConstructorArgValue(element.getAttribute(ID_ATTRIBUTE));
String feedFetcherRef = element.getAttribute("feed-fetcher");
if (StringUtils.hasText(feedFetcherRef)) {
sourceBuilder.addConstructorArgReference(feedFetcherRef);
}
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(sourceBuilder, element, "metadata-store");
return sourceBuilder.getBeanDefinition();
}

30
spring-integration-feed/src/main/java/org/springframework/integration/feed/inbound/FeedEntryMessageSource.java
View File

@@ -30,8 +30,8 @@ import org.springframework.integration.MessagingException;
import org.springframework.integration.context.IntegrationContextUtils;
import org.springframework.integration.context.IntegrationObjectSupport;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.store.metadata.MetadataStore;
import org.springframework.integration.store.metadata.SimpleMetadataStore;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.SimpleMetadataStore;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
@@ -52,6 +52,7 @@ import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
* @author Josh Long
* @author Mario Gray
* @author Oleg Zhurakousky
* @author Artem Bilan
* @since 2.0
*/
public class FeedEntryMessageSource extends IntegrationObjectSupport implements MessageSource<SyndEntry> {
@@ -62,7 +63,7 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements
private final Queue<SyndEntry> entries = new ConcurrentLinkedQueue<SyndEntry>();
private volatile String metadataKey;
private final String metadataKey;
private volatile MetadataStore metadataStore;
@@ -82,17 +83,19 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements
* If the feed URL has a protocol other than http*, consider providing a custom implementation of the
* {@link FeedFetcher} via the alternate constructor.
*/
public FeedEntryMessageSource(URL feedUrl) {
this(feedUrl, new HttpURLFeedFetcher(HashMapFeedInfoCache.getInstance()));
public FeedEntryMessageSource(URL feedUrl, String metadataKey) {
this(feedUrl, metadataKey, new HttpURLFeedFetcher(HashMapFeedInfoCache.getInstance()));
}
/**
* Creates a FeedEntryMessageSource that will use the provided FeedFetcher to read from the given feed URL.
*/
public FeedEntryMessageSource(URL feedUrl, FeedFetcher feedFetcher) {
public FeedEntryMessageSource(URL feedUrl, String metadataKey, FeedFetcher feedFetcher) {
Assert.notNull(feedUrl, "feedUrl must not be null");
Assert.notNull(metadataKey, "metadataKey must not be null");
Assert.notNull(feedFetcher, "feedFetcher must not be null");
this.feedUrl = feedUrl;
this.metadataKey = metadataKey + "." + this.feedUrl;
this.feedFetcher = feedFetcher;
}
@@ -130,18 +133,7 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements
this.metadataStore = new SimpleMetadataStore();
}
}
StringBuilder metadataKeyBuilder = new StringBuilder();
if (StringUtils.hasText(this.getComponentType())) {
metadataKeyBuilder.append(this.getComponentType() + ".");
}
if (StringUtils.hasText(this.getComponentName())) {
metadataKeyBuilder.append(this.getComponentName() + ".");
}
else if (logger.isWarnEnabled()) {
logger.warn("FeedEntryMessageSource has no name. MetadataStore key might not be unique.");
}
metadataKeyBuilder.append(this.feedUrl);
this.metadataKey = metadataKeyBuilder.toString();
String lastTimeValue = this.metadataStore.get(this.metadataKey);
if (StringUtils.hasText(lastTimeValue)) {
this.lastTime = Long.parseLong(lastTimeValue);
@@ -237,6 +229,7 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements
}
return (date2 == null) ? 1 : 0;
}
}
@@ -258,6 +251,7 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements
}
}
}
}
}

25
spring-integration-feed/src/main/resources/org/springframework/integration/feed/config/spring-integration-feed-3.0.xsd
View File

@@ -22,7 +22,28 @@
<xsd:sequence>
<xsd:element ref="integration:poller" minOccurs="0" maxOccurs="1" />
</xsd:sequence>
<xsd:attributeGroup ref="integration:channelAdapterAttributes"/>
<xsd:attribute name="id" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
The bean id of this Polling Endpoint; the MessageSource is also registered with this id
plus a suffix '.source'; also used as the
MetaDataStore key with suffix '.' + feedUrl - The URL for an RSS or ATOM feed.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="channel" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.MessageChannel" />
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
Identifies the channel attached to this adapter, to which messages will be sent.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attributeGroup ref="integration:smartLifeCycleAttributeGroup"/>
<xsd:attribute name="url" type="xsd:string" use="required">
<xsd:annotation>
<xsd:documentation>
@@ -54,7 +75,7 @@
</xsd:documentation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="org.springframework.integration.store.MetadataStore" />
<tool:expected-type type="org.springframework.integration.metadata.MetadataStore" />
</tool:annotation>
</xsd:appinfo>
</xsd:annotation>

2
spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-context.xml
View File

@@ -21,6 +21,6 @@
<bean id="fileUrlFeedFetcher" class="org.springframework.integration.feed.inbound.FileUrlFeedFetcher"/>
<bean id="metadataStore" class="org.springframework.integration.store.metadata.PropertiesPersistingMetadataStore"/>
<bean id="metadataStore" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore"/>
</beans>

21
spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-noid-context.xml
View File

@@ -1,21 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-feed="http://www.springframework.org/schema/integration/feed"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/feed http://www.springframework.org/schema/integration/feed/spring-integration-feed.xsd">
<int-feed:inbound-channel-adapter channel="feedChannelUsage"
url="file:src/test/java/org/springframework/integration/feed/sample.rss"
feed-fetcher="fileUrlFeedFetcher">
<int:poller fixed-rate="10000" max-messages-per-poll="100"/>
</int-feed:inbound-channel-adapter>
<int:service-activator id="sampleActivator" input-channel="feedChannelUsage">
<bean class="org.springframework.integration.feed.config.FeedInboundChannelAdapterParserTests$SampleServiceNoHistory" />
</int:service-activator>
<bean id="fileUrlFeedFetcher" class="org.springframework.integration.feed.inbound.FileUrlFeedFetcher"/>
</beans>

31
spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests.java
View File

@@ -33,6 +33,7 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.Message;
@@ -43,7 +44,7 @@ import org.springframework.integration.core.MessageHandler;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.feed.inbound.FeedEntryMessageSource;
import org.springframework.integration.history.MessageHistory;
import org.springframework.integration.store.metadata.MetadataStore;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.test.util.TestUtils;
import com.sun.syndication.feed.synd.SyndEntry;
@@ -83,6 +84,7 @@ public class FeedInboundChannelAdapterParserTests {
context.destroy();
}
public void validateSuccessfulHttpConfigurationWithCustomMetadataStore() {
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext(
"FeedInboundChannelAdapterParserTests-http-context.xml", this.getClass());
@@ -98,7 +100,7 @@ public class FeedInboundChannelAdapterParserTests {
@Test
public void validateSuccessfulNewsRetrievalWithFileUrlAndMessageHistory() throws Exception {
File persisterFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/", "message-store.properties");
File persisterFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/", "metadata-store.properties");
if (persisterFile.exists()) {
persisterFile.delete();
}
@@ -120,26 +122,6 @@ public class FeedInboundChannelAdapterParserTests {
context.destroy();
}
@Test
public void validateSuccessfulNewsRetrievalWithFileUrlNoPersistentIdentifier() throws Exception{
//Test file samples.rss has 3 news items
latch = spy(new CountDownLatch(3));
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext(
"FeedInboundChannelAdapterParserTests-file-usage-noid-context.xml", this.getClass());
latch.await(5, TimeUnit.SECONDS);
verify(latch, times(3)).countDown();
context.destroy();
// since we are not deleting the persister file
// in this iteration no new feeds will be received and the latch will timeout
latch = spy(new CountDownLatch(3));
context = new ClassPathXmlApplicationContext(
"FeedInboundChannelAdapterParserTests-file-usage-noid-context.xml", this.getClass());
latch.await(5, TimeUnit.SECONDS);
verify(latch, times(3)).countDown();
context.destroy();
}
@Test
@Ignore // goes against the real feed
public void validateSuccessfulNewsRetrievalWithHttpUrl() throws Exception{
@@ -204,6 +186,11 @@ public class FeedInboundChannelAdapterParserTests {
public String get(String key) {
return null;
}
@Override
public String remove(String key) {
return null;
}
}
}

16
spring-integration-feed/src/test/java/org/springframework/integration/feed/inbound/FeedEntryMessageSourceTests.java
View File

@@ -26,7 +26,7 @@ import java.net.URL;
import org.junit.Before;
import org.junit.Test;
import org.springframework.integration.Message;
import org.springframework.integration.store.metadata.PropertiesPersistingMetadataStore;
import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.fetcher.FeedFetcher;
@@ -52,14 +52,14 @@ public class FeedEntryMessageSourceTests {
@Test(expected=IllegalArgumentException.class)
public void testFailureWhenNotInitialized() throws Exception {
URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url);
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo");
feedEntrySource.receive();
}
@Test
public void testReceiveFeedWithNoEntries() throws Exception {
URL url = new URL("file:src/test/java/org/springframework/integration/feed/empty.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, this.feedFetcher);
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource.setBeanName("feedReader");
feedEntrySource.afterPropertiesSet();
assertNull(feedEntrySource.receive());
@@ -68,7 +68,7 @@ public class FeedEntryMessageSourceTests {
@Test
public void testReceiveFeedWithEntriesSorted() throws Exception {
URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource source = new FeedEntryMessageSource(url, this.feedFetcher);
FeedEntryMessageSource source = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
source.setComponentName("feedReader");
source.afterPropertiesSet();
Message<SyndEntry> message1 = source.receive();
@@ -87,7 +87,7 @@ public class FeedEntryMessageSourceTests {
@Test
public void testReceiveFeedWithRealEntriesAndRepeatWithPersistentMetadataStore() throws Exception {
URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, this.feedFetcher);
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource.setBeanName("feedReader");
PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -111,7 +111,7 @@ public class FeedEntryMessageSourceTests {
metadataStore.afterPropertiesSet();
// now test that what's been read is no longer retrieved
feedEntrySource = new FeedEntryMessageSource(url, this.feedFetcher);
feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource.setBeanName("feedReader");
metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -127,7 +127,7 @@ public class FeedEntryMessageSourceTests {
@Test
public void testReceiveFeedWithRealEntriesAndRepeatNoPersistentMetadataStore() throws Exception {
URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, this.feedFetcher);
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource.setBeanName("feedReader");
feedEntrySource.afterPropertiesSet();
SyndEntry entry1 = feedEntrySource.receive().getPayload();
@@ -146,7 +146,7 @@ public class FeedEntryMessageSourceTests {
// UNLIKE the previous test
// now test that what's been read is read AGAIN
feedEntrySource = new FeedEntryMessageSource(url, this.feedFetcher);
feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource.setBeanName("feedReader");
feedEntrySource.afterPropertiesSet();
entry1 = feedEntrySource.receive().getPayload();