INT-4153: Feed Java DSL and other improvements

JIRA: https://jira.spring.io/browse/INT-4153

* Remove deprecated `FeedFetcher` usage
* Introduce `Resource` based ctor for the `FeedEntryMessageSource`
* Add `SyndFeedInput` option and short-hand `preserveWireFeed` for internal `SyndFeedInput` instance
* Reflect the changes in the XSD for Feed
* Change ROME dependency from deprecated `rome-fetcher` to just `rome` as it is recommended by ROME team
* Port Java DSL for Feed module and reflect aforementioned changes in the `Feed` factory and `FeedEntryMessageSourceSpec` as well
* Document changes and mention Feed Java DSL, too
This commit is contained in:
Artem Bilan
2016-11-08 15:39:25 -05:00
committed by Gary Russell
parent 1871b11e85
commit 507764a3d6
18 changed files with 470 additions and 268 deletions

View File

@@ -5,4 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%c{1}: %m%n
log4j.category.org.springframework.integration=WARN
log4j.category.org.springframework.integration.feed=DEBUG
log4j.category.org.springframework.integration.feed=WARN

View File

@@ -7,11 +7,11 @@
http://www.springframework.org/schema/integration/feed http://www.springframework.org/schema/integration/feed/spring-integration-feed.xsd">
<feed:inbound-channel-adapter id="feedAdapter"
channel="feedChannel"
auto-startup="false"
feed-fetcher="fileUrlFeedFetcher"
metadata-store="customMetadataStore"
url="classpath:org/springframework/integration/feed/sample.rss">
channel="feedChannel"
auto-startup="false"
feed-input="syndFeedInput"
metadata-store="metadataStore"
resource="classpath:org/springframework/integration/feed/sample.rss">
<int:poller fixed-rate="10000" max-messages-per-poll="100" />
</feed:inbound-channel-adapter>
@@ -19,8 +19,8 @@
<int:queue/>
</int:channel>
<bean id="fileUrlFeedFetcher" class="org.springframework.integration.feed.inbound.FileUrlFeedFetcher"/>
<bean id="syndFeedInput" class="com.rometools.rome.io.SyndFeedInput"/>
<bean id="customMetadataStore" class="org.springframework.integration.feed.config.FeedInboundChannelAdapterParserTests.SampleMetadataStore"/>
<bean id="metadataStore" class="org.springframework.integration.metadata.SimpleMetadataStore"/>
</beans>

View File

@@ -9,9 +9,9 @@
<int:message-history />
<int-feed:inbound-channel-adapter id="feedAdapterUsage"
channel="feedChannelUsage"
url="classpath:org/springframework/integration/feed/sample.rss"
feed-fetcher="fileUrlFeedFetcher">
channel="feedChannelUsage"
resource="classpath:org/springframework/integration/feed/sample.rss"
preserve-wire-feed="true">
<int:poller fixed-rate="10000" max-messages-per-poll="100"/>
</int-feed:inbound-channel-adapter>
@@ -19,8 +19,9 @@
<bean class="org.springframework.integration.feed.config.FeedInboundChannelAdapterParserTests$SampleService" />
</int:service-activator>
<bean id="fileUrlFeedFetcher" class="org.springframework.integration.feed.inbound.FileUrlFeedFetcher"/>
<bean id="metadataStore" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore"/>
<bean id="metadataStore" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore">
<property name="baseDirectory"
value="#{T (org.springframework.integration.feed.config.FeedInboundChannelAdapterParserTests).tempFolder.root.absolutePath}"/>
</bean>
</beans>

View File

@@ -17,6 +17,7 @@
package org.springframework.integration.feed.config;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
@@ -25,14 +26,14 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.File;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.springframework.context.ConfigurableApplicationContext;
@@ -48,6 +49,7 @@ import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.io.SyndFeedInput;
/**
* @author Oleg Zhurakousky
@@ -55,20 +57,16 @@ import com.rometools.rome.feed.synd.SyndEntry;
* @author Gary Russell
* @author Gunnar Hillert
* @author Artem Bilan
*
* @since 2.0
*/
public class FeedInboundChannelAdapterParserTests {
private static CountDownLatch latch;
@ClassRule
public final static TemporaryFolder tempFolder = new TemporaryFolder();
@Before
public void prepare() {
File persisterFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/",
"feedAdapter.last.entry");
if (persisterFile.exists()) {
persisterFile.delete();
}
}
private static CountDownLatch latch;
@Test
public void validateSuccessfulFileConfigurationWithCustomMetadataStore() {
@@ -76,16 +74,14 @@ public class FeedInboundChannelAdapterParserTests {
"FeedInboundChannelAdapterParserTests-file-context.xml", this.getClass());
SourcePollingChannelAdapter adapter = context.getBean("feedAdapter", SourcePollingChannelAdapter.class);
FeedEntryMessageSource source = (FeedEntryMessageSource) TestUtils.getPropertyValue(adapter, "source");
MetadataStore metadataStore = (MetadataStore) TestUtils.getPropertyValue(source, "metadataStore");
assertTrue(metadataStore instanceof SampleMetadataStore);
assertEquals(metadataStore, context.getBean("customMetadataStore"));
Object fetcher = TestUtils.getPropertyValue(source, "feedFetcher");
assertEquals("FileUrlFeedFetcher", fetcher.getClass().getSimpleName());
assertSame(context.getBean(MetadataStore.class), TestUtils.getPropertyValue(source, "metadataStore"));
SyndFeedInput syndFeedInput = TestUtils.getPropertyValue(source, "syndFeedInput", SyndFeedInput.class);
assertSame(context.getBean(SyndFeedInput.class), syndFeedInput);
assertFalse(syndFeedInput.isPreserveWireFeed());
context.close();
}
@SuppressWarnings("deprecation")
@Test
public void validateSuccessfulHttpConfigurationWithCustomMetadataStore() {
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext(
@@ -93,18 +89,11 @@ public class FeedInboundChannelAdapterParserTests {
SourcePollingChannelAdapter adapter = context.getBean("feedAdapter", SourcePollingChannelAdapter.class);
FeedEntryMessageSource source = (FeedEntryMessageSource) TestUtils.getPropertyValue(adapter, "source");
assertNotNull(TestUtils.getPropertyValue(source, "metadataStore"));
Object fetcher = TestUtils.getPropertyValue(source, "feedFetcher");
assertTrue(fetcher instanceof com.rometools.fetcher.impl.HttpURLFeedFetcher);
context.close();
}
@Test
public void validateSuccessfulNewsRetrievalWithFileUrlAndMessageHistory() throws Exception {
File persisterFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/",
"metadata-store.properties");
if (persisterFile.exists()) {
persisterFile.delete();
}
//Test file samples.rss has 3 news items
latch = spy(new CountDownLatch(3));
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext(
@@ -120,6 +109,10 @@ public class FeedInboundChannelAdapterParserTests {
"FeedInboundChannelAdapterParserTests-file-usage-context.xml", this.getClass());
latch.await(500, TimeUnit.MILLISECONDS);
verify(latch, times(0)).countDown();
SourcePollingChannelAdapter adapter = context.getBean("feedAdapterUsage", SourcePollingChannelAdapter.class);
assertTrue(TestUtils.getPropertyValue(adapter, "source.syndFeedInput.preserveWireFeed", Boolean.class));
context.close();
}
@@ -127,7 +120,7 @@ public class FeedInboundChannelAdapterParserTests {
@Ignore // goes against the real feed
public void validateSuccessfulNewsRetrievalWithHttpUrl() throws Exception {
final CountDownLatch latch = new CountDownLatch(3);
MessageHandler handler = spy((MessageHandler) message -> latch.countDown());
MessageHandler handler = spy(message -> latch.countDown());
ConfigurableApplicationContext context = new ClassPathXmlApplicationContext(
"FeedInboundChannelAdapterParserTests-http-context.xml", this.getClass());
DirectChannel feedChannel = context.getBean("feedChannel", DirectChannel.class);
@@ -176,23 +169,4 @@ public class FeedInboundChannelAdapterParserTests {
}
public static class SampleMetadataStore implements MetadataStore {
@Override
public void put(String key, String value) {
}
@Override
public String get(String key) {
return null;
}
@Override
public String remove(String key) {
return null;
}
}
}

View File

@@ -0,0 +1,118 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.feed.dsl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.FileReader;
import java.util.Properties;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.metadata.MetadataStore;
import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
import org.springframework.messaging.Message;
import org.springframework.messaging.PollableChannel;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit4.SpringRunner;
import com.rometools.rome.feed.synd.SyndEntry;
/**
* @author Artem Bilan
*
* @since 5.0
*/
@RunWith(SpringRunner.class)
@DirtiesContext
public class FeedDslTests {
@ClassRule
public final static TemporaryFolder tempFolder = new TemporaryFolder();
@Autowired
private PollableChannel entries;
@Autowired
private PropertiesPersistingMetadataStore metadataStore;
@Test
@SuppressWarnings("unchecked")
public void testFeedEntryMessageSourceFlow() throws Exception {
Message<SyndEntry> message1 = (Message<SyndEntry>) this.entries.receive(10000);
Message<SyndEntry> message2 = (Message<SyndEntry>) this.entries.receive(10000);
Message<SyndEntry> message3 = (Message<SyndEntry>) this.entries.receive(10000);
long time1 = message1.getPayload().getPublishedDate().getTime();
long time2 = message2.getPayload().getPublishedDate().getTime();
long time3 = message3.getPayload().getPublishedDate().getTime();
assertTrue(time1 < time2);
assertTrue(time2 < time3);
assertNull(this.entries.receive(10));
this.metadataStore.flush();
FileReader metadataStoreFile =
new FileReader(tempFolder.getRoot().getAbsolutePath() + "/metadata-store.properties");
Properties metadataStoreProperties = new Properties();
metadataStoreProperties.load(metadataStoreFile);
assertFalse(metadataStoreProperties.isEmpty());
assertEquals(1, metadataStoreProperties.size());
assertTrue(metadataStoreProperties.containsKey("feedTest"));
}
@Configuration
@EnableIntegration
public static class ContextConfiguration {
@Value("org/springframework/integration/feed/sample.rss")
private Resource feedResource;
@Bean
public MetadataStore metadataStore() {
PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.setBaseDirectory(tempFolder.getRoot().getAbsolutePath());
return metadataStore;
}
@Bean
public IntegrationFlow feedFlow() {
return IntegrationFlows
.from(Feed.inboundAdapter(this.feedResource, "feedTest")
.metadataStore(metadataStore())
.preserveWireFeed(true),
e -> e.poller(p -> p.fixedDelay(100)))
.channel(c -> c.queue("entries"))
.get();
}
}
}

View File

@@ -26,6 +26,7 @@ import java.net.URL;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
@@ -39,13 +40,11 @@ import com.rometools.rome.feed.synd.SyndEntry;
* @author Gary Russell
* @author Aaron Loes
* @author Artem Bilan
*
* @since 2.0
*/
@SuppressWarnings("deprecation")
public class FeedEntryMessageSourceTests {
private final com.rometools.fetcher.FeedFetcher feedFetcher = new FileUrlFeedFetcher();
@Before
public void prepare() {
File metadataStoreFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/",
@@ -65,7 +64,7 @@ public class FeedEntryMessageSourceTests {
@Test
public void testReceiveFeedWithNoEntries() throws Exception {
URL url = new ClassPathResource("org/springframework/integration/feed/empty.rss").getURL();
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo");
feedEntrySource.setBeanName("feedReader");
feedEntrySource.setBeanFactory(mock(BeanFactory.class));
feedEntrySource.afterPropertiesSet();
@@ -74,8 +73,8 @@ public class FeedEntryMessageSourceTests {
@Test
public void testReceiveFeedWithEntriesSorted() throws Exception {
URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL();
FeedEntryMessageSource source = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
ClassPathResource resource = new ClassPathResource("org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource source = new FeedEntryMessageSource(resource, "foo");
source.setComponentName("feedReader");
source.setBeanFactory(mock(BeanFactory.class));
source.afterPropertiesSet();
@@ -94,8 +93,8 @@ public class FeedEntryMessageSourceTests {
// account when determining if the feed entry has been seen before
@Test
public void testEntryHavingBeenUpdatedAfterPublishAndRepeat() throws Exception {
URL url = new ClassPathResource("org/springframework/integration/feed/atom.xml").getURL();
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
ClassPathResource resource = new ClassPathResource("org/springframework/integration/feed/atom.xml");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -114,7 +113,7 @@ public class FeedEntryMessageSourceTests {
metadataStore.afterPropertiesSet();
// now test that what's been read is no longer retrieved
feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -128,8 +127,8 @@ public class FeedEntryMessageSourceTests {
// and no duplicate entries are retrieved
@Test
public void testReceiveFeedWithRealEntriesAndRepeatWithPersistentMetadataStore() throws Exception {
URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL();
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
ClassPathResource resource = new ClassPathResource("org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -154,7 +153,7 @@ public class FeedEntryMessageSourceTests {
metadataStore.afterPropertiesSet();
// now test that what's been read is no longer retrieved
feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
metadataStore = new PropertiesPersistingMetadataStore();
metadataStore.afterPropertiesSet();
@@ -170,8 +169,8 @@ public class FeedEntryMessageSourceTests {
// no persistent MetadataStore is provided and the same entries are retrieved again
@Test
public void testReceiveFeedWithRealEntriesAndRepeatNoPersistentMetadataStore() throws Exception {
URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL();
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
ClassPathResource resource = new ClassPathResource("org/springframework/integration/feed/sample.rss");
FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
feedEntrySource.setBeanFactory(mock(BeanFactory.class));
feedEntrySource.afterPropertiesSet();
@@ -191,7 +190,7 @@ public class FeedEntryMessageSourceTests {
// UNLIKE the previous test
// now test that what's been read is read AGAIN
feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher);
feedEntrySource = new FeedEntryMessageSource(resource, "foo");
feedEntrySource.setBeanName("feedReader");
feedEntrySource.setBeanFactory(mock(BeanFactory.class));
feedEntrySource.afterPropertiesSet();

View File

@@ -1,116 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.feed.inbound;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.zip.GZIPInputStream;
import org.springframework.util.Assert;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;
/**
* @author Oleg Zhurakousky
* @author Mark Fisher
* @author Artem Bilan
* @since 2.0
* @deprecated since 4.3 because 'rome-fetcher-1.6.0' is deprecated.
* Will be revised in 5.0 in favor of ROME 2.0
*
*/
@SuppressWarnings("deprecation")
@Deprecated
class FileUrlFeedFetcher extends com.rometools.fetcher.impl.AbstractFeedFetcher {
@Override
public SyndFeed retrieveFeed(URL feedUrl)
throws IOException, FeedException, com.rometools.fetcher.FetcherException {
Assert.notNull(feedUrl, "feedUrl must not be null");
URLConnection connection = feedUrl.openConnection();
com.rometools.fetcher.impl.SyndFeedInfo syndFeedInfo = new com.rometools.fetcher.impl.SyndFeedInfo();
this.refreshFeedInfo(feedUrl, syndFeedInfo, connection);
return syndFeedInfo.getSyndFeed();
}
@Override
public SyndFeed retrieveFeed(String userAgent, URL url)
throws IllegalArgumentException, IOException, FeedException, com.rometools.fetcher.FetcherException {
return retrieveFeed(url);
}
private void refreshFeedInfo(URL feedUrl, com.rometools.fetcher.impl.SyndFeedInfo syndFeedInfo,
URLConnection connection)
throws IOException, FeedException {
// need to always set the URL because this may have changed due to 3xx redirects
syndFeedInfo.setUrl(connection.getURL());
// the ID is a persistent value that should stay the same
// even if the URL for the feed changes (eg, by 3xx redirects)
syndFeedInfo.setId(feedUrl.toString());
// This will be 0 if the server doesn't support or isn't setting the last modified header
syndFeedInfo.setLastModified(connection.getLastModified());
// get the contents
InputStream inputStream = null;
try {
inputStream = connection.getInputStream();
SyndFeed syndFeed = this.readFeedFromStream(inputStream, connection);
syndFeedInfo.setSyndFeed(syndFeed);
}
finally {
try {
inputStream.close();
}
catch (Exception e) {
// ignore
}
}
}
private SyndFeed readFeedFromStream(InputStream inputStream, URLConnection connection)
throws IOException, FeedException {
BufferedInputStream bufferedInputStream;
if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
// handle gzip encoded content
bufferedInputStream = new BufferedInputStream(new GZIPInputStream(inputStream));
}
else {
bufferedInputStream = new BufferedInputStream(inputStream);
}
XmlReader reader = null;
if (connection.getHeaderField("Content-Type") != null) {
reader = new XmlReader(bufferedInputStream, connection.getHeaderField("Content-Type"), true);
}
else {
reader = new XmlReader(bufferedInputStream, true);
}
SyndFeedInput syndFeedInput = new SyndFeedInput();
syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
SyndFeed feed = syndFeedInput.build(reader);
fireEvent(com.rometools.fetcher.FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, connection, feed);
return feed;
}
}

View File

@@ -22,7 +22,7 @@ Spring Integration adapters
</title>
<link>http://www.springsource.org/extensions/se-sia</link>
<description>
Spring Integration adapters are realy cool
Spring Integration adapters are really cool
</description>
<pubDate>Tue, 23 Apr 2010 12:34:58 EST</pubDate>
</item>