From a0fdf6f0dc20b85a2762ae43be3720d4cd1a4da2 Mon Sep 17 00:00:00 2001 From: Aaron Loes Date: Fri, 17 Apr 2015 19:00:27 +0300 Subject: [PATCH] INT-3700: Fix FeedEntryMS for correct `lastTime` JIRA: https://jira.spring.io/browse/INT-3700 Change to calculate proper syndicate entry date for comparison when adding feeds for processing. I have signed and agree to the terms of the SpringSource Individual Contributor License Agreement. unit test for recent changes to FeedEntryMessageSourceTests to verify proper last seen date calculation. updates for comments from code review * newline at EOF on atom.xml * removed question from code * fetch files via classpath instead of file system * 'else' on newline * authorship on class Polishing --- .../feed/inbound/FeedEntryMessageSource.java | 9 ++-- .../springframework/integration/feed/atom.xml | 45 +++++++++++++++++ ...ChannelAdapterParserTests-file-context.xml | 10 ++-- ...lAdapterParserTests-file-usage-context.xml | 2 +- .../FeedInboundChannelAdapterParserTests.java | 4 +- .../inbound/FeedEntryMessageSourceTests.java | 50 ++++++++++++++++--- 6 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 spring-integration-feed/src/test/java/org/springframework/integration/feed/atom.xml diff --git a/spring-integration-feed/src/main/java/org/springframework/integration/feed/inbound/FeedEntryMessageSource.java b/spring-integration-feed/src/main/java/org/springframework/integration/feed/inbound/FeedEntryMessageSource.java index 98ed1c531b..a5e0ed5e61 100644 --- a/spring-integration-feed/src/main/java/org/springframework/integration/feed/inbound/FeedEntryMessageSource.java +++ b/spring-integration-feed/src/main/java/org/springframework/integration/feed/inbound/FeedEntryMessageSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,7 @@ import com.rometools.rome.feed.synd.SyndFeed; * @author Mario Gray * @author Oleg Zhurakousky * @author Artem Bilan + * @author Aaron Loes * @since 2.0 */ public class FeedEntryMessageSource extends IntegrationObjectSupport implements MessageSource { @@ -165,8 +166,10 @@ public class FeedEntryMessageSource extends IntegrationObjectSupport implements if (next == null) { return null; } - if (next.getPublishedDate() != null) { - this.lastTime = next.getPublishedDate().getTime(); + + Date lastModifiedDate = FeedEntryMessageSource.getLastModifiedDate(next); + if (lastModifiedDate != null) { + this.lastTime = lastModifiedDate.getTime(); } else { this.lastTime += 1;//NOSONAR - single poller thread diff --git a/spring-integration-feed/src/test/java/org/springframework/integration/feed/atom.xml b/spring-integration-feed/src/test/java/org/springframework/integration/feed/atom.xml new file mode 100644 index 0000000000..7f543dddfa --- /dev/null +++ b/spring-integration-feed/src/test/java/org/springframework/integration/feed/atom.xml @@ -0,0 +1,45 @@ + + + + dive into mark + + A <em>lot</em> of effort + went into making this effortless + + 2005-07-31T12:29:29Z + tag:example.org,2003:3 + + + Copyright (c) 2003, Mark Pilgrim + + Example Toolkit + + + Atom draft-07 snapshot + + + tag:example.org,2003:3.2397 + 2005-07-31T12:29:29Z + 2003-12-13T08:29:29-04:00 + + Mark Pilgrim + http://example.org/ + f8dy@example.com + + + Sam Ruby + + + Joe Gregorio + + +
+

+ [Update: The Atom draft is finished.] +

+
+ + + diff --git a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-context.xml b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-context.xml index eff5ef842a..e0ff440427 100644 --- a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-context.xml +++ b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-context.xml @@ -6,12 +6,12 @@ http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd http://www.springframework.org/schema/integration/feed http://www.springframework.org/schema/integration/feed/spring-integration-feed.xsd"> - + url="classpath:org/springframework/integration/feed/sample.rss"> @@ -23,4 +23,4 @@ - \ No newline at end of file + diff --git a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-context.xml b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-context.xml index 05b8997914..a9461b2d49 100644 --- a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-context.xml +++ b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests-file-usage-context.xml @@ -10,7 +10,7 @@ diff --git a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests.java b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests.java index cb65a9f0ea..0282e4fe5e 100644 --- a/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests.java +++ b/spring-integration-feed/src/test/java/org/springframework/integration/feed/config/FeedInboundChannelAdapterParserTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -117,7 +117,7 @@ public class FeedInboundChannelAdapterParserTests { latch = spy(new CountDownLatch(3)); context = new ClassPathXmlApplicationContext( "FeedInboundChannelAdapterParserTests-file-usage-context.xml", this.getClass()); - latch.await(5, TimeUnit.SECONDS); + latch.await(500, TimeUnit.MILLISECONDS); verify(latch, times(0)).countDown(); context.destroy(); } diff --git a/spring-integration-feed/src/test/java/org/springframework/integration/feed/inbound/FeedEntryMessageSourceTests.java b/spring-integration-feed/src/test/java/org/springframework/integration/feed/inbound/FeedEntryMessageSourceTests.java index 870636c649..b278ae385f 100644 --- a/spring-integration-feed/src/test/java/org/springframework/integration/feed/inbound/FeedEntryMessageSourceTests.java +++ b/spring-integration-feed/src/test/java/org/springframework/integration/feed/inbound/FeedEntryMessageSourceTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,8 @@ import java.net.URL; import org.junit.Before; import org.junit.Test; - import org.springframework.beans.factory.BeanFactory; +import org.springframework.core.io.ClassPathResource; import org.springframework.integration.metadata.PropertiesPersistingMetadataStore; import org.springframework.messaging.Message; @@ -38,13 +38,13 @@ import com.rometools.rome.feed.synd.SyndEntry; * @author Oleg Zhurakousky * @author Mark Fisher * @author Gary Russell + * @author Aaron Loes * @since 2.0 */ public class FeedEntryMessageSourceTests { private final FeedFetcher feedFetcher = new FileUrlFeedFetcher(); - @Before public void prepare() { File metadataStoreFile = new File(System.getProperty("java.io.tmpdir") + "/spring-integration/", @@ -56,14 +56,14 @@ public class FeedEntryMessageSourceTests { @Test(expected=IllegalArgumentException.class) public void testFailureWhenNotInitialized() throws Exception { - URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss"); + URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL(); FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo"); feedEntrySource.receive(); } @Test public void testReceiveFeedWithNoEntries() throws Exception { - URL url = new URL("file:src/test/java/org/springframework/integration/feed/empty.rss"); + URL url = new ClassPathResource("org/springframework/integration/feed/empty.rss").getURL(); FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher); feedEntrySource.setBeanName("feedReader"); feedEntrySource.setBeanFactory(mock(BeanFactory.class)); @@ -73,7 +73,7 @@ public class FeedEntryMessageSourceTests { @Test public void testReceiveFeedWithEntriesSorted() throws Exception { - URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss"); + URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL(); FeedEntryMessageSource source = new FeedEntryMessageSource(url, "foo", this.feedFetcher); source.setComponentName("feedReader"); source.setBeanFactory(mock(BeanFactory.class)); @@ -89,11 +89,45 @@ public class FeedEntryMessageSourceTests { assertNull(source.receive()); } + // verifies that when entry has been updated since publish, that is taken into + // account when determining if the feed entry has been seen before + @Test + public void testEntryHavingBeenUpdatedAfterPublishAndRepeat() throws Exception { + URL url = new ClassPathResource("org/springframework/integration/feed/atom.xml").getURL(); + FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher); + feedEntrySource.setBeanName("feedReader"); + PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore(); + metadataStore.afterPropertiesSet(); + feedEntrySource.setMetadataStore(metadataStore); + feedEntrySource.setBeanFactory(mock(BeanFactory.class)); + feedEntrySource.afterPropertiesSet(); + + SyndEntry entry1 = feedEntrySource.receive().getPayload(); + assertNull(feedEntrySource.receive()); // only 1 entries in the test feed + + assertEquals("Atom draft-07 snapshot", entry1.getTitle().trim()); + assertEquals(1071318569000L, entry1.getPublishedDate().getTime()); + assertEquals(1122812969000L, entry1.getUpdatedDate().getTime()); + + metadataStore.destroy(); + metadataStore.afterPropertiesSet(); + + // now test that what's been read is no longer retrieved + feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher); + feedEntrySource.setBeanName("feedReader"); + metadataStore = new PropertiesPersistingMetadataStore(); + metadataStore.afterPropertiesSet(); + feedEntrySource.setMetadataStore(metadataStore); + feedEntrySource.setBeanFactory(mock(BeanFactory.class)); + feedEntrySource.afterPropertiesSet(); + assertNull(feedEntrySource.receive()); + } + // will test that last feed entry is remembered between the sessions // and no duplicate entries are retrieved @Test public void testReceiveFeedWithRealEntriesAndRepeatWithPersistentMetadataStore() throws Exception { - URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss"); + URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL(); FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher); feedEntrySource.setBeanName("feedReader"); PropertiesPersistingMetadataStore metadataStore = new PropertiesPersistingMetadataStore(); @@ -135,7 +169,7 @@ public class FeedEntryMessageSourceTests { // no persistent MetadataStore is provided and the same entries are retrieved again @Test public void testReceiveFeedWithRealEntriesAndRepeatNoPersistentMetadataStore() throws Exception { - URL url = new URL("file:src/test/java/org/springframework/integration/feed/sample.rss"); + URL url = new ClassPathResource("org/springframework/integration/feed/sample.rss").getURL(); FeedEntryMessageSource feedEntrySource = new FeedEntryMessageSource(url, "foo", this.feedFetcher); feedEntrySource.setBeanName("feedReader"); feedEntrySource.setBeanFactory(mock(BeanFactory.class));