diff --git a/supplier/file-supplier/pom.xml b/supplier/file-supplier/pom.xml index b1ba628d..a2f89922 100644 --- a/supplier/file-supplier/pom.xml +++ b/supplier/file-supplier/pom.xml @@ -19,36 +19,29 @@ spring-integration-file - org.springframework.boot - spring-boot-starter-integration + org.springframework.cloud.fn + metadata-store-common + ${revision} org.springframework.cloud.fn file-common ${project.version} + - org.springframework.boot - spring-boot-starter-json - true - - - org.springframework.boot - spring-boot-starter-validation - - - org.springframework.boot - spring-boot-configuration-processor - provided - - - org.springframework.boot - spring-boot-starter-test + org.springframework.integration + spring-integration-jdbc test - io.projectreactor - reactor-test + org.springframework.boot + spring-boot-starter-jdbc + test + + + org.hsqldb + hsqldb test diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java index 50b275e5..fe2b483b 100644 --- a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 the original author or authors. + * Copyright 2020-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package org.springframework.cloud.fn.supplier.file; +import java.io.File; import java.util.function.Supplier; import org.reactivestreams.Publisher; @@ -38,6 +39,12 @@ import org.springframework.integration.dsl.IntegrationFlows; import org.springframework.integration.file.FileReadingMessageSource; import org.springframework.integration.file.dsl.FileInboundChannelAdapterSpec; import org.springframework.integration.file.dsl.Files; +import org.springframework.integration.file.filters.ChainFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; +import org.springframework.integration.file.filters.FileSystemPersistentAcceptOnceFileListFilter; +import org.springframework.integration.file.filters.RegexPatternFileListFilter; +import org.springframework.integration.file.filters.SimplePatternFileListFilter; +import org.springframework.integration.metadata.ConcurrentMetadataStore; import org.springframework.messaging.Message; import org.springframework.util.StringUtils; @@ -49,6 +56,8 @@ import org.springframework.util.StringUtils; @EnableConfigurationProperties({FileSupplierProperties.class, FileConsumerProperties.class}) public class FileSupplierConfiguration { + private static final String METADATA_STORE_PREFIX = "local-file-system-metadata-"; + private final FileSupplierProperties fileSupplierProperties; private final FileConsumerProperties fileConsumerProperties; @@ -60,22 +69,32 @@ public class FileSupplierConfiguration { public FileSupplierConfiguration(FileSupplierProperties fileSupplierProperties, FileConsumerProperties fileConsumerProperties) { + this.fileSupplierProperties = fileSupplierProperties; this.fileConsumerProperties = fileConsumerProperties; } @Bean - public FileInboundChannelAdapterSpec fileMessageSource() { - final FileInboundChannelAdapterSpec fileInboundChannelAdapterSpec = - Files.inboundAdapter(this.fileSupplierProperties.getDirectory()); + public ChainFileListFilter filter(ConcurrentMetadataStore metadataStore) { + ChainFileListFilter chainFilter = new ChainFileListFilter<>(); if (StringUtils.hasText(this.fileSupplierProperties.getFilenamePattern())) { - fileInboundChannelAdapterSpec.patternFilter(this.fileSupplierProperties.getFilenamePattern()); + chainFilter.addFilter(new SimplePatternFileListFilter(this.fileSupplierProperties.getFilenamePattern())); } else if (this.fileSupplierProperties.getFilenameRegex() != null) { - fileInboundChannelAdapterSpec.regexFilter(this.fileSupplierProperties.getFilenameRegex().pattern()); + chainFilter.addFilter(new RegexPatternFileListFilter(this.fileSupplierProperties.getFilenameRegex())); } - fileInboundChannelAdapterSpec.preventDuplicates(this.fileSupplierProperties.isPreventDuplicates()); - return fileInboundChannelAdapterSpec; + + if (this.fileSupplierProperties.isPreventDuplicates()) { + chainFilter.addFilter(new FileSystemPersistentAcceptOnceFileListFilter(metadataStore, METADATA_STORE_PREFIX)); + } + + return chainFilter; + } + + @Bean + public FileInboundChannelAdapterSpec fileMessageSource(FileListFilter fileListFilter) { + return Files.inboundAdapter(this.fileSupplierProperties.getDirectory()) + .filter(fileListFilter); } @Bean @@ -86,7 +105,7 @@ public class FileSupplierConfiguration { .subscribeOn(Schedulers.boundedElastic()) .repeatWhenEmpty(it -> it.delayElements(this.fileSupplierProperties.getDelayWhenEmpty())) .repeat() - .doOnSubscribe(s -> this.fileMessageSource.start()); + .doOnRequest(r -> this.fileMessageSource.start()); } @Bean @@ -106,4 +125,5 @@ public class FileSupplierConfiguration { return () -> Flux.from(fileReadingFlow()); } } + } diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java index 234e5fd8..e1068243 100644 --- a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 the original author or authors. + * Copyright 2020-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,16 +19,22 @@ package org.springframework.cloud.fn.supplier.file; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.FileReadingMessageSource; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; import static org.assertj.core.api.Assertions.assertThat; @@ -37,15 +43,21 @@ import static org.assertj.core.api.Assertions.assertThat; * @author Artem Bilan * @author Soby Chacko */ +@TestPropertySource(properties = "metadata.store.type = jdbc") public class DefaultFileSupplierTests extends AbstractFileSupplierTests { @Autowired @Qualifier("fileMessageSource") private FileReadingMessageSource fileMessageSource; + @Autowired + private ConcurrentMetadataStore metadataStore; + + @Autowired + private JdbcTemplate jdbcTemplate; + @Test public void testBasicFlow() throws IOException { - Path firstFile = tempDir.resolve("first.file"); Files.write(firstFile, "first.file".getBytes()); @@ -83,6 +95,16 @@ public class DefaultFileSupplierTests extends AbstractFileSupplierTests { stepVerifier.verify(); assertThat(this.fileMessageSource.isRunning()).isTrue(); + + assertThat(this.metadataStore).isInstanceOf(JdbcMetadataStore.class); + + List metadataStoreContent = + jdbcTemplate.queryForList("SELECT metadata_key FROM int_metadata_store", String.class); + + assertThat(metadataStoreContent).hasSize(2); + assertThat(metadataStoreContent.get(0)).startsWith("local-file-system-metadata-"); + assertThat(metadataStoreContent.get(0)).endsWith("first.file"); + assertThat(metadataStoreContent.get(1)).endsWith("test.file"); } } diff --git a/supplier/s3-supplier/README.adoc b/supplier/s3-supplier/README.adoc index c01ccc61..973a03b4 100644 --- a/supplier/s3-supplier/README.adoc +++ b/supplier/s3-supplier/README.adoc @@ -23,7 +23,7 @@ Once injected, you can use the `get` method of the `Supplier` to invoke it and t All configuration properties are prefixed with `s3.supplier`. There are also properties that need to be used with the prefix `s3.common` and `file.consumer`. -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java[AwsS3upplierProperties], +For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java[AwsS3SupplierProperties], link:../../common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java[FileConsumerProperties], and link:../../common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java[AmazonS3Properties].