commit 6edfb852f97419535d300e8102ea5f4ab190df6d Author: Soby Chacko Date: Mon May 4 17:50:58 2020 -0400 Initial Commit Migrating the existing structure from the following location: https://github.com/spring-cloud-stream-app-starters/stream-applications/tree/restructuring diff --git a/consumer/cassandra-consumer/.gitignore b/consumer/cassandra-consumer/.gitignore new file mode 100644 index 00000000..3a568a50 --- /dev/null +++ b/consumer/cassandra-consumer/.gitignore @@ -0,0 +1,30 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/consumer/cassandra-consumer/.toDelete b/consumer/cassandra-consumer/.toDelete new file mode 100644 index 00000000..e69de29b diff --git a/consumer/cassandra-consumer/pom.xml b/consumer/cassandra-consumer/pom.xml new file mode 100644 index 00000000..117d5875 --- /dev/null +++ b/consumer/cassandra-consumer/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + cassandra-consumer + 1.0.0.BUILD-SNAPSHOT + cassandra-consumer + Cassandra Consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + 0.8.0.BUILD-SNAPSHOT + 4.3.1.0 + + + + + org.springframework.boot + spring-boot-starter-data-cassandra-reactive + + + org.springframework.boot + spring-boot-starter-json + + + org.springframework.integration + spring-integration-cassandra + ${springIntegrationCassandara.version} + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.cassandraunit + cassandra-unit-spring + ${cassandra-unit-spring.version} + test + + + com.addthis.metrics + reporter-config3 + + + + + io.projectreactor + reactor-test + test + + + org.awaitility + awaitility + test + + + + diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java new file mode 100644 index 00000000..f458f4fb --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java @@ -0,0 +1,209 @@ +/* + * Copyright 2015-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.fn.consumer.cassandra.query.InsertQueryColumnNameExtractor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.cassandra.core.InsertOptions; +import org.springframework.data.cassandra.core.ReactiveCassandraOperations; +import org.springframework.data.cassandra.core.UpdateOptions; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.data.cassandra.core.cql.WriteOptions; +import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.integration.transformer.AbstractPayloadTransformer; +import org.springframework.messaging.MessageHandler; +import org.springframework.util.StringUtils; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.util.StdDateFormat; +import org.springframework.cloud.fn.consumer.cassandra.query.ColumnNameExtractor; +import org.springframework.cloud.fn.consumer.cassandra.query.UpdateQueryColumnNameExtractor; +import reactor.core.publisher.Mono; + +/** + * @author Artem Bilan + * @author Thomas Risberg + * @author Ashu Gairola + * @author Akos Ratku + */ +@Configuration +@EnableConfigurationProperties(CassandraConsumerProperties.class) +public class CassandraConsumerConfiguration { + + @Autowired + private CassandraConsumerProperties cassandraSinkProperties; + + @Bean + public IntegrationFlow cassandraConsumerFlow(MessageHandler cassandraSinkMessageHandler, + ObjectMapper objectMapper) { + IntegrationFlowBuilder integrationFlowBuilder = + IntegrationFlows.from(CassandraConsumerFunction.class); + if (StringUtils.hasText(this.cassandraSinkProperties.getIngestQuery())) { + integrationFlowBuilder.transform( + new PayloadToMatrixTransformer(objectMapper, this.cassandraSinkProperties.getIngestQuery(), + CassandraMessageHandler.Type.UPDATE == this.cassandraSinkProperties.getQueryType() + ? new UpdateQueryColumnNameExtractor() + : new InsertQueryColumnNameExtractor())); + } + return integrationFlowBuilder + .handle(cassandraSinkMessageHandler) + .get(); + } + + @Bean + public MessageHandler cassandraSinkMessageHandler(ReactiveCassandraOperations cassandraOperations) { + CassandraMessageHandler cassandraMessageHandler = + this.cassandraSinkProperties.getQueryType() != null + ? new CassandraMessageHandler(cassandraOperations, this.cassandraSinkProperties.getQueryType()) + : new CassandraMessageHandler(cassandraOperations); + cassandraMessageHandler.setProducesReply(true); + cassandraMessageHandler.setAsync(true); + if (this.cassandraSinkProperties.getConsistencyLevel() != null + || this.cassandraSinkProperties.getTtl() > 0) { + + WriteOptions.WriteOptionsBuilder writeOptionsBuilder = WriteOptions.builder(); + + switch (this.cassandraSinkProperties.getQueryType()) { + + case INSERT: + writeOptionsBuilder = InsertOptions.builder(); + break; + case UPDATE: + writeOptionsBuilder = UpdateOptions.builder(); + break; + } + + if (this.cassandraSinkProperties.getConsistencyLevel() != null) { + writeOptionsBuilder.consistencyLevel(this.cassandraSinkProperties.getConsistencyLevel()); + } + + if (this.cassandraSinkProperties.getTtl() > 0) { + writeOptionsBuilder.ttl(this.cassandraSinkProperties.getTtl()); + } + + cassandraMessageHandler.setWriteOptions(writeOptionsBuilder.build()); + } + if (StringUtils.hasText(this.cassandraSinkProperties.getIngestQuery())) { + cassandraMessageHandler.setIngestQuery(this.cassandraSinkProperties.getIngestQuery()); + } + else if (this.cassandraSinkProperties.getStatementExpression() != null) { + cassandraMessageHandler.setStatementExpression(this.cassandraSinkProperties.getStatementExpression()); + } + return cassandraMessageHandler; + } + + private static boolean isUuid(String uuid) { + if (uuid.length() == 36) { + String[] parts = uuid.split("-"); + if (parts.length == 5) { + return (parts[0].length() == 8) && (parts[1].length() == 4) && + (parts[2].length() == 4) && (parts[3].length() == 4) && + (parts[4].length() == 12); + } + } + return false; + } + + + private static class PayloadToMatrixTransformer extends AbstractPayloadTransformer>> { + + private final Jackson2JsonObjectMapper jsonObjectMapper; + + private final List columns = new LinkedList<>(); + + private final ISO8601StdDateFormat dateFormat = new ISO8601StdDateFormat(); + + PayloadToMatrixTransformer(ObjectMapper objectMapper, String query, ColumnNameExtractor columnNameExtractor) { + this.jsonObjectMapper = new Jackson2JsonObjectMapper(objectMapper); + this.columns.addAll(columnNameExtractor.extract(query)); + this.jsonObjectMapper.getObjectMapper() + .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + } + + @Override + @SuppressWarnings("unchecked") + protected List> transformPayload(Object payload) { + if (payload instanceof List) { + return (List>) payload; + } + else { + try { + List> model = this.jsonObjectMapper.fromJson(payload, List.class); + List> data = new ArrayList<>(model.size()); + for (Map entity : model) { + List row = new ArrayList<>(this.columns.size()); + for (String column : this.columns) { + Object value = entity.get(column); + if (value instanceof String) { + String string = (String) value; + if (this.dateFormat.looksLikeISO8601(string)) { + synchronized (this.dateFormat) { + value = new Date(this.dateFormat.parse(string).getTime()).toLocalDate(); + } + } + if (isUuid(string)) { + value = UUID.fromString(string); + } + } + row.add(value); + } + data.add(row); + } + return data; + } + catch (Exception ex) { + throw new IllegalArgumentException("Cannot parse json into matrix", ex); + } + } + } + + } + + /* + * We need this to provide visibility to the protected method. + */ + @SuppressWarnings("serial") + private static class ISO8601StdDateFormat extends StdDateFormat { + + @Override + protected boolean looksLikeISO8601(String dateStr) { + return super.looksLikeISO8601(dateStr); + } + + } + + interface CassandraConsumerFunction extends Function> { + + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java new file mode 100644 index 00000000..1af4094d --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; + +/** + * @author Artem Bilan + * @author Thomas Risberg + */ +@ConfigurationProperties("cassandra") +public class CassandraConsumerProperties { + + /** + * Time-to-live option of WriteOptions. + */ + private int ttl; + + /** + * QueryType for Cassandra Sink. + */ + private CassandraMessageHandler.Type queryType; + + /** + * Ingest Cassandra query. + */ + private String ingestQuery; + + /** + * Expression in Cassandra query DSL style. + */ + private Expression statementExpression; + + /** + * The consistency level for write operation. + */ + private ConsistencyLevel consistencyLevel; + + public int getTtl() { + return this.ttl; + } + + public void setTtl(int ttl) { + this.ttl = ttl; + } + + public CassandraMessageHandler.Type getQueryType() { + return this.queryType; + } + + public void setQueryType(CassandraMessageHandler.Type queryType) { + this.queryType = queryType; + } + + public String getIngestQuery() { + return this.ingestQuery; + } + + public void setIngestQuery(String ingestQuery) { + this.ingestQuery = ingestQuery; + } + + public Expression getStatementExpression() { + return this.statementExpression; + } + + public void setStatementExpression(Expression statementExpression) { + this.statementExpression = statementExpression; + } + + public ConsistencyLevel getConsistencyLevel() { + return this.consistencyLevel; + } + + public void setConsistencyLevel(ConsistencyLevel consistencyLevel) { + this.consistencyLevel = consistencyLevel; + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java new file mode 100644 index 00000000..06e60ab4 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Scanner; + +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.boot.autoconfigure.cassandra.CassandraProperties; +import org.springframework.boot.autoconfigure.cassandra.CqlSessionBuilderCustomizer; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.domain.EntityScanPackages; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.context.EnvironmentAware; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.data.cassandra.config.CqlSessionFactoryBean; +import org.springframework.data.cassandra.core.ReactiveCassandraTemplate; +import org.springframework.data.cassandra.core.cql.CqlTemplate; +import org.springframework.data.cassandra.core.cql.ReactiveCqlOperations; +import org.springframework.data.cassandra.core.cql.generator.CreateKeyspaceCqlGenerator; +import org.springframework.data.cassandra.core.cql.keyspace.CreateKeyspaceSpecification; +import org.springframework.util.StringUtils; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import reactor.core.publisher.Flux; + +/** + * @author Artem Bilan + * @author Thomas Risberg + * @author Rob Hardt + */ +@Configuration +@EnableConfigurationProperties(CassandraClusterProperties.class) +@Import(CassandraAppClusterConfiguration.CassandraPackageRegistrar.class) +public class CassandraAppClusterConfiguration { + + @Bean + public CqlSessionBuilderCustomizer clusterBuilderCustomizer( + CassandraClusterProperties cassandraClusterProperties) { + + PropertyMapper map = PropertyMapper.get(); + return builder -> + map.from(cassandraClusterProperties::isSkipSslValidation) + .whenTrue() + .toCall(() -> { + try { + builder.withSslContext(TrustAllSSLContextFactory.getSslContext()); + } + catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new BeanInitializationException( + "Unable to configure a Cassandra cluster using SSL.", e); + } + + }); + } + + @Bean + @ConditionalOnProperty("cassandra.cluster.create-keyspace") + public Object keyspaceCreator(CassandraProperties cassandraProperties, CqlSessionBuilder cqlSessionBuilder) { + CreateKeyspaceSpecification createKeyspaceSpecification = + CreateKeyspaceSpecification + .createKeyspace(cassandraProperties.getKeyspaceName()) + .withSimpleReplication() + .ifNotExists(); + + String createKeySpaceQuery = new CreateKeyspaceCqlGenerator(createKeyspaceSpecification).toCql(); + CqlSession systemSession = + cqlSessionBuilder.withKeyspace(CqlSessionFactoryBean.CASSANDRA_SYSTEM_SESSION).build(); + + CqlTemplate template = new CqlTemplate(systemSession); + template.execute(createKeySpaceQuery); + + return null; + } + + @Bean + @Lazy + @DependsOn("keyspaceCreator") + public CqlSession cassandraSession(CqlSessionBuilder cqlSessionBuilder) { + return cqlSessionBuilder.build(); + } + + + @Bean + @ConditionalOnProperty("cassandra.cluster.init-script") + public Object keyspaceInitializer(CassandraClusterProperties cassandraClusterProperties, + ReactiveCassandraTemplate reactiveCassandraTemplate) throws IOException { + + String scripts = + new Scanner(cassandraClusterProperties.getInitScript().getInputStream(), + StandardCharsets.UTF_8.name()) + .useDelimiter("\\A") + .next(); + + ReactiveCqlOperations reactiveCqlOperations = + reactiveCassandraTemplate.getReactiveCqlOperations(); + + Flux.fromArray(StringUtils.delimitedListToStringArray(scripts, ";", "\r\n\f")) + .filter(StringUtils::hasText) // an empty String after the last ';' + .flatMap(script -> reactiveCqlOperations.execute(script + ";")) + .blockLast(); + + return null; + + } + + static class CassandraPackageRegistrar implements ImportBeanDefinitionRegistrar, EnvironmentAware { + + private Environment environment; + + @Override + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + @Override + public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, + BeanDefinitionRegistry registry) { + + Binder.get(this.environment) + .bind("cassandra.cluster.entity-base-packages", String[].class) + .map(Arrays::asList) + .ifBound(packagesToScan -> EntityScanPackages.register(registry, packagesToScan)); + } + + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java new file mode 100644 index 00000000..67e7e549 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java @@ -0,0 +1,85 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.core.io.Resource; + +/** + * Common properties for the cassandra modules. + * + * @author Artem Bilan + * @author Thomas Risberg + * @author Rob Hardt + */ +@ConfigurationProperties("cassandra.cluster") +public class CassandraClusterProperties { + + /** + * Flag to create (or not) keyspace on application startup. + */ + private boolean createKeyspace; + + /** + * Resource with CQL scripts (delimited by ';') to initialize keyspace schema. + */ + private Resource initScript; + + /** + * Flag to validate the Servers' SSL certs + */ + private boolean skipSslValidation; + + /** + * Base packages to scan for entities annotated with Table annotations. + */ + private String[] entityBasePackages = { }; + + + public void setCreateKeyspace(boolean createKeyspace) { + this.createKeyspace = createKeyspace; + } + + public void setInitScript(Resource initScript) { + this.initScript = initScript; + } + + public void setSkipSslValidation(boolean skipSslValidation) { + this.skipSslValidation = skipSslValidation; + } + + public boolean isCreateKeyspace() { + return this.createKeyspace; + } + + public Resource getInitScript() { + return this.initScript; + } + + public boolean isSkipSslValidation() { + return this.skipSslValidation; + } + + public String[] getEntityBasePackages() { + return this.entityBasePackages; + } + + public void setEntityBasePackages(String[] entityBasePackages) { + this.entityBasePackages = entityBasePackages; + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java new file mode 100644 index 00000000..97535139 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + + +/** + * Helper to provide an SSL Context that does not validate + * certificates presented in the SSL handshake. + * + * The usual caveats apply. + * + * @author Rob Hardt + * @author Artem Bilan + */ +class TrustAllSSLContextFactory { + + static SSLContext getSslContext() throws NoSuchAlgorithmException, KeyManagementException { + + TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager() { + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + + } + }; + + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new SecureRandom()); + return sc; + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java new file mode 100644 index 00000000..19d0d769 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017-2019 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.List; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +@FunctionalInterface +public interface ColumnNameExtractor { + + List extract(String query); + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java new file mode 100644 index 00000000..967fcd0f --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2017 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +public class InsertQueryColumnNameExtractor implements ColumnNameExtractor { + + private static final Pattern PATTERN = Pattern.compile(".+\\((.+)\\).+(?:values\\s*\\((.+)\\))"); + + @Override + public List extract(String query) { + List extractedColumns = new LinkedList<>(); + Matcher matcher = PATTERN.matcher(query); + if (matcher.matches()) { + String[] columns = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); + String[] params = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); + for (int i = 0; i < columns.length; i++) { + String param = params[i]; + if (param.equals("?")) { + extractedColumns.add(columns[i]); + } + else if (param.startsWith(":")) { + extractedColumns.add(param.substring(1)); + } + } + } + else { + throw new IllegalArgumentException("Invalid CQL insert query syntax: " + query); + } + return extractedColumns; + } + +} diff --git a/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java new file mode 100644 index 00000000..462478b5 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +public class UpdateQueryColumnNameExtractor implements ColumnNameExtractor { + + private static final Pattern PATTERN = Pattern.compile("(?i)(?<=set)(.*)(?=where)where(.*)"); + + @Override + public List extract(String query) { + List extractedColumns = new LinkedList<>(); + Matcher matcher = PATTERN.matcher(query); + if (matcher.find()) { + String[] settings = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); + String[] where = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); + readPairs(extractedColumns, settings); + readPairs(extractedColumns, where); + } + else { + throw new IllegalArgumentException("Invalid CQL update query syntax: " + query); + } + return extractedColumns; + } + + protected void readPairs(List extractedColumns, String[] settings) { + for (String setting : settings) { + String[] columnValuePair = StringUtils.delimitedListToStringArray(setting, "=", " "); + if (columnValuePair[1].startsWith(":") || columnValuePair[1].equals("?")) { + extractedColumns.add(columnValuePair[0]); + } + } + } + +} diff --git a/consumer/cassandra-consumer/src/main/resources/application.properties b/consumer/cassandra-consumer/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/consumer/cassandra-consumer/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java new file mode 100644 index 00000000..df0652bd --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.cassandraunit.spring.CassandraUnitDependencyInjectionIntegrationTestExecutionListener; +import org.cassandraunit.spring.EmbeddedCassandra; +import org.cassandraunit.utils.EmbeddedCassandraServerHelper; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import reactor.core.publisher.Mono; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.cassandra.core.CassandraOperations; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestExecutionListeners; + +/** + * @author Artem Bilan + */ +@TestExecutionListeners(mergeMode = TestExecutionListeners.MergeMode.MERGE_WITH_DEFAULTS, + listeners = CassandraUnitDependencyInjectionIntegrationTestExecutionListener.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.data.cassandra.keyspaceName=" + CassandraConsumerApplicationTests.CASSANDRA_KEYSPACE, + "spring.data.cassandra.localDatacenter=datacenter1", + "cassandra.cluster.createKeyspace=true" }) +@EmbeddedCassandra(configuration = EmbeddedCassandraServerHelper.CASSANDRA_RNDPORT_YML_FILE, timeout = 120000) +@DirtiesContext +abstract class CassandraConsumerApplicationTests { + + static final String CASSANDRA_KEYSPACE = "test"; + + @Autowired + protected CassandraOperations cassandraTemplate; + + @Autowired + protected Function> cassandraConsumer; + + @BeforeAll + static void setUp() { + EmbeddedCassandraServerHelper.getSession(); + System.setProperty("spring.data.cassandra.contactPoints", + EmbeddedCassandraServerHelper.getHost() + ':' + EmbeddedCassandraServerHelper.getNativeTransportPort()); + } + + @AfterAll + static void cleanup() { + System.clearProperty("spring.data.cassandra.contactPoints"); + } + + @AfterEach + void tearDown() { + this.cassandraTemplate.truncate(Book.class); + } + + protected static List getBookList(int numBooks) { + + List books = new ArrayList<>(); + + Book b; + for (int i = 0; i < numBooks; i++) { + b = new Book(); + b.setIsbn(UUID.randomUUID()); + b.setTitle("Spring Cloud Data Flow Guide"); + b.setAuthor("SCDF Guru"); + b.setPages(i * 10 + 5); + b.setInStock(true); + b.setSaleDate(LocalDate.now()); + books.add(b); + } + + return books; + } + + @SpringBootApplication + static class TestApplication {} + +} diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java new file mode 100644 index 00000000..9ea74e77 --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.LocalDate; +import java.util.UUID; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.test.context.TestPropertySource; + +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "spring.data.cassandra.schema-action=RECREATE", + "cassandra.cluster.entity-base-packages=io.pivotal.java.function.cassandra.consumer.domain" }) +class CassandraEntityInsertTests extends CassandraConsumerApplicationTests { + + @Test + @Disabled + void testInsert() { + Book book = new Book(); + book.setIsbn(UUID.randomUUID()); + book.setTitle("Spring Integration Cassandra"); + book.setAuthor("Cassandra Guru"); + book.setPages(521); + book.setSaleDate(LocalDate.now()); + book.setInStock(true); + + Mono result = this.cassandraConsumer.apply(book); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(1)) + .verifyComplete(); + } + +} diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java new file mode 100644 index 00000000..dd3cdabd --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "insert into book (isbn, title, author, pages, saleDate, inStock) values (?, ?, ?, ?, ?, ?)" }) +class CassandraIngestInsertTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + Mono result = + this.cassandraConsumer.apply(mapper.toJson(books)); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java new file mode 100644 index 00000000..844a12cc --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.StringUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "insert into book (isbn, title, author, pages, saleDate, inStock) " + + "values (:myIsbn, :myTitle, :myAuthor, ?, ?, ?)" }) +class CassandraIngestNamedParamsTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + String booksJsonWithNamedParams = mapper.toJson(books); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "isbn", "myIsbn"); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "title", "myTitle"); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "author", "myAuthor"); + + Mono result = + this.cassandraConsumer.apply(booksJsonWithNamedParams); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java new file mode 100644 index 00000000..2f64956c --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "update book set inStock = :inStock, author = :author, pages = :pages, " + + "saleDate = :saleDate, title = :title where isbn = :isbn", + "cassandra.queryType=UPDATE" }) +class CassandraIngestUpdateTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + Mono result = + this.cassandraConsumer.apply(mapper.toJson(books)); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java new file mode 100644 index 00000000..f1d30388 --- /dev/null +++ b/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java @@ -0,0 +1,146 @@ +/* + * Copyright 2015-2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.domain; + +import java.time.LocalDate; +import java.util.UUID; + +import org.springframework.data.cassandra.core.mapping.PrimaryKey; +import org.springframework.data.cassandra.core.mapping.Table; + +/** + * Test POJO + * + * @author David Webb + * @author Artem Bilan + */ +@Table("book") +public class Book { + + @PrimaryKey + private UUID isbn; + + private String title; + + private String author; + + private int pages; + + private LocalDate saleDate; + + private boolean inStock; + + public Book() { + } + + public Book(UUID isbn, String title, String author) { + this.isbn = isbn; + this.title = title; + this.author = author; + } + + /** + * @return Returns the isbn. + */ + public UUID getIsbn() { + return this.isbn; + } + + /** + * @return Returns the saleDate. + */ + public LocalDate getSaleDate() { + return this.saleDate; + } + + /** + * @param saleDate The saleDate to set. + */ + public void setSaleDate(LocalDate saleDate) { + this.saleDate = saleDate; + } + + /** + * @return Returns the inStock. + */ + public boolean isInStock() { + return this.inStock; + } + + /** + * @param inStock The isInStock to set. + */ + public void setInStock(boolean inStock) { + this.inStock = inStock; + } + + /** + * @param isbn The isbn to set. + */ + public void setIsbn(UUID isbn) { + this.isbn = isbn; + } + + /** + * @return Returns the title. + */ + public String getTitle() { + return this.title; + } + + /** + * @param title The title to set. + */ + public void setTitle(String title) { + this.title = title; + } + + /** + * @return Returns the author. + */ + public String getAuthor() { + return this.author; + } + + /** + * @param author The author to set. + */ + public void setAuthor(String author) { + this.author = author; + } + + /** + * @return Returns the pages. + */ + public int getPages() { + return this.pages; + } + + /** + * @param pages The pages to set. + */ + public void setPages(int pages) { + this.pages = pages; + } + + @Override + public String toString() { + return ("isbn -> " + this.isbn) + "\n" + "tile -> " + this.title + "\n" + "author -> " + this.author + + "\n" + "pages -> " + this.pages + "\n"; + } + +} diff --git a/consumer/cassandra-consumer/src/test/resources/init-db.cql b/consumer/cassandra-consumer/src/test/resources/init-db.cql new file mode 100644 index 00000000..6c1397dc --- /dev/null +++ b/consumer/cassandra-consumer/src/test/resources/init-db.cql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS book; + +CREATE TABLE book ( + isbn uuid PRIMARY KEY, + author text, + instock boolean, + pages int, + saledate date, + title text +); diff --git a/consumer/counter-consumer/.gitignore b/consumer/counter-consumer/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/consumer/counter-consumer/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/consumer/counter-consumer/pom.xml b/consumer/counter-consumer/pom.xml new file mode 100644 index 00000000..1030fddf --- /dev/null +++ b/consumer/counter-consumer/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + counter-consumer + 1.0.0.BUILD-SNAPSHOT + counter-consumer + Spring Native Consumer for computing counters + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.fn + payload-converter-function + ${project.version} + + + org.springframework.boot + spring-boot-starter-integration + + + io.micrometer + micrometer-core + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java new file mode 100644 index 00000000..09bd9f0d --- /dev/null +++ b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java @@ -0,0 +1,178 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.convert.converter.Converter; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.messaging.Message; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * + * @author Christian Tzolov + */ +@Configuration +@EnableConfigurationProperties({ CounterConsumerProperties.class }) +public class CounterConsumerConfiguration { + + @Bean + public Function stringToSpelFunction(@Lazy EvaluationContext evaluationContext) { + return new StringToSpelConversionFunction(evaluationContext); + } + + @Bean + @ConfigurationPropertiesBinding + public Converter propertiesSpelConverter(Function stringToSpelFunction) { + return new Converter() { // NOTE Using lambda causes Java Generics issues. + @Override + public Expression convert(String source) { + return stringToSpelFunction.apply(source); + } + }; + } + + @Bean(name = "counterConsumer") + public Consumer> counterConsumer(CounterConsumerProperties properties, MeterRegistry[] meterRegistries, + @Qualifier(IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME) EvaluationContext context) { + + return message -> { + + String counterName = properties.getComputedNameExpression().getValue(context, message, CharSequence.class).toString(); + + // All fixed tags together are passed with every counter increment. + Tags fixedTags = this.toTags(properties.getTag().getFixed()); + + double amount = properties.getComputedAmountExpression().getValue(context, message, double.class); + + Map> allGroupedTags = new HashMap<>(); + // Tag Expressions Counter + if (properties.getTag().getExpression() != null) { + + Map> groupedTags = properties.getTag().getExpression().entrySet().stream() + // maps a pair into [, ... ] Tag array. + .map(namedExpression -> + toList(namedExpression.getValue().getValue(context, message)).stream() + .map(tagValue -> Tag.of(namedExpression.getKey(), tagValue)) + .collect(Collectors.toList())).flatMap(List::stream) + .collect(Collectors.groupingBy(tag -> tag.getKey(), Collectors.toList())); + allGroupedTags.putAll(groupedTags); + } + + this.count(meterRegistries, counterName, fixedTags, allGroupedTags, amount); + }; + } + + /** + * Converts a key/value Map into Tag(key,value) list. Filters out the empty key/value pairs. + * @param keyValueMap key/value map to convert into tags. + * @return Returns Tags list representing every non-empty key/value pair. + */ + protected Tags toTags(Map keyValueMap) { + return CollectionUtils.isEmpty(keyValueMap) ? Tags.empty() : + Tags.of(keyValueMap.entrySet().stream() + .filter(e -> StringUtils.hasText(e.getKey()) && StringUtils.hasText(e.getValue())) + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toList())); + } + + /** + * Converts the input value into an list of values. If the value is not a collection/array type the result + * is a single element list. For collection/array input value the result is the list of stringifie content of + * this collection. + * @param value input value can be array, collection or single value. + * @return Returns value list. + */ + protected List toList(Object value) { + if (value == null) { + return Collections.emptyList(); + } + + if ((value instanceof Collection) || ObjectUtils.isArray(value)) { + Collection valueCollection = (value instanceof Collection) ? (Collection) value + : Arrays.asList(ObjectUtils.toObjectArray(value)); + + return valueCollection.stream() + .filter(v -> v != null) + .map(Object::toString) + .filter(StringUtils::hasText) + .collect(Collectors.toList()); + } + else { + return Arrays.asList(value.toString()); + } + } + + private void count(MeterRegistry[] meterRegistries, String counterName, Tags fixedTags, Map> groupedTags, double amount) { + if (!CollectionUtils.isEmpty(groupedTags)) { + groupedTags.values().stream().map(List::size).max(Integer::compareTo).ifPresent( + max -> { + for (int i = 0; i < max; i++) { + Tags currentTags = Tags.of(fixedTags); + for (Map.Entry> e : groupedTags.entrySet()) { + currentTags = (e.getValue().size() > i) ? + currentTags.and(e.getValue().get(i)) : + currentTags.and(Tags.of(e.getKey(), "")); + } + + // Increment the counterName increment for every configured MaterRegistry. + for (MeterRegistry meterRegistry : meterRegistries) { + meterRegistry.counter(counterName, currentTags).increment(amount); + } + } + } + ); + } + else { + // Increment the counterName increment for every configured MaterRegistry. + for (MeterRegistry meterRegistry : meterRegistries) { + meterRegistry.counter(counterName, fixedTags).increment(amount); + } + } + } + + @Bean + @ConditionalOnMissingBean + public SimpleMeterRegistry simpleMeterRegistry() { + return new SimpleMeterRegistry(); + } +} diff --git a/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java new file mode 100644 index 00000000..a3442fd8 --- /dev/null +++ b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java @@ -0,0 +1,172 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Map; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.validation.annotation.Validated; + +/** + * @author Christian Tzolov + */ +@ConfigurationProperties("counter") +@Validated +public class CounterConsumerProperties { + + /** + * The default name of the increment + */ + @Value("${spring.application.name:counts}") + private String defaultName; + + /** + * The name of the counter to increment. The 'name' and 'nameExpression' are mutually exclusive. + * Only one can be set. + */ + private String name; + + /** + * A SpEL expression (against the incoming Message) to derive the name of the counter to increment. + * The 'name' and 'nameExpression' are mutually exclusive. Only one can be set. + */ + private Expression nameExpression; + + /** + * A SpEL expression (against the incoming Message) to derive the amount to add to the counter. + * If not set the counter is incremented by 1.0 + */ + private Expression amountExpression; + + /** + * Enables counting the number of messages processed. Uses the 'message.' counter name prefix to distinct it + * form the expression based counter. The message counter includes the fixed tags when provided. + */ + private boolean messageCounterEnabled = true; + + /** + * Fixed and computed tags to be assignee with the counter increment measurement. + */ + private MetricsTag tag = new MetricsTag(); + + public static class MetricsTag { + + /** + * Custom tags assigned to every counter increment measurements. + * This is a map so the property convention fixed tags is: counter.tag.fixed.[tag-name]=[tag-value] + */ + private Map fixed; + + /** + * Computes tags from SpEL expression. + * Single SpEL expression can produce an array of values, which in turn means distinct name/value tags. + * Every name/value tag will produce a separate counter increment. + * Tag expression format is: counter.tag.expression.[tag-name]=[SpEL expression] + */ + private Map expression; + + public Map getFixed() { + return fixed; + } + + public void setFixed(Map fixed) { + this.fixed = fixed; + } + + public Map getExpression() { + return expression; + } + + public void setExpression(Map expression) { + this.expression = expression; + } + + @Override + public String toString() { + return "MetricsTag{" + + "fixed=" + fixed + + ", expression=" + expression + + '}'; + } + } + + public MetricsTag getTag() { + return tag; + } + + public String getName() { + if (name == null && nameExpression == null) { + return defaultName; + } + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Expression getNameExpression() { + return nameExpression; + } + + public void setNameExpression(Expression nameExpression) { + this.nameExpression = nameExpression; + } + + public Expression getAmountExpression() { + return amountExpression; + } + + public void setAmountExpression(Expression amountExpression) { + this.amountExpression = amountExpression; + } + + public Expression getComputedAmountExpression() { + return (amountExpression != null ? amountExpression : new LiteralExpression("1.0")); + } + + public Expression getComputedNameExpression() { + return (nameExpression != null ? nameExpression : new LiteralExpression(getName())); + } + + public boolean isMessageCounterEnabled() { + return messageCounterEnabled; + } + + public void setMessageCounterEnabled(boolean messageCounterEnabled) { + this.messageCounterEnabled = messageCounterEnabled; + } + + @AssertTrue(message = "exactly one of 'name' and 'nameExpression' must be set") + public boolean isExclusiveOptions() { + return getName() != null ^ getNameExpression() != null; + } + + @Override + public String toString() { + return "CounterFunctionProperties{" + + "defaultName='" + defaultName + '\'' + + ", name=" + name + + ", tag=" + tag + + '}'; + } +} diff --git a/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java new file mode 100644 index 00000000..a8a5e99f --- /dev/null +++ b/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Function; + +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.ParseException; +import org.springframework.expression.spel.standard.SpelExpression; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +/** + * Converter from String to Spring Expression. + * + * TODO: This could be a top level project. + */ +public class StringToSpelConversionFunction implements Function { + + private final SpelExpressionParser parser; + + private final EvaluationContext evaluationContext; + + public StringToSpelConversionFunction(EvaluationContext evaluationContext) { + this(new SpelExpressionParser(), evaluationContext); + } + + public StringToSpelConversionFunction(SpelExpressionParser parser, EvaluationContext evaluationContext) { + this.evaluationContext = evaluationContext; + this.parser = parser; + } + + @Override + public Expression apply(String source) { + try { + Expression expression = parser.parseExpression(source); + if (expression instanceof SpelExpression) { + ((SpelExpression) expression).setEvaluationContext(evaluationContext); + } + return expression; + } + catch (ParseException e) { + throw new IllegalArgumentException(String.format( + "Could not convert '%s' into a SpEL expression", source), e); + } + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java new file mode 100644 index 00000000..14f21775 --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java @@ -0,0 +1,22 @@ +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Function; + +import org.springframework.core.convert.converter.Converter; + +/** + * @author Christian Tzolov + */ +public class ConverterFunctionAdapter implements Converter { + + private Function function; + + public ConverterFunctionAdapter(Function function) { + this.function = function; + } + + @Override + public T convert(S s) { + return this.function.apply(s); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java new file mode 100644 index 00000000..92ef115c --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.expression.foo='bar'", + "counter.amount-expression=payload.length()" +}) +class CountWithAmountTest extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + String message = "hello world message"; + double messageSize = Long.valueOf(message.length()).doubleValue(); + counterConsumer.accept(new GenericMessage(message)); + assertThat(meterRegistry.find("counter666").counter().count()).isEqualTo(messageSize); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java new file mode 100644 index 00000000..78f4790f --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Consumer; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) +@DirtiesContext +public class CounterConsumerParentTest { + + @Autowired + protected SimpleMeterRegistry meterRegistry; + + @Autowired + protected Consumer> counterConsumer; + + protected Message message(String payload) { + return MessageBuilder.withPayload(payload.getBytes()).build(); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java new file mode 100644 index 00000000..898e67ac --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Collection; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=", + "counter.tag.expression.tag666=#jsonPath(payload,'$..noField')", + "counter.tag.expression.test=#jsonPath(payload,'$..test')", +}) +class EmptyTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + counterConsumer.accept(message("{\"test\": \"Bar\"}")); + + Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); + assertThat(fixedTagsCounters.size()).isEqualTo(0); + + Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); + assertThat(expressionTagsCounters.size()).isEqualTo(0); + + Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); + assertThat(testExpTagsCounters.size()).isEqualTo(1); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java new file mode 100644 index 00000000..99d5c881 --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name-expression=payload" +}) +public class ExpressionCounterNameTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + assertThat(meterRegistry.find("hello").counter().count()).isEqualTo(13.0); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java new file mode 100644 index 00000000..a2debb41 --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; +import java.util.stream.StreamSupport; + +import io.micrometer.core.instrument.Meter; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=bar", + "counter.tag.fixed.gork=bork" +}) +public class FixedTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + Meter counterMeter = meterRegistry.find("counter666").meter(); + assertThat(StreamSupport.stream(counterMeter.measure().spliterator(), false) + .mapToDouble(m -> m.getValue()).sum()).isEqualTo(13.0); + + assertThat(counterMeter.getId().getTags().size()).isEqualTo(2); + assertThat(counterMeter.getId().getTag("foo")).isEqualTo("bar"); + assertThat(counterMeter.getId().getTag("gork")).isEqualTo("bork"); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java new file mode 100644 index 00000000..63a783e8 --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.expression.foo='bar'", + "counter.tag.expression.gork='bork'" +}) +public class LiteralTagExpressionsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + + Counter fooCounter = meterRegistry.find("counter666").tag("foo", "bar").counter(); + assertThat(fooCounter.count()).isEqualTo(13.0); + + Counter gorkCounter = meterRegistry.find("counter666").tag("gork", "bork").counter(); + assertThat(gorkCounter.count()).isEqualTo(13.0); + + assertThat(fooCounter.getId()).isEqualTo(gorkCounter.getId()); + } +} diff --git a/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java new file mode 100644 index 00000000..79c75708 --- /dev/null +++ b/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Collection; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=", + "counter.tag.expression.tag666=#jsonPath(payload,'$..noField')", + "counter.tag.expression.test=#jsonPath(payload,'$..test')", +}) +public class NullTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + counterConsumer.accept(message("{\"test\": null}")); + + Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); + assertThat(fixedTagsCounters.size()).isEqualTo(0); + + Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); + assertThat(expressionTagsCounters.size()).isEqualTo(0); + + Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); + assertThat(testExpTagsCounters.size()).isEqualTo(0); + } +} diff --git a/consumer/file-consumer/pom.xml b/consumer/file-consumer/pom.xml new file mode 100644 index 00000000..9ab3918d --- /dev/null +++ b/consumer/file-consumer/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + file-consumer + 1.0.0.BUILD-SNAPSHOT + file-consumer + file consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-file + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + + diff --git a/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java b/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java new file mode 100644 index 00000000..d00f57f5 --- /dev/null +++ b/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java @@ -0,0 +1,73 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.util.function.Consumer; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.file.DefaultFileNameGenerator; +import org.springframework.integration.file.FileNameGenerator; +import org.springframework.integration.file.FileWritingMessageHandler; +import org.springframework.messaging.Message; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(FileConsumerProperties.class) +public class FileConsumerConfiguration { + + private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); + + private final FileConsumerProperties properties; + + public FileConsumerConfiguration(FileConsumerProperties properties) { + this.properties = properties; + } + + @Bean + public Consumer> fileConsumer() { + return fileWritingMessageHandler()::handleMessage; + } + + @Bean + public FileWritingMessageHandler fileWritingMessageHandler() { + FileWritingMessageHandler handler = (properties.getDirectoryExpression() != null) + ? new FileWritingMessageHandler(EXPRESSION_PARSER.parseExpression(properties.getDirectoryExpression())) + : new FileWritingMessageHandler(properties.getDirectory()); + handler.setAutoCreateDirectory(true); + handler.setAppendNewLine(!properties.isBinary()); + handler.setCharset(properties.getCharset()); + handler.setExpectReply(false); + handler.setFileExistsMode(properties.getMode()); + handler.setFileNameGenerator(fileNameGenerator()); + return handler; + } + + @Bean + public FileNameGenerator fileNameGenerator() { + DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); + fileNameGenerator.setExpression(properties.getNameExpression()); + return fileNameGenerator; + } +} diff --git a/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java b/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java new file mode 100644 index 00000000..eb861a61 --- /dev/null +++ b/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java @@ -0,0 +1,162 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.file.support.FileExistsMode; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; + +/** + * Properties for the file sink. + * + * @author Mark Fisher + * @author Gary Russell + */ +@ConfigurationProperties("file.consumer") +@Validated +public class FileConsumerProperties { + + static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + "file-consumer"; + + private static final String DEFAULT_NAME = "file-consumer"; + + /** + * A flag to indicate whether adding a newline after the write should be suppressed. + */ + private boolean binary = false; + + /** + * The charset to use when writing text content. + */ + private String charset = "UTF-8"; + + /** + * The parent directory of the target file. + */ + private File directory = new File(DEFAULT_DIR); + + /** + * The expression to evaluate for the parent directory of the target file. + */ + private String directoryExpression; + + /** + * The FileExistsMode to use if the target file already exists. + */ + private FileExistsMode mode = FileExistsMode.APPEND; + + /** + * The name of the target file. + */ + private String name = DEFAULT_NAME; + + /** + * The expression to evaluate for the name of the target file. + */ + private String nameExpression; + + /** + * The suffix to append to file name. + */ + private String suffix = ""; + + public boolean isBinary() { + return binary; + } + + public void setBinary(boolean binary) { + this.binary = binary; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public File getDirectory() { + return directory; + } + + public void setDirectory(File directory) { + this.directory = directory; + } + + public String getDirectoryExpression() { + return directoryExpression; + } + + public void setDirectoryExpression(String directoryExpression) { + this.directoryExpression = directoryExpression; + } + + public FileExistsMode getMode() { + return mode; + } + + public void setMode(FileExistsMode mode) { + this.mode = mode; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getNameExpression() { + return (nameExpression != null) + ? nameExpression + " + '" + getSuffix() + "'" + : "'" + name + getSuffix() + "'"; + } + + public void setNameExpression(String nameExpression) { + this.nameExpression = nameExpression; + } + + public String getSuffix() { + String suffixWithDotIfNecessary = ""; + if (StringUtils.hasText(suffix)) { + suffixWithDotIfNecessary = suffix.startsWith(".") ? suffix : "." + suffix; + } + return suffixWithDotIfNecessary; + } + + public void setSuffix(String suffix) { + this.suffix = suffix; + } + + @AssertTrue(message = "Exactly one of 'name' or 'nameExpression' must be set") + public boolean isMutuallyExclusiveNameAndNameExpression() { + return DEFAULT_NAME.equals(name) || nameExpression == null; + } + + @AssertTrue(message = "Exactly one of 'directory' or 'directoryExpression' must be set") + public boolean isMutuallyExclusiveDirectoryAndDirectoryExpression() { + return new File(DEFAULT_DIR).equals(directory) || directoryExpression == null; + } + +} diff --git a/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java new file mode 100644 index 00000000..e886e5ec --- /dev/null +++ b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.nio.file.Path; +import java.util.function.Consumer; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class AbstractFileConsumerTests { + + @TempDir + static Path tempDir; + + @Autowired + Consumer> fileConsumer; + + @BeforeAll + public static void beforeAll() { + System.setProperty("file.consumer.directory", tempDir.toAbsolutePath().toString()); + } + + @AfterAll + public static void afterAll() { + System.clearProperty("file.consumer.directory"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java new file mode 100644 index 00000000..15b4ac02 --- /dev/null +++ b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.binary = true") +public class BinaryFileTests extends AbstractFileConsumerTests { + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer".getBytes()).build()); + File file = new File(tempDir.toFile(), "file-consumer"); + assertThat(file.exists()).isTrue(); + byte[] results = FileCopyUtils.copyToByteArray(file); + assertThat("hello file-consumer".getBytes()).isEqualTo(results); + } + +} diff --git a/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java new file mode 100644 index 00000000..5eeb4cfd --- /dev/null +++ b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; +import java.io.FileReader; +import java.nio.file.Path; +import java.util.function.Consumer; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + * + * We don't need a separate SpringBootApplication for this test as there is already one available in this package. + * {@link AbstractFileConsumerTests}. + */ +@SpringBootTest(properties = {"file.consumer.nameExpression = payload.substring(0, 4)", + "file.consumer.directoryExpression = '${java.io.tmpdir}'+'/'+headers.dir", + "file.consumer.suffix=out"}) +@DirtiesContext +public class ExpressionTests { + + @TempDir + static Path tempDir; + + @Autowired + Consumer> fileConsumer; + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("this is something").setHeader("dir", "expression").build()); + File file = new File(System.getProperty("java.io.tmpdir") + File.separator + "expression", "this.out"); + file.deleteOnExit(); + assertThat(file.exists()).isTrue(); + assertThat("this is something" + System.lineSeparator()) + .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); + } +} diff --git a/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java new file mode 100644 index 00000000..38aa77a0 --- /dev/null +++ b/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; +import java.io.FileReader; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.name = test", "file.consumer.suffix=txt"}) +public class TextFileTests extends AbstractFileConsumerTests { + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer").build()); + File file = new File(tempDir.toFile(), "test.txt"); + assertThat(file.exists()).isTrue(); + assertThat("hello file-consumer" + System.lineSeparator()) + .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); + } + +} diff --git a/consumer/jdbc-consumer/pom.xml b/consumer/jdbc-consumer/pom.xml new file mode 100644 index 00000000..fbd5fd1c --- /dev/null +++ b/consumer/jdbc-consumer/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + jdbc-consumer + 1.0.0.BUILD-SNAPSHOT + jdbc-consumer + jdbc consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-jdbc + + + org.springframework.boot + spring-boot-starter-json + + + org.springframework.boot + spring-boot-starter-jdbc + + + com.h2database + h2 + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.awaitility + awaitility + test + + + junit + junit + + + + + + diff --git a/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java new file mode 100644 index 00000000..90d671c4 --- /dev/null +++ b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.core.io.ByteArrayResource; + +/** + * An in-memory script crafted for dropping-creating the table we're working with. + * All columns are created as VARCHAR(2000). + * + * @author Eric Bottard + * @author Thomas Risberg + */ +public class DefaultInitializationScriptResource extends ByteArrayResource { + + private static final Log logger = LogFactory.getLog(DefaultInitializationScriptResource.class); + + public DefaultInitializationScriptResource(String tableName, Collection columns) { + super(scriptFor(tableName, columns).getBytes(StandardCharsets.UTF_8)); + } + + private static String scriptFor(String tableName, Collection columns) { + StringBuilder result = new StringBuilder("DROP TABLE "); + result.append(tableName).append(";\n\n"); + + result.append("CREATE TABLE ").append(tableName).append('('); + int i = 0; + for (String column : columns) { + if (i++ > 0) { + result.append(", "); + } + result.append(column).append(" VARCHAR(2000)"); + } + result.append(");\n"); + logger.debug(String.format("Generated the following initializing script for table %s:\n%s", tableName, + result.toString())); + return result.toString(); + } + +} diff --git a/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java new file mode 100644 index 00000000..ebe29278 --- /dev/null +++ b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java @@ -0,0 +1,314 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import javax.sql.DataSource; + +import com.fasterxml.jackson.databind.JsonNode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ResourceLoader; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.EvaluationException; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.SpelParseException; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.integration.aggregator.DefaultAggregatingMessageGroupProcessor; +import org.springframework.integration.aggregator.MessageCountReleaseStrategy; +import org.springframework.integration.config.AggregatorFactoryBean; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.integration.expression.ValueExpression; +import org.springframework.integration.jdbc.JdbcMessageHandler; +import org.springframework.integration.jdbc.SqlParameterSourceFactory; +import org.springframework.integration.json.JsonPropertyAccessor; +import org.springframework.integration.store.MessageGroupStore; +import org.springframework.integration.store.SimpleMessageStore; +import org.springframework.integration.support.MutableMessage; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; +import org.springframework.jdbc.datasource.init.DataSourceInitializer; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MimeTypeUtils; +import org.springframework.util.MultiValueMap; + +/** + * + * @author Eric Bottard + * @author Thomas Risberg + * @author Robert St. John + * @author Oliver Flasch + * @author Artem Bilan + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@Configuration +@EnableConfigurationProperties(JdbcConsumerProperties.class) +public class JdbcConsumerConfiguration { + + private static final Log logger = LogFactory.getLog(JdbcConsumerConfiguration.class); + + private static final Object NOT_SET = new Object(); + + private final JdbcConsumerProperties properties; + + private SpelExpressionParser spelExpressionParser = new SpelExpressionParser(); + + private EvaluationContext evaluationContext; + + public JdbcConsumerConfiguration(JdbcConsumerProperties properties, BeanFactory beanFactory) { + this.properties = properties; + this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(beanFactory); + StandardEvaluationContext standardEvaluationContext = (StandardEvaluationContext) this.evaluationContext; + standardEvaluationContext.addPropertyAccessor(new JsonPropertyAccessor()); + } + + @Bean + IntegrationFlow jdbcConsumerFlow(@Qualifier("aggregator") MessageHandler aggregator, + JdbcMessageHandler jdbcMessageHandler) { + + final IntegrationFlowBuilder builder = + IntegrationFlows.from(Consumer.class, gateway -> gateway.beanName("jdbcConsumer")); + if (properties.getBatchSize() > 1 || properties.getIdleTimeout() > 0) { + builder.handle(aggregator); + } + return builder.handle(jdbcMessageHandler).get(); + } + + @Bean + FactoryBean aggregator(MessageGroupStore messageGroupStore) { + AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean(); + aggregatorFactoryBean.setCorrelationStrategy(message -> message.getPayload().getClass().getName()); + aggregatorFactoryBean.setReleaseStrategy(new MessageCountReleaseStrategy(this.properties.getBatchSize())); + if (this.properties.getIdleTimeout() >= 0) { + aggregatorFactoryBean.setGroupTimeoutExpression(new ValueExpression<>(this.properties.getIdleTimeout())); + } + aggregatorFactoryBean.setMessageStore(messageGroupStore); + aggregatorFactoryBean.setProcessorBean(new DefaultAggregatingMessageGroupProcessor()); + aggregatorFactoryBean.setExpireGroupsUponCompletion(true); + aggregatorFactoryBean.setSendPartialResultOnExpiry(true); + return aggregatorFactoryBean; + } + + @Bean + MessageGroupStore messageGroupStore() { + SimpleMessageStore messageGroupStore = new SimpleMessageStore(); + messageGroupStore.setTimeoutOnIdle(true); + messageGroupStore.setCopyOnGet(false); + return messageGroupStore; + } + + @Bean + public JdbcMessageHandler jdbcMessageHandler(DataSource dataSource) { + final MultiValueMap columnExpressionVariations = new LinkedMultiValueMap<>(); + for (Map.Entry entry : this.properties.getColumnsMap().entrySet()) { + String value = entry.getValue(); + columnExpressionVariations.add(entry.getKey(), this.spelExpressionParser.parseExpression(value)); + if (!value.startsWith("payload")) { + String qualified = "payload." + value; + try { + columnExpressionVariations.add(entry.getKey(), + this.spelExpressionParser.parseExpression(qualified)); + } + catch (SpelParseException e) { + logger.info("failed to parse qualified fallback expression " + qualified + + "; be sure your expression uses the 'payload.' prefix where necessary"); + } + } + } + JdbcMessageHandler jdbcMessageHandler = new JdbcMessageHandler(dataSource, + generateSql(this.properties.getTableName(), columnExpressionVariations.keySet())) { + + @Override + protected void handleMessageInternal(final Message message) { + Message convertedMessage = message; + if (message.getPayload() instanceof byte[] || message.getPayload() instanceof Iterable) { + + final String contentType = message.getHeaders().containsKey(MessageHeaders.CONTENT_TYPE) + ? message.getHeaders().get(MessageHeaders.CONTENT_TYPE).toString() + : MimeTypeUtils.APPLICATION_JSON_VALUE; + if (message.getPayload() instanceof Iterable) { + Stream messageStream = + StreamSupport.stream(((Iterable) message.getPayload()).spliterator(), false) + .map(payload -> { + if (payload instanceof byte[]) { + return convertibleContentType(contentType) ? + new String(((byte[]) payload)) : payload; + } + else { + return payload; + } + }); + convertedMessage = new MutableMessage<>(messageStream.collect(Collectors.toList()), + message.getHeaders()); + } + else { + if (convertibleContentType(contentType)) { + convertedMessage = new MutableMessage<>(new String(((byte[]) message.getPayload())), + message.getHeaders()); + } + } + } + super.handleMessageInternal(convertedMessage); + } + }; + SqlParameterSourceFactory parameterSourceFactory = + new ParameterFactory(columnExpressionVariations, this.evaluationContext); + jdbcMessageHandler.setSqlParameterSourceFactory(parameterSourceFactory); + return jdbcMessageHandler; + } + + @ConditionalOnProperty("jdbc.consumer.initialize") + @Bean + public DataSourceInitializer nonBootDataSourceInitializer(DataSource dataSource, ResourceLoader resourceLoader) { + DataSourceInitializer dataSourceInitializer = new DataSourceInitializer(); + dataSourceInitializer.setDataSource(dataSource); + ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); + databasePopulator.setIgnoreFailedDrops(true); + dataSourceInitializer.setDatabasePopulator(databasePopulator); + if ("true".equals(properties.getInitialize())) { + databasePopulator.addScript( + new DefaultInitializationScriptResource(this.properties.getTableName(), + this.properties.getColumnsMap().keySet())); + } + else { + databasePopulator.addScript(resourceLoader.getResource(this.properties.getInitialize())); + } + return dataSourceInitializer; + } + + @Bean + public static ShorthandMapConverter shorthandMapConverter() { + return new ShorthandMapConverter(); + } + + private static boolean convertibleContentType(String contentType) { + return contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple"); + } + + private static String generateSql(String tableName, Set columns) { + StringBuilder builder = new StringBuilder("INSERT INTO "); + StringBuilder questionMarks = new StringBuilder(") VALUES ("); + builder.append(tableName).append("("); + int i = 0; + + for (String column : columns) { + if (i++ > 0) { + builder.append(", "); + questionMarks.append(", "); + } + builder.append(column); + questionMarks.append(':').append(column); + } + builder.append(questionMarks).append(")"); + return builder.toString(); + } + + private static final class ParameterFactory implements SqlParameterSourceFactory { + + private final MultiValueMap columnExpressions; + + private final EvaluationContext context; + + ParameterFactory(MultiValueMap columnExpressions, EvaluationContext context) { + this.columnExpressions = columnExpressions; + this.context = context; + } + + @Override + public SqlParameterSource createParameterSource(Object o) { + if (!(o instanceof Message)) { + throw new IllegalArgumentException("Unable to handle type " + o.getClass().getName()); + } + Message message = (Message) o; + MapSqlParameterSource parameterSource = new MapSqlParameterSource(); + for (Map.Entry> entry : this.columnExpressions.entrySet()) { + String key = entry.getKey(); + List spels = entry.getValue(); + Object value = NOT_SET; + EvaluationException lastException = null; + for (Expression spel : spels) { + try { + value = spel.getValue(context, message); + break; + } + catch (EvaluationException e) { + lastException = e; + } + } + if (value == NOT_SET) { + if (lastException != null) { + logger.info("Could not find value for column '" + key + "': " + lastException.getMessage()); + } + parameterSource.addValue(key, null); + } + else { + if (value instanceof JsonPropertyAccessor.ToStringFriendlyJsonNode) { + // Need to do some reflection until we have a getter for the Node + DirectFieldAccessor dfa = new DirectFieldAccessor(value); + JsonNode node = (JsonNode) dfa.getPropertyValue("node"); + Object valueToUse; + if (node == null || node.isNull()) { + valueToUse = null; + } + else if (node.isNumber()) { + valueToUse = node.numberValue(); + } + else if (node.isBoolean()) { + valueToUse = node.booleanValue(); + } + else { + valueToUse = node.textValue(); + } + parameterSource.addValue(key, valueToUse); + } + else { + parameterSource.addValue(key, value); + } + } + } + return parameterSource; + } + + } + +} diff --git a/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java new file mode 100644 index 00000000..3afae4a1 --- /dev/null +++ b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java @@ -0,0 +1,109 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Eric Bottard + * @author Artem Bilan + * @author Oliver Flasch + */ +@ConfigurationProperties("jdbc.consumer") +public class JdbcConsumerProperties { + + @Autowired + private ShorthandMapConverter shorthandMapConverter; + + /** + * The name of the table to write into. + */ + private String tableName = "messages"; + + /** + * The comma separated colon-based pairs of column names and SpEL expressions for values to insert/update. + * Names are used at initialization time to issue the DDL. + */ + private String columns = "payload:payload.toString()"; + + /** + * 'true', 'false' or the location of a custom initialization script for the table. + */ + private String initialize = "false"; + + /** + * Threshold in number of messages when data will be flushed to database table. + */ + private int batchSize = 1; + + /** + * Idle timeout in milliseconds when data is automatically flushed to database table. + */ + private long idleTimeout = -1L; + + private Map columnsMap; + + public String getTableName() { + return this.tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getColumns() { + return this.columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + + public String getInitialize() { + return this.initialize; + } + + public void setInitialize(String initialize) { + this.initialize = initialize; + } + + public int getBatchSize() { + return this.batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public long getIdleTimeout() { + return this.idleTimeout; + } + + public void setIdleTimeout(long idleTimeout) { + this.idleTimeout = idleTimeout; + } + + Map getColumnsMap() { + if (this.columnsMap == null) { + this.columnsMap = this.shorthandMapConverter.convert(this.columns); + } + return this.columnsMap; + } +} diff --git a/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java new file mode 100644 index 00000000..ec925b85 --- /dev/null +++ b/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.util.Assert; + +/** + * A Converter from String to Map that accepts csv {@literal key:value} pairs + * (similar to what comes out of the box in Spring Core) but also simple + * {@literal key} items, in which case the value is assumed to be equal to the key. + *

+ *

Additionally, commas and colons can be escaped by using a backslash, which is + * useful if said mappings are to be used for SpEL for example.

+ * + * @author Eric Bottard + * @author Artem Bilan + */ +public class ShorthandMapConverter implements Converter> { + + @Override + public Map convert(String source) { + Map result = new LinkedHashMap<>(); + + // Split on comma if not preceded by backslash + String[] mappings = source.split("(? message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + } + Awaitility.await().until(() -> jdbcOperations + .queryForObject("select count(*) from messages", Integer.class), value -> value == numberOfInserts); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java new file mode 100644 index 00000000..f7857f93 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a") +public class DataReceivedAsByteArrayTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertionWhenDataReceivedAsByteArray() { + String hello = "{\"a\": \"hello\"}"; + final Message message = MessageBuilder.withPayload(hello.getBytes()).build(); + jdbcConsumer.accept(message); + final Integer count = + jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class, "hello"); + assertThat(count).isEqualTo(1); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java new file mode 100644 index 00000000..375db87a --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = { "jdbc.consumer.tableName=foobar", + "jdbc.consumer.initialize=classpath:explicit-script.sql", + "jdbc.consumer.columns=a,b" }) +public class ExplicitTableCreationTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = + jdbcOperations.query("select a, b from foobar", new BeanPropertyRowMapper<>(Payload.class)) + .get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java new file mode 100644 index 00000000..c5cc13f8 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a: headers[foo]") +public class HeaderInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testHeaderInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent) + .setHeader("foo", "bar").build(); + jdbcConsumer.accept(message); + assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ?", + Integer.class, "bar")).isEqualTo(1); + } +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java new file mode 100644 index 00000000..3915f29f --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = { + "jdbc.consumer.tableName=no_script", + "jdbc.consumer.initialize=true", + "jdbc.consumer.columns=a,b" }) +public class ImplicitTableCreationTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = jdbcOperations + .query("select a, b from no_script", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java new file mode 100644 index 00000000..e4bff00a --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.function.Consumer; + +import org.junit.jupiter.api.AfterEach; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class JdbcConsumerApplicationTests { + + @Autowired + Consumer> jdbcConsumer; + + @Autowired + JdbcOperations jdbcOperations; + + @Autowired + JdbcTemplate jdbcTemplate; + + @AfterEach + public void cleanup() { + jdbcOperations.execute("DROP TABLE MESSAGES IF EXISTS"); + } + + static class Payload { + + private String a; + + private Integer b; + + public Payload() { + } + + public Payload(String a, Integer b) { + this.a = a; + this.b = b; + } + + public String getA() { + return a; + } + + public Integer getB() { + return b; + } + + public void setA(String a) { + this.a = a; + } + + public void setB(Integer b) { + this.b = b; + } + + @Override + public String toString() { + return a + b; + } + + } + + @SpringBootApplication + static class TestApplication {} + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java new file mode 100644 index 00000000..0ffcc35e --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class JsonStringPayloadInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + String stringA = "{\"a\": \"hello1\", \"b\": 42}"; + String stringB = "{\"a\": \"hello2\", \"b\": null}"; + String stringC = "{\"a\": \"hello3\"}"; + final Message message1 = MessageBuilder.withPayload(stringA).build(); + jdbcConsumer.accept(message1); + final Message message2 = MessageBuilder.withPayload(stringB).build(); + jdbcConsumer.accept(message2); + final Message message3 = MessageBuilder.withPayload(stringC).build(); + jdbcConsumer.accept(message3); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b = ?", + Integer.class, "hello1", 42)).isEqualTo(1); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b IS NULL", + Integer.class, "hello2")).isEqualTo(1); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b IS NULL", + Integer.class, "hello3")).isEqualTo(1); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java new file mode 100644 index 00000000..e5b38384 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class MapPayloadInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + NamedParameterJdbcOperations namedParameterJdbcOperations = new NamedParameterJdbcTemplate(jdbcOperations); + Map mapA = new HashMap<>(); + mapA.put("a", "hello1"); + mapA.put("b", 42); + Map mapB = new HashMap<>(); + mapB.put("a", "hello2"); + mapB.put("b", null); + Map mapC = new HashMap<>(); + mapC.put("a", "hello3"); + final Message> message1 = MessageBuilder.withPayload(mapA).build(); + jdbcConsumer.accept(message1); + final Message> message2 = MessageBuilder.withPayload(mapB).build(); + jdbcConsumer.accept(message2); + final Message> message3 = MessageBuilder.withPayload(mapC).build(); + jdbcConsumer.accept(message3); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b = :b", mapA, Integer.class)).isEqualTo(1); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b IS NULL", mapB, Integer.class)).isEqualTo(1); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b IS NULL", mapC, Integer.class)).isEqualTo(1); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java new file mode 100644 index 00000000..ecf8e287 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.batchSize=1000") +public class SimpleBatchInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testBatchInsertion() { + final int numberOfInserts = 5000; + Payload sent = new Payload("hello", 42); + for (int i = 0; i < numberOfInserts; i++) { + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + } + int result = jdbcOperations.queryForObject("select count(*) from messages", Integer.class); + assertThat(result).isEqualTo(numberOfInserts); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java new file mode 100644 index 00000000..f0826573 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +public class SimpleInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testSimpleInsert() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + String result = jdbcOperations.queryForObject("select payload from messages", String.class); + assertThat(result).isEqualTo(("hello42")); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java new file mode 100644 index 00000000..428a266b --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class SimpleMappingTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java new file mode 100644 index 00000000..9eb8fd9b --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +// annotation below relies on java.util.Properties so backslash needs to be doubled +@TestPropertySource(properties = "jdbc.consumer.columns=a: a.substring(0\\\\, 4), b: b + 624") +public class SpELTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload expected = new Payload("hell", 666); + Payload result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(expected); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java new file mode 100644 index 00000000..9fb69a97 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a: new StringBuilder(payload.a).reverse().toString(), b") +public class UnqualifiableColumnExpressionTests extends JdbcConsumerApplicationTests { + + @Test + public void doesNotFailParsingUnqualifiableExpression() { + // if the app initializes, the test condition passes, but go ahead and apply the column expression anyway + jdbcConsumer.accept(MessageBuilder.withPayload(new Payload("desrever", 123)).build()); + assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?", + Integer.class, "reversed", 123)).isEqualTo(1); + } + +} diff --git a/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java new file mode 100644 index 00000000..bb48f807 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class VaryingInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload a = new Payload("hello", 42); + Payload b = new Payload("world", 12); + Payload c = new Payload("bonjour", null); + Payload d = new Payload(null, 22); + final Message message1 = MessageBuilder.withPayload(a).build(); + jdbcConsumer.accept(message1); + final Message message2 = MessageBuilder.withPayload(b).build(); + jdbcConsumer.accept(message2); + final Message message3 = MessageBuilder.withPayload(c).build(); + jdbcConsumer.accept(message3); + final Message message4 = MessageBuilder.withPayload(d).build(); + jdbcConsumer.accept(message4); + List result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)); + Assertions.assertThat(result).extracting("a").containsExactly("hello", "world", "bonjour", null); + Assertions.assertThat(result).extracting("b").contains(42, 12, 22, null); + } + +} diff --git a/consumer/jdbc-consumer/src/test/resources/explicit-script.sql b/consumer/jdbc-consumer/src/test/resources/explicit-script.sql new file mode 100644 index 00000000..ac07d2d0 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/resources/explicit-script.sql @@ -0,0 +1,6 @@ +-- Used in test for explicit script + +create table foobar( + a varchar(2000), + b VARCHAR (2000) +); diff --git a/consumer/jdbc-consumer/src/test/resources/schema.sql b/consumer/jdbc-consumer/src/test/resources/schema.sql new file mode 100644 index 00000000..65e85283 --- /dev/null +++ b/consumer/jdbc-consumer/src/test/resources/schema.sql @@ -0,0 +1,7 @@ +-- Run by default by Boot infrastructure + +create table messages( + a varchar(2000), + b VARCHAR (2000), + payload VARCHAR (2000) +); diff --git a/consumer/log-consumer/.gitignore b/consumer/log-consumer/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/consumer/log-consumer/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/consumer/log-consumer/pom.xml b/consumer/log-consumer/pom.xml new file mode 100644 index 00000000..384dd2c7 --- /dev/null +++ b/consumer/log-consumer/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + log-consumer + 1.0.0.BUILD-SNAPSHOT + log-consumer + Log Consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + io.pivotal.java.function + payload-converter-function + ${project.version} + + + + org.springframework.boot + spring-boot-starter-integration + + + + org.hibernate.validator + hibernate-validator + true + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.integration + spring-integration-test + test + + + + diff --git a/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java b/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java new file mode 100644 index 00000000..3b17411e --- /dev/null +++ b/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import java.util.function.Consumer; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.messaging.Message; + +/** + * The Configuration class for {@link Consumer} which logs incoming data. + * For the logging logic a Spring Integration {@link org.springframework.integration.handler.LoggingHandler} + * is used. + * If incoming payload is a {@code byte[]} and incoming message {@code contentType} header is text-compatible + * (e.g. {@code application/json}), it is converted into a {@link String}. + * Otherwise the payload is passed to logger as is. + * + * @author Artem Bilan + */ +@Configuration +@EnableConfigurationProperties(LogConsumerProperties.class) +public class LogConsumerConfiguration { + + @Bean + IntegrationFlow logConsumerFlow(LogConsumerProperties logSinkProperties) { + return IntegrationFlows.from(MessageConsumer.class, (gateway) -> gateway.beanName("logConsumer")) + .handle((payload, headers) -> payload) + .log(logSinkProperties.getLevel(), logSinkProperties.getName(), logSinkProperties.getExpression()) + .get(); + } + + private interface MessageConsumer extends Consumer> {} + +} diff --git a/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java b/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java new file mode 100644 index 00000000..ff9b9528 --- /dev/null +++ b/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import static org.springframework.integration.handler.LoggingHandler.Level.INFO; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.handler.LoggingHandler; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the Log Sink app. + * + * @author Gary Russell + * @author Eric Bottard + * @author Chris Schaefer + * @author Artem Bilan + */ +@ConfigurationProperties("log") +@Validated +public class LogConsumerProperties { + + /** + * The name of the logger to use. + */ + @Value("${spring.application.name:log.consumer}") + private String name; + + /** + * A SpEL expression (against the incoming message) to evaluate as the logged message. + */ + private String expression = "payload"; + + /** + * The level at which to log messages. + */ + private LoggingHandler.Level level = INFO; + + @NotBlank + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @NotBlank + public String getExpression() { + return expression; + } + + public void setExpression(String expression) { + this.expression = expression; + } + + @NotNull + public LoggingHandler.Level getLevel() { + return level; + } + + public void setLevel(LoggingHandler.Level level) { + this.level = level; + } + +} diff --git a/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java b/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java new file mode 100644 index 00000000..cf2b837f --- /dev/null +++ b/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import java.util.function.Consumer; + +import org.apache.commons.logging.Log; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.handler.LoggingHandler; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author Artem Bilan + */ +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@SpringBootTest({ "log.name=foo", "log.level=warn", "log.expression=payload.toUpperCase()" }) +class LogConsumerApplicationTests { + + @Autowired + private Consumer> logConsumer; + + @Autowired + @Qualifier("logConsumerFlow.logging-channel-adapter#0") + private LoggingHandler loggingHandler; + + @Test + public void testJsonContentType() { + Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") + .setHeader("contentType", new MimeType("json")) + .build(); + testMessage(message, "{\"foo\":\"bar\"}"); + } + + private void testMessage(Message message, String expectedPayload) { + assertThat(this.loggingHandler.getLevel()).isEqualTo(LoggingHandler.Level.WARN); + Log logger = TestUtils.getPropertyValue(this.loggingHandler, "messageLogger", Log.class); + assertThat(TestUtils.getPropertyValue(logger, "logger.name")).isEqualTo("foo"); + logger = spy(logger); + new DirectFieldAccessor(this.loggingHandler).setPropertyValue("messageLogger", logger); + this.logConsumer.accept(message); + ArgumentCaptor captor = ArgumentCaptor.forClass(Object.class); + verify(logger).warn(captor.capture()); + assertThat(captor.getValue()).isEqualTo(expectedPayload.toUpperCase()); + this.loggingHandler.setLogExpressionString("#this"); + this.logConsumer.accept(message); + verify(logger, times(2)).warn(captor.capture()); + + Message captorMessage = (Message) captor.getAllValues().get(2); + assertThat(captorMessage.getPayload()).isEqualTo(expectedPayload); + + MessageHeaders messageHeaders = captorMessage.getHeaders(); + assertThat(messageHeaders).hasSize(3); + + assertThat(messageHeaders) + .containsEntry(MessageHeaders.CONTENT_TYPE, message.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/consumer/mongodb-consumer/.gitignore b/consumer/mongodb-consumer/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/consumer/mongodb-consumer/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/consumer/mongodb-consumer/pom.xml b/consumer/mongodb-consumer/pom.xml new file mode 100644 index 00000000..c4cca0ed --- /dev/null +++ b/consumer/mongodb-consumer/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + mongodb-consumer + 1.0.0.BUILD-SNAPSHOT + mongodb-consumer + Mongo DB consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-mongodb + + + org.mongodb + mongodb-driver-reactivestreams + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + io.projectreactor + reactor-test + test + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java b/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java new file mode 100644 index 00000000..e2315858 --- /dev/null +++ b/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import java.util.function.Function; + +import reactor.core.publisher.Mono; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.ReactiveMongoTemplate; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.integration.mongodb.outbound.ReactiveMongoDbStoringMessageHandler; +import org.springframework.messaging.Message; +import org.springframework.messaging.ReactiveMessageHandler; + +/** + * A configuration for MongoDB Consumer function. Uses a + * {@link ReactiveMongoDbStoringMessageHandler} to save payload contents to Mongo DB. + * + * @author Artem Bilan + * @author David Turanski + * + */ +@Configuration +@EnableConfigurationProperties({ MongoDbConsumerProperties.class }) +public class MongoDbConsumerConfiguration { + + private final MongoDbConsumerProperties properties; + + private final ReactiveMongoTemplate mongoTemplate; + + public MongoDbConsumerConfiguration(MongoDbConsumerProperties properties, ReactiveMongoTemplate mongoTemplate) { + this.properties = properties; + this.mongoTemplate = mongoTemplate; + } + + @Bean + public Function, Mono> mongodbConsumer(ReactiveMessageHandler mongoConsumerMessageHandler) { + return mongoConsumerMessageHandler::handleMessage; + } + + @Bean + public ReactiveMessageHandler mongoConsumerMessageHandler() { + ReactiveMongoDbStoringMessageHandler mongoDbMessageHandler = new ReactiveMongoDbStoringMessageHandler( + this.mongoTemplate); + Expression collectionExpression = this.properties.getCollectionExpression(); + if (collectionExpression == null) { + collectionExpression = new LiteralExpression(this.properties.getCollection()); + } + mongoDbMessageHandler.setCollectionNameExpression(collectionExpression); + return mongoDbMessageHandler; + } +} diff --git a/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java b/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java new file mode 100644 index 00000000..838cbf4c --- /dev/null +++ b/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; +/** + * @author Artem Bilan + * @author David Turanski + * + */ +@ConfigurationProperties("mongodb.consumer") +@Validated +public class MongoDbConsumerProperties { + + /** + * The MongoDB collection to store data + */ + private String collection; + + /** + * The SpEL expression to evaluate MongoDB collection + */ + private Expression collectionExpression; + + public void setCollection(String collection) { + this.collection = collection; + } + + public String getCollection() { + return this.collection; + } + + public void setCollectionExpression(Expression collectionExpression) { + this.collectionExpression = collectionExpression; + } + + public Expression getCollectionExpression() { + return collectionExpression; + } + + @AssertTrue(message = "One of 'collection' or 'collectionExpression' is required") + private boolean isValid() { + return StringUtils.hasText(this.collection) || this.collectionExpression != null; + } +} diff --git a/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java b/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java new file mode 100644 index 00000000..d3c1a3fa --- /dev/null +++ b/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import java.time.Duration; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import java.util.function.Function; +import org.bson.Document; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.mongodb.core.ReactiveMongoTemplate; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author David Turanski + */ +@SpringBootTest(properties = { + "spring.data.mongodb.port=0", + "mongodb.consumer.collection=testing"}) +class MongoDbConsumerApplicationTests { + + @Autowired + private MongoDbConsumerProperties properties; + + @Autowired + private Function, Mono> mongoDbConsumer; + + @Autowired + private ReactiveMongoTemplate mongoTemplate; + + @Test + void testMongodbConsumer() { + Map data1 = new HashMap<>(); + data1.put("foo", "bar"); + + Map data2 = new HashMap<>(); + data2.put("firstName", "Foo"); + data2.put("lastName", "Bar"); + + Flux> messages = Flux.just( + new GenericMessage<>(data1), + new GenericMessage<>(data2), + new GenericMessage<>("{\"my_data\": \"THE DATA\"}") + ); + + messages.flatMap(mongoDbConsumer::apply).blockLast(Duration.ofSeconds(10)); + + StepVerifier.create(this.mongoTemplate.findAll(Document.class, properties.getCollection()) + .sort(Comparator.comparing(d -> d.get("_id").toString()))) + .assertNext(document -> { + assertThat(document.get("foo")).isEqualTo("bar"); + }) + .assertNext(document-> { + assertThat(document.get("firstName")).isEqualTo("Foo"); + assertThat(document.get("lastName")).isEqualTo("Bar"); + }) + .assertNext(document-> { + assertThat(document.get("my_data")).isEqualTo("THE DATA"); + }) + .verifyComplete(); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/consumer/rabbit-consumer/pom.xml b/consumer/rabbit-consumer/pom.xml new file mode 100644 index 00000000..4443782d --- /dev/null +++ b/consumer/rabbit-consumer/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + rabbit-consumer + 1.0.0.BUILD-SNAPSHOT + rabbit-consumer + Rabbit consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-amqp + + + org.springframework.boot + spring-boot-starter-amqp + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + javax.validation + validation-api + + + + diff --git a/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java b/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java new file mode 100644 index 00000000..dd89486e --- /dev/null +++ b/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java @@ -0,0 +1,150 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.rabbit; + +import java.util.function.Function; + +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionNameStrategy; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.Expression; +import org.springframework.integration.amqp.dsl.Amqp; +import org.springframework.integration.amqp.dsl.AmqpOutboundChannelAdapterSpec; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; + +@EnableConfigurationProperties(RabbitConsumerProperties.class) +@Configuration +public class RabbitConsumerConfiguration implements DisposableBean { + + @Autowired + private RabbitProperties bootProperties; + + @Autowired + private ObjectProvider connectionNameStrategy; + + @Autowired + private RabbitConsumerProperties properties; + + @Value("#{${rabbit.converterBeanName:null}}") + private MessageConverter messageConverter; + + private CachingConnectionFactory ownConnectionFactory; + + @Bean + public Function, Object> rabbitConsumer(@Qualifier("amqpChannelAdapter") MessageHandler messageHandler) { + return o -> { + messageHandler.handleMessage(o); + return ""; + }; + } + + @Bean + public MessageHandler amqpChannelAdapter(ConnectionFactory rabbitConnectionFactory) + throws Exception { + + AmqpOutboundChannelAdapterSpec handler = Amqp + .outboundAdapter(rabbitTemplate(this.properties.isOwnConnection() + ? buildLocalConnectionFactory() : rabbitConnectionFactory)) + .mappedRequestHeaders(properties.getMappedRequestHeaders()) + .defaultDeliveryMode(properties.getPersistentDeliveryMode() + ? MessageDeliveryMode.PERSISTENT + : MessageDeliveryMode.NON_PERSISTENT); + + Expression exchangeExpression = this.properties.getExchangeExpression(); + if (exchangeExpression != null) { + handler.exchangeNameExpression(exchangeExpression); + } + else { + handler.exchangeName(this.properties.getExchange()); + } + + Expression routingKeyExpression = this.properties.getRoutingKeyExpression(); + if (routingKeyExpression != null) { + handler.routingKeyExpression(routingKeyExpression); + } + else { + handler.routingKey(this.properties.getRoutingKey()); + } + return handler.get(); + } + + private ConnectionFactory buildLocalConnectionFactory() throws Exception { + this.ownConnectionFactory = new AutoConfig.Creator().rabbitConnectionFactory( + this.bootProperties, this.connectionNameStrategy); + return this.ownConnectionFactory; + } + + @Bean + public RabbitTemplate rabbitTemplate(ConnectionFactory rabbitConnectionFactory) { + RabbitTemplate rabbitTemplate = new RabbitTemplate(rabbitConnectionFactory); + if (this.messageConverter != null) { + rabbitTemplate.setMessageConverter(this.messageConverter); + } + return rabbitTemplate; + } + + @Bean + @ConditionalOnProperty(name = "rabbit.converterBeanName", + havingValue = RabbitConsumerProperties.JSON_CONVERTER) + public Jackson2JsonMessageConverter jsonConverter() { + return new Jackson2JsonMessageConverter(); + } + + @Override + public void destroy() { + if (this.ownConnectionFactory != null) { + this.ownConnectionFactory.destroy(); + } + } + +} + +class AutoConfig extends RabbitAutoConfiguration { + + static class Creator extends RabbitConnectionFactoryCreator { + + @Override + public CachingConnectionFactory rabbitConnectionFactory(RabbitProperties config, + ObjectProvider connectionNameStrategy) + throws Exception { + CachingConnectionFactory cf = super.rabbitConnectionFactory(config, + connectionNameStrategy); + cf.setConnectionNameStrategy( + connectionFactory -> "rabbit.sink.own.connection"); + cf.afterPropertiesSet(); + return cf; + } + + } + +} diff --git a/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java b/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java new file mode 100644 index 00000000..654a9c2b --- /dev/null +++ b/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.rabbit; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.validation.annotation.Validated; + +@ConfigurationProperties("rabbit") +@Validated +public class RabbitConsumerProperties { + + public static final String JSON_CONVERTER = "jsonConverter"; + + /** + * Exchange name - overridden by exchangeNameExpression, if supplied. + */ + private String exchange = ""; + + /** + * A SpEL expression that evaluates to an exchange name. + */ + private Expression exchangeExpression; + + /** + * Routing key - overridden by routingKeyExpression, if supplied. + */ + private String routingKey; + + /** + * A SpEL expression that evaluates to a routing key. + */ + private Expression routingKeyExpression; + + /** + * Default delivery mode when 'amqp_deliveryMode' header is not present, + * true for PERSISTENT. + */ + private boolean persistentDeliveryMode; + + /** + * Headers that will be mapped. + */ + private String[] mappedRequestHeaders = { "*" }; + + /** + * The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. + * If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you. + */ + private String converterBeanName; + + /** + * When true, use a separate connection based on the boot properties. + */ + private boolean ownConnection; + + public String getExchange() { + return this.exchange; + } + + public void setExchange(String exchange) { + this.exchange = exchange; + } + + public Expression getExchangeExpression() { + return this.exchangeExpression; + } + + public void setExchangeExpression(Expression exchangeExpression) { + this.exchangeExpression = exchangeExpression; + } + + public String getRoutingKey() { + return this.routingKey; + } + + public void setRoutingKey(String routingKey) { + this.routingKey = routingKey; + } + + public Expression getRoutingKeyExpression() { + return this.routingKeyExpression; + } + + public void setRoutingKeyExpression(Expression routingKeyExpression) { + this.routingKeyExpression = routingKeyExpression; + } + + public boolean getPersistentDeliveryMode() { + return this.persistentDeliveryMode; + } + + public void setPersistentDeliveryMode(boolean persistentDeliveryMode) { + this.persistentDeliveryMode = persistentDeliveryMode; + } + + public String[] getMappedRequestHeaders() { + return this.mappedRequestHeaders; + } + + public void setMappedRequestHeaders(String[] mappedRequestHeaders) { + this.mappedRequestHeaders = mappedRequestHeaders; + } + + public String getConverterBeanName() { + return this.converterBeanName; + } + + public void setConverterBeanName(String converterBeanName) { + this.converterBeanName = converterBeanName; + } + + @AssertTrue(message = "routingKey or routingKeyExpression is required") + public boolean isRoutingKeyProvided() { + return this.routingKey != null || this.routingKeyExpression != null; + } + + public boolean isOwnConnection() { + return this.ownConnection; + } + + public void setOwnConnection(boolean ownConnection) { + this.ownConnection = ownConnection; + } + +} \ No newline at end of file diff --git a/function/filter-function/.gitignore b/function/filter-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/function/filter-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/function/filter-function/pom.xml b/function/filter-function/pom.xml new file mode 100644 index 00000000..5e5b72d5 --- /dev/null +++ b/function/filter-function/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + filter-function + 1.0.0.BUILD-SNAPSHOT + filter-function + Spring Native Function for applying filter SpEL expressions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.fn + spel-function + 1.0.0.BUILD-SNAPSHOT + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java b/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java new file mode 100644 index 00000000..9859446c --- /dev/null +++ b/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.filter; + +import java.util.Optional; +import java.util.function.Function; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.cloud.fn.spel.SpelFunctionConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; + +@Configuration +@Import(SpelFunctionConfiguration.class) +public class FilterFunctionConfiguration { + + @Bean + public Function, Message> filterFunction( + @Qualifier("spelFunction") Function, Message> spelFunction) { + + return message -> + Optional.of(message) + .filter(m -> (Boolean) spelFunction.apply(m).getPayload()) + .orElse(null); + } + +} diff --git a/function/filter-function/src/main/resources/application.properties b/function/filter-function/src/main/resources/application.properties new file mode 100644 index 00000000..86e445ae --- /dev/null +++ b/function/filter-function/src/main/resources/application.properties @@ -0,0 +1 @@ +spel.function.expression=true diff --git a/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java b/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java new file mode 100644 index 00000000..23afb9be --- /dev/null +++ b/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.filter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(properties = "spel.function.expression=payload.length() > 5") +@DirtiesContext +public class FilterFunctionApplicationTests { + + @Autowired + @Qualifier("filterFunction") + Function, Message> filter; + + @Test + public void testFilter() { + Message filtered = this.filter.apply(new GenericMessage<>("hello")); + assertThat(filtered).isNull(); + filtered = this.filter.apply(new GenericMessage<>("hello world")); + assertThat(filtered).isNotNull() + .extracting(Message::getPayload) + .isEqualTo("hello world"); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/function/payload-converter-function/pom.xml b/function/payload-converter-function/pom.xml new file mode 100644 index 00000000..846b9c54 --- /dev/null +++ b/function/payload-converter-function/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + payload-converter-function + 1.0.0.BUILD-SNAPSHOT + payload-converter-function + Utility message conversion functions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework + spring-messaging + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java b/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java new file mode 100644 index 00000000..2aecc905 --- /dev/null +++ b/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functions; + +import java.util.function.Function; + +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.MimeTypeUtils; + +/** + * + * @author Christian Tzolov + */ +public class ByteArrayTextToString implements Function, Message> { + + @Override + public Message apply(Message message) { + + if (message.getPayload() instanceof byte[]) { + final MessageHeaders headers = message.getHeaders(); + String contentType = headers.containsKey(MessageHeaders.CONTENT_TYPE) + ? headers.get(MessageHeaders.CONTENT_TYPE).toString() + : MimeTypeUtils.APPLICATION_JSON_VALUE; + + if (contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple")) { + message = MessageBuilder.withPayload(new String(((byte[]) message.getPayload()))) + .copyHeaders(message.getHeaders()) + .build(); + } + } + + return message; + } +} + diff --git a/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java b/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java new file mode 100644 index 00000000..24b8a93e --- /dev/null +++ b/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functions; + +import java.util.Collections; +import java.util.function.Function; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ByteArrayTextToStringTests { + + private static final String MESSAGE = "hello world"; + private static Function, Message> converter; + + @BeforeAll + static void before() { + converter = new ByteArrayTextToString(); + } + + @Test + public void testDefaultNoContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes())); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + + converted = converter.apply(new GenericMessage<>(MESSAGE)); // String + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + } + + @Test + public void testApplicationJsonContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo("hello world"); + } + + @Test + public void testPlainTextContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + } + + @Test + public void testOctetContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); + } + + @Test + public void testRandomNonTextContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, "Random Content Type"))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); + } + +} diff --git a/function/spel-function/.gitignore b/function/spel-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/function/spel-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/function/spel-function/pom.xml b/function/spel-function/pom.xml new file mode 100644 index 00000000..f11c25e7 --- /dev/null +++ b/function/spel-function/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + spel-function + 1.0.0.BUILD-SNAPSHOT + spel-function + Spring Native Function for applying SpEL expressions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.function + payload-converter-function + ${project.version} + + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java b/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java new file mode 100644 index 00000000..9e682b1e --- /dev/null +++ b/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import java.util.function.Function; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.transformer.ExpressionEvaluatingTransformer; +import org.springframework.messaging.Message; + +@Configuration +@EnableConfigurationProperties(SpelFunctionProperties.class) +public class SpelFunctionConfiguration { + + @Bean + public Function, Message> spelFunction( + ExpressionEvaluatingTransformer expressionEvaluatingTransformer) { + + return message -> expressionEvaluatingTransformer.transform(message); + } + + @Bean + public ExpressionEvaluatingTransformer expressionEvaluatingTransformer( + SpelFunctionProperties spelFunctionProperties) { + + return new ExpressionEvaluatingTransformer(new SpelExpressionParser() + .parseExpression(spelFunctionProperties.getExpression())); + } + +} diff --git a/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java b/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java new file mode 100644 index 00000000..89bfec3f --- /dev/null +++ b/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +/** + * Configuration properties for the SpEL function. + * + * @author Gary Russell + * @author Artem Bilan + */ +@ConfigurationProperties("spel.function") +public class SpelFunctionProperties { + + private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); + + /** + * A SpEL expression to apply. + */ + private String expression = DEFAULT_EXPRESSION.getExpressionString(); + + public void setExpression(String expression) { + this.expression = expression; + } + + public String getExpression() { + return this.expression; + } + +} diff --git a/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java b/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java new file mode 100644 index 00000000..1ef016c3 --- /dev/null +++ b/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = "spel.function.expression=payload.toUpperCase()") +@DirtiesContext +public class SpelFunctionApplicationTests { + + @Autowired + Function, Message> transformer; + + @Test + public void testTransform() { + final Message transformed = this.transformer.apply(new GenericMessage<>("hello,world")); + assertThat(transformed.getPayload()).isEqualTo("HELLO,WORLD"); + } + + @Test + public void testJson() { + Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON).build(); + final Message transformed = this.transformer.apply(message); + assertThat(transformed.getPayload()).isEqualTo("{\"FOO\":\"BAR\"}"); + } + + @SpringBootApplication + static class TestApplication { + + } +} diff --git a/function/splitter-function/.gitignore b/function/splitter-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/function/splitter-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/function/splitter-function/pom.xml b/function/splitter-function/pom.xml new file mode 100644 index 00000000..fcdeaf2d --- /dev/null +++ b/function/splitter-function/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + splitter-function + 1.0.0.BUILD-SNAPSHOT + splitter-function + Spring Native Function for Splitter + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-validation + + + + org.springframework.integration + spring-integration-file + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java b/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java new file mode 100644 index 00000000..7d1897c7 --- /dev/null +++ b/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import java.nio.charset.Charset; +import java.util.List; +import java.util.function.Function; + +import org.reactivestreams.Publisher; + +import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.channel.ReactiveStreamsSubscribableChannel; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.splitter.AbstractMessageSplitter; +import org.springframework.integration.splitter.DefaultMessageSplitter; +import org.springframework.integration.splitter.ExpressionEvaluatingSplitter; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +import reactor.core.publisher.Flux; + +@Configuration +@EnableConfigurationProperties(SplitterFunctionProperties.class) +public class SplitterFunctionConfiguration { + + @Bean + public Function, List>> splitterFunction(AbstractMessageSplitter messageSplitter, + SplitterFunctionProperties splitterFunctionProperties) { + + messageSplitter.setApplySequence(splitterFunctionProperties.isApplySequence()); + ThreadLocalFluxSinkMessageChannel outputChannel = new ThreadLocalFluxSinkMessageChannel(); + messageSplitter.setOutputChannel(outputChannel); + return message -> { + messageSplitter.handleMessage(message); + return outputChannel.publisherThreadLocal.get(); + }; + } + + @Bean + @ConditionalOnProperty(prefix = "splitter", name = "expression") + public AbstractMessageSplitter expressionSplitter(SplitterFunctionProperties splitterFunctionProperties) { + return new ExpressionEvaluatingSplitter( + new SpelExpressionParser() + .parseExpression(splitterFunctionProperties.getExpression())); + } + + @Bean + @ConditionalOnMissingBean + @Conditional(FileSplitterCondition.class) + public AbstractMessageSplitter fileSplitter(SplitterFunctionProperties splitterFunctionProperties) { + Boolean markers = splitterFunctionProperties.getFileMarkers(); + String charset = splitterFunctionProperties.getCharset(); + if (markers == null) { + markers = false; + } + FileSplitter fileSplitter = new FileSplitter(true, markers, splitterFunctionProperties.getMarkersJson()); + if (charset != null) { + fileSplitter.setCharset(Charset.forName(charset)); + } + return fileSplitter; + } + + + @Bean + @ConditionalOnMissingBean + public AbstractMessageSplitter defaultSplitter(SplitterFunctionProperties splitterFunctionProperties) { + DefaultMessageSplitter defaultMessageSplitter = new DefaultMessageSplitter(); + defaultMessageSplitter.setDelimiters(splitterFunctionProperties.getDelimiters()); + return defaultMessageSplitter; + } + + static class FileSplitterCondition extends AnyNestedCondition { + + FileSplitterCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(prefix = "splitter", name = "charset") + static class Charset { } + + @ConditionalOnProperty(prefix = "splitter", name = "fileMarkers") + static class FileMarkers { } + + } + + private static final class ThreadLocalFluxSinkMessageChannel + implements MessageChannel, ReactiveStreamsSubscribableChannel { + + private final ThreadLocal>> publisherThreadLocal = new ThreadLocal<>(); + + @Override + @SuppressWarnings("unchecked") + public void subscribeTo(Publisher> publisher) { + this.publisherThreadLocal.set(Flux.from(publisher).collectList().cast(List.class).block()); + } + + @Override + public boolean send(Message message, long l) { + throw new UnsupportedOperationException("This channel only supports a reactive 'subscribeTo()' "); + } + + } + +} diff --git a/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java b/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java new file mode 100644 index 00000000..9749f95f --- /dev/null +++ b/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java @@ -0,0 +1,128 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the Splitter Processor app. + * + * @author Gary Russell + * @author Artem Bilan + */ +@ConfigurationProperties("splitter") +@Validated +public class SplitterFunctionProperties { + + /** + * A SpEL expression for splitting payloads. + */ + private String expression; + + /** + * When expression is null, delimiters to use when tokenizing + * {@link String} payloads. + */ + private String delimiters; + + /** + * Set to true or false to use a {@code FileSplitter} (to split + * text-based files by line) that includes + * (or not) beginning/end of file markers. + */ + private Boolean fileMarkers; + + /** + * When 'fileMarkers == true', specify if they should be produced + * as FileSplitter.FileMarker objects or JSON. + */ + private boolean markersJson = true; + + /** + * The charset to use when converting bytes in text-based files + * to String. + */ + private String charset; + + /** + * Add correlation/sequence information in headers to facilitate later + * aggregation. + */ + private boolean applySequence = true; + + public void setExpression(String expression) { + this.expression = expression; + } + + public String getExpression() { + return this.expression; + } + + public String getDelimiters() { + return this.delimiters; + } + + public void setDelimiters(String delimiters) { + this.delimiters = delimiters; + } + + public Boolean getFileMarkers() { + return this.fileMarkers; + } + + public void setFileMarkers(Boolean fileMarkers) { + this.fileMarkers = fileMarkers; + } + + public boolean getMarkersJson() { + return this.markersJson; + } + + public void setMarkersJson(boolean markersJson) { + this.markersJson = markersJson; + } + + public String getCharset() { + return this.charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public boolean isApplySequence() { + return this.applySequence; + } + + public void setApplySequence(boolean applySequence) { + this.applySequence = applySequence; + } + + @AssertTrue(message = "'delimiters' is not allowed when an 'expression' is provided") + public boolean isDelimitersAllowed() { + return this.expression == null || this.delimiters == null; + } + + @AssertTrue(message = "File properties are not allowed when an 'expression' or 'delimiters' property is provided") + public boolean isFilePropsAllowed() { + return !(this.expression != null || this.delimiters != null) || this.fileMarkers == null && this.charset == null; + } + +} diff --git a/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java b/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java new file mode 100644 index 00000000..6a8a5f94 --- /dev/null +++ b/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(properties = "splitter.expression=payload.split(',')") +@DirtiesContext +public class SplitterFunctionApplicationTests { + + @Autowired + Function, List>> splitter; + + @Test + public void testExpressionSplitter() { + List> messageList = this.splitter.apply(new GenericMessage<>("hello,world")); + assertThat(messageList).extracting(m -> m.getPayload().toString()).contains("hello", "world"); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..cfd3fd98 --- /dev/null +++ b/pom.xml @@ -0,0 +1,205 @@ + + + 4.0.0 + org.springframework.cloud.fn + java-functions-parent + 1.0.0.BUILD-SNAPSHOT + java-functions-parent + Pivotal Java Functions Parent + pom + + + 1.8 + 3.1.1 + 3.2.1 + 2.22.2 + UTF-8 + UTF-8 + ${java.version} + ${java.version} + + + + consumer/cassandra-consumer + consumer/counter-consumer + consumer/file-consumer + consumer/jdbc-consumer + consumer/log-consumer + consumer/mongodb-consumer + consumer/rabbit-consumer + + function/filter-function + function/spel-function + function/payload-converter-function + function/splitter-function + + supplier/file-supplier + supplier/http-supplier + supplier/jdbc-supplier + supplier/mongodb-supplier + supplier/time-supplier + + spring-functions-parent + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + Copyright 2014-2020 the original author or authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. + + See the License for the specific language governing permissions and + limitations under the License. + + + + scm:git:git://github.com/pivotal/java-functions.git + scm:git:ssh://git@github.com/pivotal/java-functions.git + https://github.com/pivotal/java-functions + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + javadoc + package + + jar + + + + + true + + + + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + package + + jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + **/*Tests.java + **/*Test.java + + + **/Abstract*.java + + + + + + + + repo.spring.io + Spring Release Repository + https://repo.spring.io/libs-release-local + + + repo.spring.io + Spring Snapshot Repository + https://repo.spring.io/libs-snapshot-local + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + + + + + milestone + + + repo.spring.io + Spring Milestone Repository + https://repo.spring.io/libs-milestone-local + + + + + central + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + + + + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + + + diff --git a/spring-functions-parent/pom.xml b/spring-functions-parent/pom.xml new file mode 100644 index 00000000..b645f407 --- /dev/null +++ b/spring-functions-parent/pom.xml @@ -0,0 +1,40 @@ + + + + java-functions-parent + org.springframework.cloud.fn + 1.0.0.BUILD-SNAPSHOT + .. + + 4.0.0 + + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + pom + + + 2.3.0.M4 + 3.0.3.RELEASE + + + + + org.springframework.boot + spring-boot-starter-parent + ${spring-boot.version} + import + pom + + + org.springframework.cloud + spring-cloud-function-dependencies + ${spring-cloud-function.version} + import + pom + + + + + \ No newline at end of file diff --git a/supplier/file-supplier/pom.xml b/supplier/file-supplier/pom.xml new file mode 100644 index 00000000..43d085de --- /dev/null +++ b/supplier/file-supplier/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + file-supplier + 1.0.0.BUILD-SNAPSHOT + file-supplier + file supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-file + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-json + true + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + + diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java new file mode 100644 index 00000000..b5c08234 --- /dev/null +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * TODO: This will be used in other apps like (S)FTP and S3. Therefore, it might be moved to a common place. + * + * @author David Turanski + * @author Artem Bilan + */ +@ConfigurationProperties("file.consumer") +@Validated +public class FileConsumerProperties { + + /** + * The FileReadingMode to use for file reading sources. + * Values are 'ref' - The File object, + * 'lines' - a message per line, or + * 'contents' - the contents as bytes. + */ + private FileReadingMode mode = FileReadingMode.contents; + + /** + * Set to true to emit start of file/end of file marker messages before/after the data. + * Only valid with FileReadingMode 'lines'. + */ + private Boolean withMarkers = null; + + /** + * When 'fileMarkers == true', specify if they should be produced + * as FileSplitter.FileMarker objects or JSON. + */ + private boolean markersJson = true; + + @NotNull + public FileReadingMode getMode() { + return this.mode; + } + + public void setMode(FileReadingMode mode) { + this.mode = mode; + } + + public Boolean getWithMarkers() { + return this.withMarkers; + } + + public void setWithMarkers(Boolean withMarkers) { + this.withMarkers = withMarkers; + } + + public boolean getMarkersJson() { + return this.markersJson; + } + + public void setMarkersJson(boolean markersJson) { + this.markersJson = markersJson; + } + + @AssertTrue(message = "withMarkers can only be supplied when FileReadingMode is 'lines'") + public boolean isWithMarkersValid() { + return this.withMarkers == null || FileReadingMode.lines == this.mode; + } +} diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java new file mode 100644 index 00000000..3d357733 --- /dev/null +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +/** + * Defines the supported modes of reading and processing files. + * + * @author Gunnar Hillert + * @author David Turanski + */ +public enum FileReadingMode { + ref, + lines, + contents; +} diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java new file mode 100644 index 00000000..ff4139a6 --- /dev/null +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java @@ -0,0 +1,104 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.util.function.Supplier; + +import org.reactivestreams.Publisher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.file.FileReadingMessageSource; +import org.springframework.integration.file.dsl.FileInboundChannelAdapterSpec; +import org.springframework.integration.file.dsl.Files; +import org.springframework.messaging.Message; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * @author Artem Bilan + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties({FileSupplierProperties.class, FileConsumerProperties.class}) +public class FileSupplierConfiguration { + + private final FileSupplierProperties fileSupplierProperties; + + private final FileConsumerProperties fileConsumerProperties; + + @Autowired + @Lazy + @Qualifier("fileMessageSource") + private FileReadingMessageSource fileMessageSource; + + public FileSupplierConfiguration(FileSupplierProperties fileSupplierProperties, + FileConsumerProperties fileConsumerProperties) { + this.fileSupplierProperties = fileSupplierProperties; + this.fileConsumerProperties = fileConsumerProperties; + } + + @Bean + public FileInboundChannelAdapterSpec fileMessageSource() { + final FileInboundChannelAdapterSpec fileInboundChannelAdapterSpec = + Files.inboundAdapter(this.fileSupplierProperties.getDirectory()); + if (StringUtils.hasText(this.fileSupplierProperties.getFilenamePattern())) { + fileInboundChannelAdapterSpec.patternFilter(this.fileSupplierProperties.getFilenamePattern()); + } + else if (this.fileSupplierProperties.getFilenameRegex() != null) { + fileInboundChannelAdapterSpec.regexFilter(this.fileSupplierProperties.getFilenameRegex().pattern()); + } + fileInboundChannelAdapterSpec.preventDuplicates(this.fileSupplierProperties.isPreventDuplicates()); + return fileInboundChannelAdapterSpec; + } + + @Bean + public Flux> fileMessageFlux() { + return Mono.>create(monoSink -> + monoSink.onRequest(value -> + monoSink.success(this.fileMessageSource.receive()))) + .subscribeOn(Schedulers.boundedElastic()) + .repeatWhenEmpty(it -> it.delayElements(this.fileSupplierProperties.getDelayWhenEmpty())) + .repeat(); + } + + @Bean + @ConditionalOnExpression("environment['file.consumer.mode'] != 'ref'") + public Publisher> fileReadingFlow() { + IntegrationFlowBuilder flowBuilder = IntegrationFlows.from(fileMessageFlux()); + return FileUtils.enhanceFlowForReadingMode(flowBuilder, this.fileConsumerProperties) + .toReactivePublisher(); + } + + @Bean + public Supplier>> fileSupplier() { + if (this.fileConsumerProperties.getMode() == FileReadingMode.ref) { + return this::fileMessageFlux; + } + else { + return () -> Flux.from(fileReadingFlow()); + } + } +} diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java new file mode 100644 index 00000000..3e14582b --- /dev/null +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java @@ -0,0 +1,112 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.File; +import java.time.Duration; +import java.util.regex.Pattern; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Properties for the file supplier. + * + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@ConfigurationProperties("file.supplier") +@Validated +public class FileSupplierProperties { + + private static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + + File.separator + "file-supplier"; + + /** + * The directory to poll for new files. + */ + private File directory = new File(DEFAULT_DIR); + + /** + * Set to true to include an AcceptOnceFileListFilter which prevents duplicates. + */ + private boolean preventDuplicates = true; + + /** + * A simple ant pattern to match files. + */ + private String filenamePattern; + + /** + * A regex pattern to match files. + */ + private Pattern filenameRegex; + + /** + * Duration of delay when no new files are detected. + */ + private Duration delayWhenEmpty = Duration.ofSeconds(1); + + public File getDirectory() { + return this.directory; + } + + public void setDirectory(File directory) { + this.directory = directory; + } + + public boolean isPreventDuplicates() { + return this.preventDuplicates; + } + + public void setPreventDuplicates(boolean preventDuplicates) { + this.preventDuplicates = preventDuplicates; + } + + public String getFilenamePattern() { + return this.filenamePattern; + } + + public void setFilenamePattern(String filenamePattern) { + this.filenamePattern = filenamePattern; + } + + public Pattern getFilenameRegex() { + return this.filenameRegex; + } + + public void setFilenameRegex(Pattern filenameRegex) { + this.filenameRegex = filenameRegex; + } + + //@AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") + + public boolean isExclusivePatterns() { + return !(this.filenamePattern != null && this.filenameRegex != null); + } + + public Duration getDelayWhenEmpty() { + return delayWhenEmpty; + } + + public void setDelayWhenEmpty(Duration delayWhenEmpty) { + this.delayWhenEmpty = delayWhenEmpty; + } + + +} diff --git a/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java new file mode 100644 index 00000000..7e1e04e0 --- /dev/null +++ b/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java @@ -0,0 +1,103 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.util.Collections; + +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.file.transformer.FileToByteArrayTransformer; +import org.springframework.integration.transformer.StreamTransformer; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.MimeTypeUtils; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Christian Tzolov + * + */ +public class FileUtils { + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new FileToByteArrayTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_JSON_VALUE)); + break; + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode."); + } + return flowBuilder; + } + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}; used for streaming sources. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceStreamFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new StreamTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode when streaming."); + } + return flowBuilder; + } + +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java new file mode 100644 index 00000000..56c3f4e8 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.nio.file.Path; +import java.util.function.Supplier; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import reactor.core.publisher.Flux; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class AbstractFileSupplierTests { + + @TempDir + static Path tempDir; + + @Autowired + Supplier>> fileSupplier; + + @BeforeAll + public static void beforeAll() { + System.setProperty("file.supplier.directory", tempDir.toAbsolutePath().toString()); + } + + @AfterAll + public static void afterAll() { + System.clearProperty("file.supplier.directory"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java new file mode 100644 index 00000000..b933e9a3 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.FileHeaders; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +public class DefaultFileSupplierTests extends AbstractFileSupplierTests { + + @Test + public void testBasicFlow() throws IOException { + + Path firstFile = tempDir.resolve("first.file"); + Files.write(firstFile, "first.file".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + //create file after subscription + Path tempFile = tempDir.resolve("test.file"); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo("first.file".getBytes()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); + } + ) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo("testing".getBytes()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "test.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "test.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, tempFile.toFile()); + }) + .thenCancel() + .verifyLater(); + Files.write(tempFile, "testing".getBytes()); + stepVerifier.verify(); + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java new file mode 100644 index 00000000..c420f74b --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.FileHeaders; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.mode=ref") +public class FileModeRefTests extends AbstractFileSupplierTests { + + @Test + public void testBasicFlow() throws IOException { + + Path firstFile = tempDir.resolve("first.file"); + Files.write(firstFile, "first.file".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(firstFile.toAbsolutePath().toFile()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); + } + ) + .thenCancel() + .verify(); + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java new file mode 100644 index 00000000..e1c81697 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenamePattern = *.txt"}) +public class FilePayloadWithPatternTests extends AbstractFileSupplierTests { + + @Test + public void testPattern() throws IOException { + + Path txtFile1 = tempDir.resolve("test1.txt"); + Files.write(txtFile1, "one".getBytes()); + Path nonTxtExtension = tempDir.resolve("hello.bin"); + Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); + Path txtFile2 = tempDir.resolve("test2.txt"); + Files.write(txtFile2, "two".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile1.toAbsolutePath().toFile()); + } + ) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile2.toAbsolutePath().toFile()); + }) + .expectNoEvent(Duration.ofSeconds(1)) + .thenCancel() + .verify(); + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java new file mode 100644 index 00000000..0bf999c1 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenameRegex = t.*.txt"}) +public class FilePayloadWithRegexTests extends AbstractFileSupplierTests { + + @Test + public void testRegexPattern() throws IOException { + + Path txtFile1 = tempDir.resolve("test1.txt"); + Files.write(txtFile1, "one".getBytes()); + Path nonTxtExtension = tempDir.resolve("hello.bin"); + Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); + Path txtFile2 = tempDir.resolve("abc.txt"); + Files.write(txtFile2, "two".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile1.toAbsolutePath().toFile()); + } + ) + .expectNoEvent(Duration.ofSeconds(1)) + .expectNoEvent(Duration.ofSeconds(1)) + .thenCancel() + .verifyLater(); + + stepVerifier.verify(); + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java new file mode 100644 index 00000000..f03bea85 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.json.JsonPathUtils; +import org.springframework.integration.support.json.JsonObjectMapperProvider; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=lines", "file.consumer.withMarkers = true"}) +public class LinesAndMarkersAsJsonPayloadTests extends AbstractFileSupplierTests { + + @Test + public void testLinesWithMarkers() throws Exception { + Path firstFile = tempDir.resolve("test.file"); + Files.write(firstFile, "first line\n".getBytes()); + Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + try { + final Object evaluate = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); + assertThat(evaluate).isEqualTo(FileSplitter.FileMarker.Mark.START.name()); + } catch (IOException e) { + // passt through + } + } + ) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) + .assertNext((message) -> { + try { + final Object fileMarker = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); + assertThat(fileMarker).isEqualTo(FileSplitter.FileMarker.Mark.END.name()); + FileSplitter.FileMarker fileMarker1 = JsonObjectMapperProvider.newInstance() + .fromJson(fileMarker, FileSplitter.FileMarker.class); + assertThat(FileSplitter.FileMarker.Mark.END).isEqualTo(fileMarker1.getMark()); + assertThat(firstFile.toAbsolutePath()).isEqualTo(fileMarker1.getFilePath()); + assertThat(fileMarker1.getLineCount()).isEqualTo(2); + } catch (IOException e) { + // passt through + } + } + ) + .thenCancel() + .verify(); + } +} diff --git a/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java new file mode 100644 index 00000000..a70a0841 --- /dev/null +++ b/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.mode=lines") +public class LinesPayloadTests extends AbstractFileSupplierTests { + + @Test + public void testLines() throws IOException { + Path firstFile = tempDir.resolve("test.file"); + Files.write(firstFile, "first line\n".getBytes()); + Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) + .thenCancel() + .verify(); + + } +} diff --git a/supplier/http-supplier/.gitignore b/supplier/http-supplier/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/supplier/http-supplier/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/supplier/http-supplier/pom.xml b/supplier/http-supplier/pom.xml new file mode 100644 index 00000000..0e4ab57b --- /dev/null +++ b/supplier/http-supplier/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + http-supplier + 1.0.0.BUILD-SNAPSHOT + http-supplier + HTTP Supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter-webflux + + + + org.springframework.integration + spring-integration-webflux + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + javax.validation + validation-api + + + org.hibernate.validator + hibernate-validator + 6.1.0.Final + + + + diff --git a/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java b/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java new file mode 100644 index 00000000..4210d5bb --- /dev/null +++ b/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.http; + +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.http.support.DefaultHttpHeaderMapper; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.cors.CorsConfiguration; + +/** + * Configuration properties for the HTTP Supplier. + * + * @author Artem Bilan + */ +@ConfigurationProperties("http") +@Validated +public class HttpSourceProperties { + + /** + * HTTP endpoint path mapping. + */ + private String pathPattern = "/"; + + /** + * Headers that will be mapped. + */ + private String[] mappedRequestHeaders = { DefaultHttpHeaderMapper.HTTP_REQUEST_HEADER_NAME_PATTERN }; + + /** + * CORS properties. + */ + private Cors cors = new Cors(); + + @NotEmpty + public String getPathPattern() { + return this.pathPattern; + } + + public void setPathPattern(String pathPattern) { + this.pathPattern = pathPattern; + } + + public String[] getMappedRequestHeaders() { + return this.mappedRequestHeaders; + } + + public void setMappedRequestHeaders(String[] mappedRequestHeaders) { + this.mappedRequestHeaders = mappedRequestHeaders; + } + + public Cors getCors() { + return this.cors; + } + + public void setCors(Cors cors) { + this.cors = cors; + } + + public static class Cors { + + /** + * List of allowed origins, e.g. "http://domain1.com". + */ + private String[] allowedOrigins = { CorsConfiguration.ALL }; + + /** + * List of request headers that can be used during the actual request. + */ + private String[] allowedHeaders = { CorsConfiguration.ALL }; + + /** + * Whether the browser should include any cookies associated with the domain of the request being annotated. + */ + private Boolean allowCredentials; + + @NotEmpty + public String[] getAllowedOrigins() { + return this.allowedOrigins; + } + + public void setAllowedOrigins(String[] allowedOrigins) { + this.allowedOrigins = allowedOrigins; + } + + @NotEmpty + public String[] getAllowedHeaders() { + return this.allowedHeaders; + } + + public void setAllowedHeaders(String[] allowedHeaders) { + this.allowedHeaders = allowedHeaders; + } + + public Boolean getAllowCredentials() { + return allowCredentials; + } + + public void setAllowCredentials(Boolean allowCredentials) { + this.allowCredentials = allowCredentials; + } + + } + +} diff --git a/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java b/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java new file mode 100644 index 00000000..a59bd7d6 --- /dev/null +++ b/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2011-2018 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.http; + +import java.util.function.Supplier; + +import org.reactivestreams.Publisher; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.expression.ValueExpression; +import org.springframework.integration.http.support.DefaultHttpHeaderMapper; +import org.springframework.integration.mapping.HeaderMapper; +import org.springframework.integration.webflux.dsl.WebFlux; +import org.springframework.integration.webflux.inbound.WebFluxInboundEndpoint; +import org.springframework.messaging.Message; + +import reactor.core.publisher.Flux; + +/** + * Configuration for the HTTP Supplier. + * + * @author Artem Bilan + */ +@EnableConfigurationProperties(HttpSourceProperties.class) +@Configuration +public class HttpSupplierConfiguration { + + @Bean + public Publisher> httpSupplierFlow(HttpSourceProperties httpSourceProperties) { + return IntegrationFlows.from( + WebFlux.inboundChannelAdapter(httpSourceProperties.getPathPattern()) + .requestPayloadType(byte[].class) + .statusCodeExpression(new ValueExpression<>(HttpStatus.ACCEPTED)) + .mappedRequestHeaders(httpSourceProperties.getMappedRequestHeaders()) + .crossOrigin(crossOrigin -> + crossOrigin.origin(httpSourceProperties.getCors().getAllowedOrigins()) + .allowedHeaders(httpSourceProperties.getCors().getAllowedHeaders()) + .allowCredentials(httpSourceProperties.getCors().getAllowCredentials())) + .autoStartup(false)) + .toReactivePublisher(); + } + + @Bean + public HeaderMapper httpHeaderMapper() { + return DefaultHttpHeaderMapper.inboundMapper(); + } + + @Bean + public Supplier>> httpSupplier( + Publisher> httpRequestPublisher, + WebFluxInboundEndpoint webFluxInboundEndpoint) { + + return () -> Flux.from(httpRequestPublisher) + .doOnSubscribe((subscription) -> webFluxInboundEndpoint.start()) + .doOnTerminate(webFluxInboundEndpoint::stop); + } + +} diff --git a/supplier/http-supplier/src/main/resources/application.properties b/supplier/http-supplier/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/supplier/http-supplier/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java b/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java new file mode 100644 index 00000000..c7d229c7 --- /dev/null +++ b/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java @@ -0,0 +1,123 @@ +package org.springframework.cloud.fn.supplier.http; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.http.MediaType; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.integration.http.HttpHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.web.reactive.function.client.WebClient; + +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import reactor.core.publisher.Flux; +import reactor.netty.http.client.HttpClient; +import reactor.test.StepVerifier; + +/** + * The test for HTTP Supplier. + * + * @author Artem Bilan + */ +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "server.ssl.key-store=classpath:test.jks", + "server.ssl.key-password=password", + "server.ssl.trust-store=classpath:test.jks", + "server.ssl.client-auth=want" + }) +public class HttpSupplierApplicationTests { + + @Autowired + private Supplier>> httpSupplier; + + @LocalServerPort + private int port; + + @Test + public void testHttpSupplier() { + Flux> messageFlux = this.httpSupplier.get(); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .isEqualTo("test1".getBytes())) + .satisfies((msg) -> assertThat(msg.getHeaders()) + .containsEntry(MessageHeaders.CONTENT_TYPE, + new MediaType("text", "plain", StandardCharsets.UTF_8)) + .extractingByKey(HttpHeaders.REQUEST_URL).asString() + .startsWith("https://")) + ) + .assertNext((message) -> + assertThat(message) + .extracting(Message::getPayload) + .isEqualTo("{\"name\":\"test2\"}".getBytes())) + .assertNext((message) -> + assertThat(message) + .extracting(Message::getPayload) + .isEqualTo("{\"name\":\"test3\"}".getBytes())) + .thenCancel() + .verifyLater(); + + HttpClient httpClient = + HttpClient.create() + .secure(sslSpec -> + sslSpec.sslContext(SslContextBuilder.forClient() + .sslProvider(SslProvider.JDK) + .trustManager(InsecureTrustManagerFactory.INSTANCE))); + + WebClient webClient = + WebClient.builder() + .clientConnector(new ReactorClientHttpConnector(httpClient)) + .baseUrl("https://localhost:" + port) + .build(); + + WebClient.RequestBodySpec requestBodySpec = webClient.post().uri("/"); + requestBodySpec.bodyValue("test1").exchange().block(Duration.ofSeconds(10)); + requestBodySpec.bodyValue(new TestPojo("test2")).exchange().block(Duration.ofSeconds(10)); + requestBodySpec.bodyValue(new TestPojo("test3")).exchange().block(Duration.ofSeconds(10)); + + stepVerifier.verify(); + } + + private static class TestPojo { + + private String name; + + public TestPojo() { + } + + public TestPojo(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + } + + @SpringBootApplication + static class TestApplication { } + +} diff --git a/supplier/http-supplier/src/test/resources/test.jks b/supplier/http-supplier/src/test/resources/test.jks new file mode 100644 index 00000000..0fc3e802 Binary files /dev/null and b/supplier/http-supplier/src/test/resources/test.jks differ diff --git a/supplier/jdbc-supplier/pom.xml b/supplier/jdbc-supplier/pom.xml new file mode 100644 index 00000000..ac7d52be --- /dev/null +++ b/supplier/jdbc-supplier/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + jdbc-supplier + 1.0.0.BUILD-SNAPSHOT + jdbc-supplier + JDBC supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-jdbc + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.cloud + spring-cloud-function-context + ${spring-cloud-function.version} + + + io.pivotal.java.function + splitter-function + ${project.version} + + + org.springframework.boot + spring-boot-configuration-processor + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + com.h2database + h2 + test + + + io.projectreactor + reactor-test + test + + + + diff --git a/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java b/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java new file mode 100644 index 00000000..ec6a59ee --- /dev/null +++ b/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import io.pivotal.java.function.splitter.function.SplitterFunctionConfiguration; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.sql.DataSource; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.function.context.PollableBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.integration.core.MessageSource; +import org.springframework.integration.jdbc.JdbcPollingChannelAdapter; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@Configuration +@EnableConfigurationProperties(JdbcSupplierProperties.class) +@Import(SplitterFunctionConfiguration.class) +public class JdbcSupplierConfiguration { + + private final JdbcSupplierProperties properties; + + private final DataSource dataSource; + + public JdbcSupplierConfiguration(JdbcSupplierProperties properties, DataSource dataSource) { + this.properties = properties; + this.dataSource = dataSource; + } + + @Bean + public MessageSource jdbcMessageSource() { + JdbcPollingChannelAdapter jdbcPollingChannelAdapter = + new JdbcPollingChannelAdapter(this.dataSource, this.properties.getQuery()); + jdbcPollingChannelAdapter.setMaxRows(this.properties.getMaxRows()); + jdbcPollingChannelAdapter.setUpdateSql(this.properties.getUpdate()); + return jdbcPollingChannelAdapter; + } + + @Bean(name = "jdbcSupplier") + @PollableBean(splittable = true) + @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", matchIfMissing = true) + public Supplier>> splittedSupplier(Function, List>> splitterFunction) { + return () -> { + Message received = jdbcMessageSource().receive(); + if (received != null) { + return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> + } + else { + return Flux.empty(); + } + }; + } + + @Bean + @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", havingValue = "false") + public Supplier> jdbcSupplier() { + return () -> jdbcMessageSource().receive(); + } + +} diff --git a/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java b/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java new file mode 100644 index 00000000..6b6d14fc --- /dev/null +++ b/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@ConfigurationProperties("jdbc.supplier") +@Validated +public class JdbcSupplierProperties { + + /** + * The query to use to select data. + */ + private String query; + + /** + * An SQL update statement to execute for marking polled messages as 'seen'. + */ + private String update; + + /** + * Whether to split the SQL result as individual messages. + */ + private boolean split = true; + + /** + * Max numbers of rows to process for query. + */ + private int maxRows = 0; + + @NotNull + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public String getUpdate() { + return update; + } + + public void setUpdate(String update) { + this.update = update; + } + + public boolean isSplit() { + return split; + } + + public void setSplit(boolean split) { + this.split = split; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(int maxRows) { + this.maxRows = maxRows; + } + +} diff --git a/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java b/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java new file mode 100644 index 00000000..49e3a24a --- /dev/null +++ b/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = "jdbc.supplier.query=select id, name from test order by id") +@DirtiesContext +public class DefaultJdbcSupplierTests { + + @Autowired + Supplier>> jdbcSupplier; + + @Test + void testExtraction() { + final Flux> messageFlux = jdbcSupplier.get(); + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(1L) && map.get("NAME").equals("Bob"); + }) + )) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(2L) && map.get("NAME").equals("Jane"); + }) + )) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(3L) && map.get("NAME").equals("John"); + }) + )) + .thenCancel() + .verifyLater(); + stepVerifier.verify(); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java b/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java new file mode 100644 index 00000000..60e000b0 --- /dev/null +++ b/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.fn.supplier.jdbc; + +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = {"jdbc.supplier.query=select id, name from test order by id", "jdbc.supplier.split=false"}) +@DirtiesContext +public class NonSplitJdbcSupplierTests { + + @Autowired + Supplier> jdbcSupplier; + + @Test + void testExtraction() { + final Message message = jdbcSupplier.get(); + final List> payload = (List>) message.getPayload(); + assertThat(payload.size()).isEqualTo(3); + Map map = payload.get(0); + assertThat(map.get("ID")).isEqualTo(1L); + assertThat(map.get("NAME")).isEqualTo("Bob"); + map = payload.get(1); + assertThat(map.get("ID")).isEqualTo(2L); + assertThat(map.get("NAME")).isEqualTo("Jane"); + map = payload.get(2); + assertThat(map.get("ID")).isEqualTo(3L); + assertThat(map.get("NAME")).isEqualTo("John"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/supplier/jdbc-supplier/src/test/resources/schema.sql b/supplier/jdbc-supplier/src/test/resources/schema.sql new file mode 100644 index 00000000..bb5835cb --- /dev/null +++ b/supplier/jdbc-supplier/src/test/resources/schema.sql @@ -0,0 +1,10 @@ +-- Run by default by Boot infrastructure + +create table test( + id bigint, + name varchar (2000), + tag char(1) +); +insert into test values (1, 'Bob', NULL); +insert into test values (2, 'Jane', NULL); +insert into test values (3, 'John', NULL); \ No newline at end of file diff --git a/supplier/mongodb-supplier/.gitignore b/supplier/mongodb-supplier/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/supplier/mongodb-supplier/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/supplier/mongodb-supplier/pom.xml b/supplier/mongodb-supplier/pom.xml new file mode 100644 index 00000000..15845cc1 --- /dev/null +++ b/supplier/mongodb-supplier/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + mongodb-supplier + 1.0.0.BUILD-SNAPSHOT + mongodb-supplier + Mongo DB supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-mongodb + + + org.mongodb + mongodb-driver-sync + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.cloud + spring-cloud-function-context + ${spring-cloud-function.version} + + + org.springframework.cloud.fn + splitter-function + ${project.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + io.projectreactor + reactor-test + test + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java b/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java new file mode 100644 index 00000000..a82a8c68 --- /dev/null +++ b/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration; +import org.springframework.context.annotation.Configuration; +import reactor.core.publisher.Flux; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.function.context.PollableBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.integration.mongodb.inbound.MongoDbMessageSource; +import org.springframework.messaging.Message; + +/** + * A configuration for MongoDB Source applications. Produces + * {@link MongoDbMessageSource} which polls collection with the query after startup + * according to the polling properties. + * + * @author Adam Zwickey + * @author Artem Bilan + * @author David Turanski + * + */ +@Configuration +@EnableConfigurationProperties({ MongodbSupplierProperties.class }) +@Import(SplitterFunctionConfiguration.class) +public class MongodbSupplierConfiguration { + + private final MongodbSupplierProperties properties; + + private final MongoTemplate mongoTemplate; + + public MongodbSupplierConfiguration(MongodbSupplierProperties properties, MongoTemplate mongoTemplate) { + this.properties = properties; + this.mongoTemplate = mongoTemplate; + } + + @Bean(name = "mongodbSupplier") + @PollableBean(splittable = true) + @ConditionalOnProperty(prefix = "mongodb", name = "split", matchIfMissing = true) + public Supplier>> splittedSupplier(Function, List>> splitterFunction) { + return () -> { + Message received = mongoSource().receive(); + if (received != null) { + return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> + } + else { + return Flux.empty(); + } + }; + } + + @Bean + @ConditionalOnProperty(prefix = "mongodb", name = "split", havingValue = "false") + public Supplier> mongodbSupplier() { + return () -> mongoSource().receive(); + } + + /** + * The inheritors can consider to override this method for their purpose or just adjust + * options for the returned instance + * @return a {@link MongoDbMessageSource} instance + */ + @Bean + public MongoDbMessageSource mongoSource() { + Expression queryExpression = (this.properties.getQueryExpression() != null + ? this.properties.getQueryExpression() + : new LiteralExpression(this.properties.getQuery())); + MongoDbMessageSource mongoDbMessageSource = new MongoDbMessageSource(this.mongoTemplate, queryExpression); + mongoDbMessageSource.setCollectionNameExpression(new LiteralExpression(this.properties.getCollection())); + mongoDbMessageSource.setEntityClass(String.class); + return mongoDbMessageSource; + } + +} diff --git a/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java b/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java new file mode 100644 index 00000000..31ba05e6 --- /dev/null +++ b/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java @@ -0,0 +1,91 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.validation.annotation.Validated; + +/** + * @author Adam Zwickey + * @author Artem Bilan + * @author Chris Schaefer + * @author David Turanski + * + */ +@ConfigurationProperties("mongodb.supplier") +@Validated +public class MongodbSupplierProperties { + + /** + * The MongoDB collection to query + */ + private String collection; + + /** + * The MongoDB query + */ + private String query = "{ }"; + + /** + * The SpEL expression in MongoDB query DSL style + */ + private Expression queryExpression; + + /** + * Whether to split the query result as individual messages. + */ + private boolean split = true; + + @NotEmpty(message = "Query is required") + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Expression getQueryExpression() { + return queryExpression; + } + + public void setQueryExpression(Expression queryExpression) { + this.queryExpression = queryExpression; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + @NotBlank(message = "Collection name is required") + public String getCollection() { + return collection; + } + + public boolean isSplit() { + return split; + } + + public void setSplit(boolean split) { + this.split = split; + } + +} diff --git a/supplier/mongodb-supplier/src/main/resources/application.properties b/supplier/mongodb-supplier/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/supplier/mongodb-supplier/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java b/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java new file mode 100644 index 00000000..0c332137 --- /dev/null +++ b/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +@SpringBootTest(properties = { + "spring.data.mongodb.port=0", + "mongodb.supplier.collection=testing"}) +class MongodbSupplierApplicationTests { + + private ObjectMapper objectMapper = new ObjectMapper(); + + @Autowired + private Supplier>> mongodbSupplier; + + @Autowired + private MongoClient mongo; + + @BeforeEach + public void setUp() { + MongoDatabase database = this.mongo.getDatabase("test"); + database.createCollection("testing"); + MongoCollection collection = database.getCollection("testing"); + collection.insertOne( + new Document("greeting", "hello") + .append("name", "foo")); + collection.insertOne( + new Document("greeting", "hola") + .append("name", "bar")); + } + + @Test + void testMongodbSupplier() { + Flux> messageFlux = this.mongodbSupplier.get(); + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(payload(message)).contains( + entry("greeting","hello"), + entry("name", "foo"))) + .assertNext((message) -> + assertThat(payload(message)).contains( + entry("greeting","hola"), + entry("name", "bar"))) + .thenCancel() + .verify(); + } + + private Map payload(Message message) { + Map map = null; + try { + map = objectMapper.readValue(message.getPayload().toString(),HashMap.class); + } + catch (Exception e) { + e.printStackTrace(); + } + return map; + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/supplier/time-supplier/pom.xml b/supplier/time-supplier/pom.xml new file mode 100644 index 00000000..6b871e62 --- /dev/null +++ b/supplier/time-supplier/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + time-supplier + 1.0.0.BUILD-SNAPSHOT + time-supplier + time supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter + + + org.apache.commons + commons-lang3 + + + + org.hibernate.validator + hibernate-validator + true + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + + diff --git a/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java new file mode 100644 index 00000000..77ff2986 --- /dev/null +++ b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.text.SimpleDateFormat; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; + +/** + * The annotated String must be a valid {@link java.text.SimpleDateFormat} pattern. + * + * @author Eric Bottard + * @author Soby Chacko + */ +@Target({ ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE, ElementType.CONSTRUCTOR, + ElementType.PARAMETER }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Constraint(validatedBy = { DateFormat.DateFormatValidator.class }) +public @interface DateFormat { + + String DEFAULT_MESSAGE = ""; + + String message() default DEFAULT_MESSAGE; + + Class[] groups() default { }; + + Class[] payload() default { }; + + public static class DateFormatValidator implements ConstraintValidator { + + private String message; + + @Override + public void initialize(DateFormat constraintAnnotation) { + this.message = constraintAnnotation.message(); + } + + @Override + public boolean isValid(CharSequence value, ConstraintValidatorContext context) { + if (value == null) { + return true; + } + try { + new SimpleDateFormat(value.toString()); + } + catch (IllegalArgumentException e) { + if (DEFAULT_MESSAGE.equals(this.message)) { + context.disableDefaultConstraintViolation(); + context.buildConstraintViolationWithTemplate(e.getMessage()).addConstraintViolation(); + } + return false; + } + return true; + } + + } + +} diff --git a/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java new file mode 100644 index 00000000..3c59377c --- /dev/null +++ b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * @author Soby Chacko + */ +@ConfigurationProperties("time") +@Validated +public class TimeProperties { + + /** + * Format for the date value. + */ + private String dateFormat = "MM/dd/yy HH:mm:ss"; + + @DateFormat + public String getDateFormat() { + return this.dateFormat; + } + + public void setDateFormat(String dateFormat) { + this.dateFormat = dateFormat; + } + +} diff --git a/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java new file mode 100644 index 00000000..40224405 --- /dev/null +++ b/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.util.Date; +import java.util.function.Supplier; + +import org.apache.commons.lang3.time.FastDateFormat; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(TimeProperties.class) +public class TimeSupplierConfiguration { + + @Bean + public Supplier timeSupplier(TimeProperties timeProperties) { + FastDateFormat fastDateFormat = FastDateFormat.getInstance(timeProperties.getDateFormat()); + return () -> fastDateFormat.format(new Date()); + } + +} diff --git a/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java new file mode 100644 index 00000000..4e426863 --- /dev/null +++ b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.junit.jupiter.api.Test; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +public class SimpleTimeSupplierTests extends TimeSupplierApplicationTests { + + @Test + public void testTimeSupplier() { + final String time = timeSupplier.get(); + SimpleDateFormat dateFormat = new SimpleDateFormat(new TimeProperties().getDateFormat()); + assertThatCode(() -> { + Date date = dateFormat.parse(time); + assertThat(date).isNotNull(); + }).doesNotThrowAnyException(); + } + +} diff --git a/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java new file mode 100644 index 00000000..df379308 --- /dev/null +++ b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.util.function.Supplier; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest +public abstract class TimeSupplierApplicationTests { + + @Autowired + Supplier timeSupplier; + + @Autowired + TimeProperties timeProperties; + + protected abstract void testTimeSupplier(); + + @SpringBootApplication + static class TestApplication {} +} diff --git a/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java new file mode 100644 index 00000000..bcbe99d8 --- /dev/null +++ b/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest({ "time.dateFormat=MMddyyyy HH:mm:ss" }) +public class VariationToSimpleTests extends TimeSupplierApplicationTests { + + @Test + public void testTimeSupplier() { + final String time = timeSupplier.get(); + SimpleDateFormat dateFormat = new SimpleDateFormat(timeProperties.getDateFormat()); + assertThatCode(() -> { + Date date = dateFormat.parse(time); + assertThat(date).isNotNull(); + }).doesNotThrowAnyException(); + } + + @Test + public void testInvalidDateFormat() { + TimeProperties timeProperties = new TimeProperties(); + timeProperties.setDateFormat("AA/dd/yyyy HH:mm:ss"); + assertThatIllegalArgumentException().isThrownBy(() -> new SimpleDateFormat(timeProperties.getDateFormat())); + } + +}