commit 517ccd5b40932d65d6c0c34a2fb0633182416dbc Author: Soby Chacko Date: Mon May 4 17:50:58 2020 -0400 Initial Commit Migrating the existing structure from the following location: https://github.com/spring-cloud-stream-app-starters/stream-applications/tree/restructuring diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f5712a10 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +apps/ +/application.yml +/application.properties +asciidoctor.css +*~ +.#* +*# +target/ +build/ +bin/ +_site/ +.classpath +.project +.settings +.springBeans +.DS_Store +*.sw* +*.iml +*.ipr +*.iws +.idea/ +.factorypath +spring-xd-samples/*/xd +dump.rdb +coverage-error.log +.apt_generated +aws.credentials.properties diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 00000000..0e7dabef --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 00000000..3b8cf46e --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000..c32394f1 --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,117 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.5"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 00000000..0d5e6498 Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..7d59a01f --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.2/apache-maven-3.6.2-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar diff --git a/README.adoc b/README.adoc new file mode 100644 index 00000000..a202629f --- /dev/null +++ b/README.adoc @@ -0,0 +1 @@ +== Stream applications \ No newline at end of file diff --git a/applications/apps-core/.settings.xml b/applications/apps-core/.settings.xml new file mode 100644 index 00000000..d8e1afac --- /dev/null +++ b/applications/apps-core/.settings.xml @@ -0,0 +1,81 @@ + + + + + repo.spring.io + ${env.CI_DEPLOY_USERNAME} + ${env.CI_DEPLOY_PASSWORD} + + + + + + spring + + true + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + + gemstone-release + GemStone Maven Release Repository + http://dist.gemstone.com/maven/release + + true + always + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + + + diff --git a/applications/apps-core/CODE_OF_CONDUCT.adoc b/applications/apps-core/CODE_OF_CONDUCT.adoc new file mode 100644 index 00000000..17783c7c --- /dev/null +++ b/applications/apps-core/CODE_OF_CONDUCT.adoc @@ -0,0 +1,44 @@ += Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of fostering an open +and welcoming community, we pledge to respect all people who contribute through reporting +issues, posting feature requests, updating documentation, submitting pull requests or +patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for +everyone, regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, body size, race, ethnicity, age, +religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic addresses, + without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or reject comments, +commits, code, wiki edits, issues, and other contributions that are not aligned to this +Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors +that they deem inappropriate, threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to fairly and +consistently applying these principles to every aspect of managing this project. Project +maintainers who do not follow or enforce the Code of Conduct may be permanently removed +from the project team. + +This Code of Conduct applies both within project spaces and in public spaces when an +individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by +contacting a project maintainer at spring-code-of-conduct@pivotal.io . All complaints will +be reviewed and investigated and will result in a response that is deemed necessary and +appropriate to the circumstances. Maintainers are obligated to maintain confidentiality +with regard to the reporter of an incident. + +This Code of Conduct is adapted from the +https://contributor-covenant.org[Contributor Covenant], version 1.3.0, available at +https://contributor-covenant.org/version/1/3/0/[contributor-covenant.org/version/1/3/0/] diff --git a/applications/apps-core/LICENSE b/applications/apps-core/LICENSE new file mode 100644 index 00000000..9b259bdf --- /dev/null +++ b/applications/apps-core/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/applications/apps-core/README.adoc b/applications/apps-core/README.adoc new file mode 100644 index 00000000..1f493a1c --- /dev/null +++ b/applications/apps-core/README.adoc @@ -0,0 +1,2 @@ +#Core components shared by other projects in the app starters organization +This module consists of core dependencies and other common artifacts. diff --git a/applications/apps-core/common/pom.xml b/applications/apps-core/common/pom.xml new file mode 100644 index 00000000..213da9a8 --- /dev/null +++ b/applications/apps-core/common/pom.xml @@ -0,0 +1,25 @@ + + + + stream-apps-parent + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-common + pom + stream-apps-common + + + stream-apps-file-common + stream-apps-ftp-common + stream-apps-test-support + stream-apps-postprocessor-common + stream-apps-micrometer-common + stream-apps-metadata-store-common + stream-apps-task-launch-request-common + stream-apps-security-common + + + diff --git a/applications/apps-core/common/stream-apps-file-common/pom.xml b/applications/apps-core/common/stream-apps-file-common/pom.xml new file mode 100644 index 00000000..ac0e8a43 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/pom.xml @@ -0,0 +1,21 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-file-common + stream-apps-file-common + + + + org.springframework.integration + spring-integration-file + true + + + + diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileConsumerProperties.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileConsumerProperties.java new file mode 100644 index 00000000..33ff4b0c --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileConsumerProperties.java @@ -0,0 +1,82 @@ +/* + * Copyright 2015-2016 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.file; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * @author David Turanski + * @author Artem Bilan + */ +@ConfigurationProperties("file.consumer") +@Validated +public class FileConsumerProperties { + + /** + * The FileReadingMode to use for file reading sources. + * Values are 'ref' - The File object, + * 'lines' - a message per line, or + * 'contents' - the contents as bytes. + */ + private FileReadingMode mode = FileReadingMode.contents; + + /** + * Set to true to emit start of file/end of file marker messages before/after the data. + * Only valid with FileReadingMode 'lines'. + */ + private Boolean withMarkers = null; + + /** + * When 'fileMarkers == true', specify if they should be produced + * as FileSplitter.FileMarker objects or JSON. + */ + private boolean markersJson = true; + + @NotNull + public FileReadingMode getMode() { + return this.mode; + } + + public void setMode(FileReadingMode mode) { + this.mode = mode; + } + + public Boolean getWithMarkers() { + return this.withMarkers; + } + + public void setWithMarkers(Boolean withMarkers) { + this.withMarkers = withMarkers; + } + + public boolean getMarkersJson() { + return this.markersJson; + } + + public void setMarkersJson(boolean markersJson) { + this.markersJson = markersJson; + } + + @AssertTrue(message = "withMarkers can only be supplied when FileReadingMode is 'lines'") + public boolean isWithMarkersValid() { + return this.withMarkers == null || FileReadingMode.lines == this.mode; + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileReadingMode.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileReadingMode.java new file mode 100644 index 00000000..593fc757 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileReadingMode.java @@ -0,0 +1,30 @@ +/* + * Copyright 2015 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.file; + + +/** + * Defines the supported modes of reading and processing files for the + * {@code File}, {@code FTP} and {@code SFTP} sources. + * + * @author Gunnar Hillert + * @author David Turanski + */ +public enum FileReadingMode { + ref, + lines, + contents; +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileUtils.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileUtils.java new file mode 100644 index 00000000..4537ee66 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/FileUtils.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.file; + +import java.util.Collections; + +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.file.transformer.FileToByteArrayTransformer; +import org.springframework.integration.transformer.StreamTransformer; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.MimeTypeUtils; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Christian Tzolov + * + */ +public class FileUtils { + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new FileToByteArrayTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_JSON_VALUE)); + break; + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode."); + } + return flowBuilder; + } + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}; used for streaming sources. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceStreamFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new StreamTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode when streaming."); + } + return flowBuilder; + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileProperties.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileProperties.java new file mode 100644 index 00000000..cb24e8e9 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileProperties.java @@ -0,0 +1,71 @@ +/* + * Copyright 2015-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.file.remote; + +import org.hibernate.validator.constraints.NotBlank; + +/** + * @deprecated - properties are flattened. + * + * @author Gary Russell + * + */ +@Deprecated +public abstract class AbstractRemoteFileProperties { + + /** + * The remote FTP directory. + */ + private String remoteDir = "/"; + + /** + * The suffix to use while the transfer is in progress. + */ + private String tmpFileSuffix = ".tmp"; + + /** + * The remote file separator. + */ + private String remoteFileSeparator = "/"; + + @NotBlank + public String getRemoteDir() { + return remoteDir; + } + + public final void setRemoteDir(String remoteDir) { + this.remoteDir = remoteDir; + } + + @NotBlank + public String getTmpFileSuffix() { + return tmpFileSuffix; + } + + public void setTmpFileSuffix(String tmpFileSuffix) { + this.tmpFileSuffix = tmpFileSuffix; + } + + @NotBlank + public String getRemoteFileSeparator() { + return remoteFileSeparator; + } + + public void setRemoteFileSeparator(String remoteFileSeparator) { + this.remoteFileSeparator = remoteFileSeparator; + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSinkProperties.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSinkProperties.java new file mode 100644 index 00000000..b3db48dc --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSinkProperties.java @@ -0,0 +1,102 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.file.remote; + +import javax.validation.constraints.NotNull; + +import org.hibernate.validator.constraints.NotBlank; + +import org.springframework.expression.Expression; +import org.springframework.integration.file.support.FileExistsMode; + +/** + * @deprecated - properties are flattened. + * + * @author Gary Russell + * + */ +@Deprecated +public abstract class AbstractRemoteFileSinkProperties extends AbstractRemoteFileProperties { + + /** + * A temporary directory where the file will be written if {@link #isUseTemporaryFilename()} + * is true. + */ + private String temporaryRemoteDir = "/"; + + /** + * Whether or not to create the remote directory. + */ + private boolean autoCreateDir = true; + + /** + * Action to take if the remote file already exists. + */ + private FileExistsMode mode = FileExistsMode.REPLACE; + + /** + * Whether or not to write to a temporary file and rename. + */ + private boolean useTemporaryFilename = true; + + /** + * A SpEL expression to generate the remote file name. + */ + private Expression filenameExpression; + + @NotBlank + public String getTemporaryRemoteDir() { + return this.temporaryRemoteDir; + } + + public void setTemporaryRemoteDir(String temporaryRemoteDir) { + this.temporaryRemoteDir = temporaryRemoteDir; + } + + public boolean isAutoCreateDir() { + return this.autoCreateDir; + } + + public void setAutoCreateDir(boolean autoCreateDir) { + this.autoCreateDir = autoCreateDir; + } + + @NotNull + public FileExistsMode getMode() { + return this.mode; + } + + public void setMode(FileExistsMode mode) { + this.mode = mode; + } + + public boolean isUseTemporaryFilename() { + return this.useTemporaryFilename; + } + + public void setUseTemporaryFilename(boolean useTemporaryFilename) { + this.useTemporaryFilename = useTemporaryFilename; + } + + public Expression getFilenameExpression() { + return this.filenameExpression; + } + + public void setFilenameExpression(Expression filenameExpression) { + this.filenameExpression = filenameExpression; + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSourceProperties.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSourceProperties.java new file mode 100644 index 00000000..73173f75 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteFileSourceProperties.java @@ -0,0 +1,120 @@ +/* + * Copyright 2015-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.file.remote; + +import java.io.File; +import java.util.regex.Pattern; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotNull; + +/** + * Common properties for remote file sources (e.g. (S)FTP). + * + * @deprecated - properties are flattened. + * + * @author David Turanski + * @author Gary Russell + * + */ +@Deprecated +public abstract class AbstractRemoteFileSourceProperties extends AbstractRemoteFileProperties { + + /** + * Set to true to delete remote files after successful transfer. + */ + private boolean deleteRemoteFiles = false; + + /** + * The local directory to use for file transfers. + */ + private File localDir = new File(System.getProperty("java.io.tmpdir") + "/xd/ftp"); + + /** + * Set to true to create the local directory if it does not exist. + */ + private boolean autoCreateLocalDir = true; + + /** + * A filter pattern to match the names of files to transfer. + */ + private String filenamePattern; + + /** + * A filter regex pattern to match the names of files to transfer. + */ + private Pattern filenameRegex; + + /** + * Set to true to preserve the original timestamp. + */ + private boolean preserveTimestamp = true; + + public boolean isAutoCreateLocalDir() { + return autoCreateLocalDir; + } + + public void setAutoCreateLocalDir(boolean autoCreateLocalDir) { + this.autoCreateLocalDir = autoCreateLocalDir; + } + + public boolean isDeleteRemoteFiles() { + return deleteRemoteFiles; + } + + public void setDeleteRemoteFiles(boolean deleteRemoteFiles) { + this.deleteRemoteFiles = deleteRemoteFiles; + } + + @NotNull + public File getLocalDir() { + return localDir; + } + + public final void setLocalDir(File localDir) { + this.localDir = localDir; + } + + public String getFilenamePattern() { + return filenamePattern; + } + + public void setFilenamePattern(String filenamePattern) { + this.filenamePattern = filenamePattern; + } + + public Pattern getFilenameRegex() { + return filenameRegex; + } + + public void setFilenameRegex(Pattern filenameRegex) { + this.filenameRegex = filenameRegex; + } + + public boolean isPreserveTimestamp() { + return preserveTimestamp; + } + + public void setPreserveTimestamp(boolean preserveTimestamp) { + this.preserveTimestamp = preserveTimestamp; + } + + @AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") + public boolean isExclusivePatterns() { + return !(this.filenamePattern != null && this.filenameRegex != null); + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteServerProperties.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteServerProperties.java new file mode 100644 index 00000000..a5c3ee35 --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/AbstractRemoteServerProperties.java @@ -0,0 +1,86 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.file.remote; + +import org.hibernate.validator.constraints.NotBlank; + +/** + * Common properties for remote servers (e.g. (S)FTP). + * + * @deprecated - properties are flattened. + * + * @author David Turanski + * @author Gary Russell + * + */ +@Deprecated +public abstract class AbstractRemoteServerProperties { + + /** + * The host name of the server. + */ + private String host = "localhost"; + + /** + * The username to use to connect to the server. + */ + + private String username; + /** + * The password to use to connect to the server. + */ + private String password; + + /** + * Cache sessions + */ + private Boolean cacheSessions; + + @NotBlank + public String getHost() { + return this.host; + } + + public void setHost(String host) { + this.host = host; + } + + @NotBlank + public String getUsername() { + return this.username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return this.password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Boolean getCacheSessions() { + return this.cacheSessions; + } + + public void setCacheSessions(Boolean cacheSessions) { + this.cacheSessions = cacheSessions; + } + +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/FilePathUtils.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/FilePathUtils.java new file mode 100644 index 00000000..f2d0306b --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/FilePathUtils.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.file.remote; + +import java.nio.file.Paths; + +import org.springframework.integration.file.FileHeaders; +import org.springframework.lang.Nullable; +import org.springframework.messaging.Message; +import org.springframework.util.Assert; + +/** + * @author David Turanski + **/ +public abstract class FilePathUtils { + /** + * Returns a remote file path for a message with a file name as payload and {@link FileHeaders#REMOTE_DIRECTORY} + * included as a message header. + * + * @param message the message containing the header. + * @return the file path. + */ + @Nullable + public static String getRemoteFilePath(Message message) { + if (message.getHeaders().containsKey(FileHeaders.REMOTE_DIRECTORY)) { + String filename = (String) message.getPayload(); + String remoteDirectory = (String) message.getHeaders().get(FileHeaders.REMOTE_DIRECTORY); + return getPath(remoteDirectory, filename); + } + return null; + } + + public static String getLocalFilePath(String localDirectory, String filename) { + if (localDirectory != null) { + return getPath(localDirectory, filename); + } + return filename; + } + + private static String getPath(String dirName, String fileName) { + return Paths.get(dirName, fileName).toString(); + } +} diff --git a/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/RemoteFileDeletingTransactionSynchronizationProcessor.java b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/RemoteFileDeletingTransactionSynchronizationProcessor.java new file mode 100644 index 00000000..7c4518ef --- /dev/null +++ b/applications/apps-core/common/stream-apps-file-common/src/main/java/org/springframework/cloud/stream/app/file/remote/RemoteFileDeletingTransactionSynchronizationProcessor.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.file.remote; + +import org.springframework.integration.file.FileHeaders; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.transaction.IntegrationResourceHolder; +import org.springframework.integration.transaction.TransactionSynchronizationProcessor; + +/** + * A {@link TransactionSynchronizationProcessor} that deletes a remote file on + * success. + * + * @author Gary Russell + * + */ +public class RemoteFileDeletingTransactionSynchronizationProcessor implements TransactionSynchronizationProcessor { + + private final RemoteFileTemplate template; + + private final String remoteFileSeparator; + + /** + * Construct an instance with the provided template and separator. + * @param template the template. + * @param remoteFileSeparator the separator. + */ + public RemoteFileDeletingTransactionSynchronizationProcessor(RemoteFileTemplate template, + String remoteFileSeparator) { + this.template = template; + this.remoteFileSeparator = remoteFileSeparator; + } + + @Override + public void processBeforeCommit(IntegrationResourceHolder holder) { + } + + @Override + public void processAfterRollback(IntegrationResourceHolder holder) { + } + + @Override + public void processAfterCommit(IntegrationResourceHolder holder) { + String remoteDir = (String) holder.getMessage().getHeaders().get(FileHeaders.REMOTE_DIRECTORY); + String remoteFile = (String) holder.getMessage().getHeaders().get(FileHeaders.REMOTE_FILE); + this.template.remove(remoteDir + this.remoteFileSeparator + remoteFile); + } + +} diff --git a/applications/apps-core/common/stream-apps-ftp-common/pom.xml b/applications/apps-core/common/stream-apps-ftp-common/pom.xml new file mode 100644 index 00000000..d1c410f5 --- /dev/null +++ b/applications/apps-core/common/stream-apps-ftp-common/pom.xml @@ -0,0 +1,26 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-ftp-common + stream-apps-ftp-common + + + + org.springframework.integration + spring-integration-ftp + true + + + org.springframework.cloud.stream.app + stream-apps-file-common + ${project.version} + + + + diff --git a/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryConfiguration.java b/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryConfiguration.java new file mode 100644 index 00000000..b28b9c2b --- /dev/null +++ b/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryConfiguration.java @@ -0,0 +1,56 @@ +/* + * Copyright 2015-2016 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.ftp; + +import org.apache.commons.net.ftp.FTPFile; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.file.remote.session.CachingSessionFactory; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.ftp.session.DefaultFtpSessionFactory; + +/** + * FTP Session factory configuration. + * + * @author David Turanski + * @author Gary Russell + */ +@Configuration +@EnableConfigurationProperties(FtpSessionFactoryProperties.class) +public class FtpSessionFactoryConfiguration { + + @Bean + @ConditionalOnMissingBean + public SessionFactory ftpSessionFactory(FtpSessionFactoryProperties properties) { + DefaultFtpSessionFactory ftpSessionFactory = new DefaultFtpSessionFactory(); + ftpSessionFactory.setHost(properties.getHost()); + ftpSessionFactory.setPort(properties.getPort()); + ftpSessionFactory.setUsername(properties.getUsername()); + ftpSessionFactory.setPassword(properties.getPassword()); + ftpSessionFactory.setClientMode(properties.getClientMode().getMode()); + if (properties.getCacheSessions() != null) { + CachingSessionFactory csf = new CachingSessionFactory<>(ftpSessionFactory); + return csf; + } + else { + return ftpSessionFactory; + } + } + +} diff --git a/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryProperties.java b/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryProperties.java new file mode 100644 index 00000000..cf4ea143 --- /dev/null +++ b/applications/apps-core/common/stream-apps-ftp-common/src/main/java/org/springframework/cloud/stream/app/ftp/FtpSessionFactoryProperties.java @@ -0,0 +1,82 @@ +/* + * Copyright 2015-2016 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.ftp; + +import javax.validation.constraints.NotNull; + +import org.apache.commons.net.ftp.FTPClient; +import org.hibernate.validator.constraints.Range; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.app.file.remote.AbstractRemoteServerProperties; +import org.springframework.validation.annotation.Validated; + +/** + * FTP {@code SessionFactory} properties. + * + * @author David Turanski + * @author Gary Russell + */ +@ConfigurationProperties("ftp.factory") +@Validated +public class FtpSessionFactoryProperties extends AbstractRemoteServerProperties { + + /** + * The port of the server. + */ + private int port = 21; + + /** + * The client mode to use for the FTP session. + */ + private ClientMode clientMode = ClientMode.PASSIVE; + + @Range(min = 0, max = 65535) + public int getPort() { + return this.port; + } + + public void setPort(int port) { + this.port = port; + } + + @NotNull + public ClientMode getClientMode() { + return this.clientMode; + } + + public void setClientMode(ClientMode clientMode) { + this.clientMode = clientMode; + } + + public static enum ClientMode { + + ACTIVE(FTPClient.ACTIVE_LOCAL_DATA_CONNECTION_MODE), + PASSIVE(FTPClient.PASSIVE_LOCAL_DATA_CONNECTION_MODE); + + private final int mode; + + private ClientMode(int mode) { + this.mode = mode; + } + + public int getMode() { + return mode; + } + + } + +} diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/README.adoc b/applications/apps-core/common/stream-apps-metadata-store-common/README.adoc new file mode 100644 index 00000000..d1a32b96 --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/README.adoc @@ -0,0 +1,181 @@ +=== `MetadataStore` Common Module + +This artifact contains a Spring Boot auto-configuration for the `MetadataStore`which can be used in various Spring Integration scenarios, like file polling, idempotent receiver, offset management etc. +See Spring Integration "`https://docs.spring.io/spring-integration/docs/5.0.6.RELEASE/reference/html/system-management-chapter.html#metadata-store[Reference Manual]`" for more information. + +In addition to the standard Spring Boot configuration properties this module exposes a `MetadataStoreProperties` with the `metadata.store` prefix. + +To auto-configure particular `MetadataStore` you just need to bring respective dependencies into the target app starter: + +==== Redis + +The `RedisMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-redis[Spring Data Redis] and minimal set of dependencies is like this: + +[source,xml] +---- + + org.springframework.integration + spring-integration-redis + + + org.springframework.boot + spring-boot-starter-data-redis + +---- + +Additional configuration property for `RedisMetadataStore` is: + +$$metadata.store.redis.key$$:: $$Redis key for metadata.$$ *($$String$$, default: `$$MetaData$$`)* + +==== MongoDb + +The `MongoDbMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-mongodb[Spring Data MongoDB] and minimal set of dependencies is like this: + +[source,xml] +---- + + org.springframework.integration + spring-integration-mongodb + + + org.springframework.boot + spring-boot-starter-data-mongodb + +---- + +Additional configuration property for `MongoDbMetadataStore` is: + +$$metadata.store.mongo-db.collection$$:: $$MongoDB collection name for metadata.$$ *($$String$$, default: `$$metadataStore$$`)* + +==== Pivotal Gemfire / Apache Geode + +The `GemfireMetadataStore` requires these dependencies for auto-configuration: + +[source,xml] +---- + + org.springframework.integration + spring-integration-gemfire + +---- + +or when your environment is based on the Open Source https://geode.apache.org/[Apache Geode]: + +[source,xml] +---- + + org.springframework.integration + spring-integration-gemfire + + + org.springframework.data + spring-data-gemfire + + + + + org.springframework.data + spring-data-geode + +---- + +You also can consider to use https://github.com/spring-projects/spring-boot-data-geode[Spring Boot Data Geode] instead for automatic dependency management and proper Spring Boot auto-configuration for Pivotal Gemfire/Apache Geode. + +Additional configuration property for `GemfireMetadataStore` is: + +$$metadata.store.gemfire.region$$:: $$Gemfire region name for metadata.$$ *($$String$$, default: `$$MetaData$$`)* + +In addition, for the `GemfireMetadataStore`, a `MetadataStoreListener` bean can be configured in the application context to react to the `MetadataStore` events. + +A default auto-configured `ClientRegionFactoryBean`, based on the auto-configured `GemFireCache`, bean can be overridden in the target application. + +==== Hazelcast + +The `HazelcastMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-caching-provider-hazelcast[Hazelcast] and minimal set of dependencies is like this: + +[source,xml] +---- + + org.springframework.integration + spring-integration-hazelcast + +---- + +There are no additional configuration properties for the `HazelcastMetadataStore`, however a `MetadataStoreListener` bean can be configured in the application context to react to the `MetadataStore` events. + +==== Zookeeper + +The `ZookeeperMetadataStore` requires this dependency for auto-configuration: + +[source,xml] +---- + + org.springframework.integration + spring-integration-zookeeper + +---- + +The configuration properties for `ZookeeperMetadataStore` are: + +$$metadata.store.zookeeper.connect-string$$:: $$Zookeeper connect string in form HOST:PORT.$$ *($$String$$, default: `$$127.0.0.1:2181$$`)* +$$metadata.store.zookeeper.retry-interval$$:: $$Retry interval for Zookeeper operations in milliseconds.$$ *($$int$$, default: `$$1000$$`)* +$$metadata.store.zookeeper.encoding$$:: $$Encoding to use when storing data in Zookeeper.$$ *($$Charset$$, default: `$$UTF-8$$`)* +$$metadata.store.zookeeper.root$$:: $$Root node - store entries are children of this node.$$ *($$String$$, default: `$$/SpringIntegration-MetadataStore$$`)* + +In addition, for the `ZookeeperMetadataStore`, a `MetadataStoreListener` bean can be configured in the application context to react to the `MetadataStore` events. +Also a `CuratorFramework` bean can be provided to override a default auto-configured one. + +==== AWS DymanoDb + +The `DynamoDbMetadataStore` requires regular Spring Cloud AWS auto-configuration for https://cloud.spring.io/spring-cloud-static/spring-cloud-aws/2.0.0.RELEASE/single/spring-cloud-aws.html#_spring_boot_auto_configuration[Spring Boot] and minimal set of dependencies is like this: + +[source,xml] +---- + + org.springframework.integration + spring-integration-aws + + + com.amazonaws + aws-java-sdk-dynamodb + +---- + +Additional configuration properties for `DynamoDbMetadataStore` are: + +$$metadata.store.dynamo-db.table:: $$Table name for metadata.$$ *($$String$$, default: `$$SpringIntegrationMetadataStore$$`)* +$$metadata.store.dynamo-db.read-capacity:: $$Read capacity on the table.$$ *($$long$$, default: `$$1$$`)* +$$metadata.store.dynamo-db.write-capacity:: $$Write capacity on the table.$$ *($$long$$, default: `$$1$$`)* +$$metadata.store.dynamo-db.create-delay:: $$Delay between create table retries.$$ *($$int$$, default: `$$1$$`)* +$$metadata.store.dynamo-db.create-retries:: $$Retry number for create table request.$$ *($$int$$, default: `$$25$$`)* +$$metadata.store.dynamo-db.time-to-live:: $$TTL for table entries.$$ *($$Integer$$, default: `$$$$`)* + +A default, auto-configured `AmazonDynamoDBAsync` bean can be overridden in the target application. + +==== JDBC + +The `JdbcMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-sql[JDBC DataSource] and minimal set of dependencies is like this: + +[source,xml] +---- + + org.springframework.integration + spring-integration-jdbc + + + org.springframework.boot + spring-boot-starter-jdbc + +---- + +Plus vendor-specific JDBC driver artifact(s). + +Additional configuration properties for `JdbcMetadataStore` are: + +$$metadata.store.jdbc.table-prefix:: $$Prefix for the custom table name.$$ *($$String$$, default: `$$INT_$$`)* +$$metadata.store.jdbc.region:: $$Unique grouping identifier for messages persisted with this store.$$ *($$String$$, default: `$$DEFAULT$$`)* + + + +When no any of those technologies dependencies are preset, an in-memory `SimpleMetadataStore` is auto-configured. +The target application can also provide its own `MetadataStore` bean to override any auto-configuration hooks. diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/pom.xml b/applications/apps-core/common/stream-apps-metadata-store-common/pom.xml new file mode 100644 index 00000000..37733f6a --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/pom.xml @@ -0,0 +1,152 @@ + + + 4.0.0 + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + + stream-apps-metadata-store-common + stream-apps-metadata-store-common + + + 1.11.439 + 2.0.0.RELEASE + 1.0.0.RELEASE + 4.0.1 + + + + + + org.springframework.integration + spring-integration-core + + + + + org.springframework.integration + spring-integration-redis + true + + + org.springframework.boot + spring-boot-starter-data-redis + true + + + + + org.springframework.integration + spring-integration-mongodb + true + + + org.springframework.boot + spring-boot-starter-data-mongodb + true + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + + + org.springframework.boot + spring-boot-starter-logging + true + + + org.apache.logging.log4j + log4j-to-slf4j + + + + + org.springframework.integration + spring-integration-gemfire + true + + + org.springframework.data + spring-data-gemfire + + + + + + org.springframework.data + spring-data-geode + true + + + + + + org.springframework.integration + spring-integration-jdbc + true + + + org.springframework.boot + spring-boot-starter-jdbc + true + + + org.hsqldb + hsqldb + test + + + + + org.springframework.integration + spring-integration-zookeeper + true + + + org.apache.curator + curator-test + ${curator.version} + test + + + + + org.springframework.integration + spring-integration-hazelcast + ${spring-integration-hazelcast.version} + true + + + + + org.springframework.integration + spring-integration-aws + ${spring-integration-aws.version} + true + + + com.amazonaws + aws-java-sdk-dynamodb + ${aws-java-sdk.version} + true + + + org.springframework.integration + spring-integration-test + test + + + + diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/ClientCacheAutoConfiguration.java b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/ClientCacheAutoConfiguration.java new file mode 100644 index 00000000..72831fc1 --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/ClientCacheAutoConfiguration.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.metadata; + +import org.apache.geode.cache.GemFireCache; +import org.apache.geode.cache.client.ClientCache; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.gemfire.client.ClientCacheFactoryBean; +import org.springframework.data.gemfire.config.annotation.ClientCacheApplication; +import org.springframework.data.gemfire.config.annotation.EnablePdx; + +/** + * TODO + * + * This is the copy of {@code org.springframework.boot.data.geode.autoconfigure.ClientCacheAutoConfiguration} + * until {@code geode-spring-boot-starter} is released. + * + * @author John Blum + */ +@Configuration +@ConditionalOnClass({ ClientCacheFactoryBean.class, ClientCache.class }) +@ConditionalOnMissingBean(GemFireCache.class) +@ClientCacheApplication +@EnablePdx +public class ClientCacheAutoConfiguration { + +} diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfiguration.java b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfiguration.java new file mode 100644 index 00000000..8a29af6a --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfiguration.java @@ -0,0 +1,234 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.metadata; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryForever; +import org.apache.geode.cache.GemFireCache; +import org.apache.geode.cache.Region; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.aws.core.region.RegionProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.gemfire.client.ClientRegionFactoryBean; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; +import org.springframework.integration.gemfire.metadata.GemfireMetadataStore; +import org.springframework.integration.hazelcast.metadata.HazelcastMetadataStore; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.integration.metadata.MetadataStoreListener; +import org.springframework.integration.metadata.SimpleMetadataStore; +import org.springframework.integration.mongodb.metadata.MongoDbMetadataStore; +import org.springframework.integration.redis.metadata.RedisMetadataStore; +import org.springframework.integration.zookeeper.metadata.ZookeeperMetadataStore; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder; +import com.hazelcast.core.HazelcastInstance; + +/** + * @author Artem Bilan + * + * @since 2.0.2 + */ +@Configuration +@ConditionalOnClass(ConcurrentMetadataStore.class) +@EnableConfigurationProperties(MetadataStoreProperties.class) +public class MetadataStoreAutoConfiguration { + + @ConditionalOnClass(RedisMetadataStore.class) + @ConditionalOnBean(RedisTemplate.class) + static class Redis { + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore redisMetadataStore(RedisTemplate redisTemplate, + MetadataStoreProperties metadataStoreProperties) { + + return new RedisMetadataStore(redisTemplate, metadataStoreProperties.getRedis().getKey()); + } + + } + + @ConditionalOnClass(MongoDbMetadataStore.class) + @ConditionalOnBean(MongoTemplate.class) + static class Mongo { + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore mongoDbMetadataStore(MongoTemplate mongoTemplate, + MetadataStoreProperties metadataStoreProperties) { + + return new MongoDbMetadataStore(mongoTemplate, metadataStoreProperties.getMongoDb().getCollection()); + } + + } + + @ConditionalOnClass(GemfireMetadataStore.class) + @Import(ClientCacheAutoConfiguration.class) + static class Gemfire { + + @Bean + @ConditionalOnMissingBean + public ClientRegionFactoryBean gemfireRegion(GemFireCache cache, + MetadataStoreProperties metadataStoreProperties) { + + ClientRegionFactoryBean clientRegionFactoryBean = new ClientRegionFactoryBean<>(); + clientRegionFactoryBean.setCache(cache); + clientRegionFactoryBean.setName(metadataStoreProperties.getGemfire().getRegion()); + return clientRegionFactoryBean; + } + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore gemfireMetadataStore(Region region, + ObjectProvider metadataStoreListenerObjectProvider) { + + @SuppressWarnings("unchecked") + GemfireMetadataStore gemfireMetadataStore = new GemfireMetadataStore((Region) region); + metadataStoreListenerObjectProvider.ifAvailable(gemfireMetadataStore::addListener); + + return gemfireMetadataStore; + } + + } + + @ConditionalOnClass(HazelcastMetadataStore.class) + static class Hazelcast { + + @Bean + @ConditionalOnMissingBean + public HazelcastInstance hazelcastInstance() { + return com.hazelcast.core.Hazelcast.newHazelcastInstance(); + } + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore hazelcastMetadataStore(HazelcastInstance hazelcastInstance, + ObjectProvider metadataStoreListenerObjectProvider) { + + HazelcastMetadataStore hazelcastMetadataStore = new HazelcastMetadataStore(hazelcastInstance); + metadataStoreListenerObjectProvider.ifAvailable(hazelcastMetadataStore::addListener); + return hazelcastMetadataStore; + } + + } + + @ConditionalOnClass({ ZookeeperMetadataStore.class, CuratorFramework.class }) + static class Zookeeper { + + @Bean(initMethod = "start") + @ConditionalOnMissingBean + public CuratorFramework curatorFramework(MetadataStoreProperties metadataStoreProperties) { + MetadataStoreProperties.Zookeeper zookeeperProperties = metadataStoreProperties.getZookeeper(); + return CuratorFrameworkFactory.newClient(zookeeperProperties.getConnectString(), + new RetryForever(zookeeperProperties.getRetryInterval())); + } + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore zookeeperMetadataStore(CuratorFramework curatorFramework, + MetadataStoreProperties metadataStoreProperties, + ObjectProvider metadataStoreListenerObjectProvider) { + + MetadataStoreProperties.Zookeeper zookeeperProperties = metadataStoreProperties.getZookeeper(); + ZookeeperMetadataStore zookeeperMetadataStore = new ZookeeperMetadataStore(curatorFramework); + zookeeperMetadataStore.setEncoding(zookeeperProperties.getEncoding().name()); + zookeeperMetadataStore.setRoot(zookeeperProperties.getRoot()); + metadataStoreListenerObjectProvider.ifAvailable(zookeeperMetadataStore::addListener); + return zookeeperMetadataStore; + } + + } + + @ConditionalOnClass(DynamoDbMetadataStore.class) + @ConditionalOnBean({ AWSCredentialsProvider.class, RegionProvider.class }) + static class DynamoDb { + + @Bean + @ConditionalOnMissingBean + public AmazonDynamoDBAsync dynamoDB(AWSCredentialsProvider awsCredentialsProvider, + RegionProvider regionProvider) { + + return AmazonDynamoDBAsyncClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withRegion( + regionProvider.getRegion() + .getName()) + .build(); + } + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore dynamoDbMetadataStore(AmazonDynamoDBAsync dynamoDB, + MetadataStoreProperties metadataStoreProperties) { + + MetadataStoreProperties.DynamoDb dynamoDbProperties = metadataStoreProperties.getDynamoDb(); + + DynamoDbMetadataStore dynamoDbMetadataStore = + new DynamoDbMetadataStore(dynamoDB, dynamoDbProperties.getTable()); + + dynamoDbMetadataStore.setReadCapacity(dynamoDbProperties.getReadCapacity()); + dynamoDbMetadataStore.setWriteCapacity(dynamoDbProperties.getWriteCapacity()); + dynamoDbMetadataStore.setCreateTableDelay(dynamoDbProperties.getCreateDelay()); + dynamoDbMetadataStore.setCreateTableRetries(dynamoDbProperties.getCreateRetries()); + if (dynamoDbProperties.getTimeToLive() != null) { + dynamoDbMetadataStore.setTimeToLive(dynamoDbProperties.getTimeToLive()); + } + + return dynamoDbMetadataStore; + } + + } + + @ConditionalOnClass(JdbcMetadataStore.class) + @ConditionalOnBean(JdbcTemplate.class) + static class Jdbc { + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore jdbcMetadataStore(JdbcTemplate jdbcTemplate, + MetadataStoreProperties metadataStoreProperties) { + + MetadataStoreProperties.Jdbc jdbcProperties = metadataStoreProperties.getJdbc(); + + JdbcMetadataStore jdbcMetadataStore = new JdbcMetadataStore(jdbcTemplate); + jdbcMetadataStore.setTablePrefix(jdbcProperties.getTablePrefix()); + jdbcMetadataStore.setRegion(jdbcProperties.getRegion()); + + return jdbcMetadataStore; + } + + } + + @Bean + @ConditionalOnMissingBean + public ConcurrentMetadataStore simpleMetadataStore() { + return new SimpleMetadataStore(); + } + +} diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreProperties.java b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreProperties.java new file mode 100644 index 00000000..263ff4a4 --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/java/org/springframework/cloud/stream/app/metadata/MetadataStoreProperties.java @@ -0,0 +1,290 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.metadata; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; +import org.springframework.integration.gemfire.metadata.GemfireMetadataStore; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.redis.metadata.RedisMetadataStore; + +/** + * @author Artem Bilan + * + * @since 2.0.2 + */ +@ConfigurationProperties("metadata.store") +public class MetadataStoreProperties { + + private final Mongo mongoDb = new Mongo(); + + private final Gemfire gemfire = new Gemfire(); + + private final Redis redis = new Redis(); + + private final DynamoDb dynamoDb = new DynamoDb(); + + private final Jdbc jdbc = new Jdbc(); + + private final Zookeeper zookeeper = new Zookeeper(); + + public Mongo getMongoDb() { + return this.mongoDb; + } + + public Gemfire getGemfire() { + return this.gemfire; + } + + public Redis getRedis() { + return this.redis; + } + + public DynamoDb getDynamoDb() { + return this.dynamoDb; + } + + public Jdbc getJdbc() { + return this.jdbc; + } + + public Zookeeper getZookeeper() { + return this.zookeeper; + } + + public static class Mongo { + + /** + * MongoDB collection name for metadata. + */ + private String collection = "metadataStore"; + + public String getCollection() { + return this.collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + } + + public static class Gemfire { + + /** + * Gemfire region name for metadata. + */ + private String region = GemfireMetadataStore.KEY; + + public String getRegion() { + return this.region; + } + + public void setRegion(String region) { + this.region = region; + } + + } + + public static class Redis { + + /** + * Redis key for metadata. + */ + private String key = RedisMetadataStore.KEY; + + public String getKey() { + return this.key; + } + + public void setKey(String key) { + this.key = key; + } + + } + + public static class DynamoDb { + + /** + * Table name for metadata. + */ + private String table = DynamoDbMetadataStore.DEFAULT_TABLE_NAME; + + /** + * Read capacity on the table. + */ + private long readCapacity = 1L; + + /** + * Write capacity on the table. + */ + private long writeCapacity = 1L; + + /** + * Delay between create table retries. + */ + private int createDelay = 1; + + /** + * Retry number for create table request. + */ + private int createRetries = 25; + + /** + * TTL for table entries. + */ + private Integer timeToLive; + + public String getTable() { + return this.table; + } + + public void setTable(String table) { + this.table = table; + } + + public long getReadCapacity() { + return this.readCapacity; + } + + public void setReadCapacity(long readCapacity) { + this.readCapacity = readCapacity; + } + + public long getWriteCapacity() { + return this.writeCapacity; + } + + public void setWriteCapacity(long writeCapacity) { + this.writeCapacity = writeCapacity; + } + + public int getCreateDelay() { + return this.createDelay; + } + + public void setCreateDelay(int createDelay) { + this.createDelay = createDelay; + } + + public int getCreateRetries() { + return this.createRetries; + } + + public void setCreateRetries(int createRetries) { + this.createRetries = createRetries; + } + + public Integer getTimeToLive() { + return this.timeToLive; + } + + public void setTimeToLive(Integer timeToLive) { + this.timeToLive = timeToLive; + } + + } + + public static class Jdbc { + + /** + * Prefix for the custom table name. + */ + private String tablePrefix = JdbcMetadataStore.DEFAULT_TABLE_PREFIX; + + /** + * Unique grouping identifier for messages persisted with this store. + */ + private String region = "DEFAULT"; + + public String getTablePrefix() { + return this.tablePrefix; + } + + public void setTablePrefix(String tablePrefix) { + this.tablePrefix = tablePrefix; + } + + public String getRegion() { + return this.region; + } + + public void setRegion(String region) { + this.region = region; + } + + } + + public static class Zookeeper { + + /** + * Zookeeper connect string in form HOST:PORT. + */ + private String connectString = "127.0.0.1:2181"; + + /** + * Retry interval for Zookeeper operations in milliseconds. + */ + private int retryInterval = 1000; + + /** + * Encoding to use when storing data in Zookeeper. + */ + private Charset encoding = StandardCharsets.UTF_8; + + /** + * Root node - store entries are children of this node. + */ + private String root = "/SpringIntegration-MetadataStore"; + + public String getConnectString() { + return connectString; + } + + public void setConnectString(String connectString) { + this.connectString = connectString; + } + + public int getRetryInterval() { + return this.retryInterval; + } + + public void setRetryInterval(int retryInterval) { + this.retryInterval = retryInterval; + } + + public Charset getEncoding() { + return this.encoding; + } + + public void setEncoding(Charset encoding) { + this.encoding = encoding; + } + + public String getRoot() { + return this.root; + } + + public void setRoot(String root) { + this.root = root; + } + + } + +} diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/src/main/resources/META-INF/spring.factories b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..adb3df0d --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.springframework.cloud.stream.app.metadata.MetadataStoreAutoConfiguration diff --git a/applications/apps-core/common/stream-apps-metadata-store-common/src/test/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfigurationTests.java b/applications/apps-core/common/stream-apps-metadata-store-common/src/test/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfigurationTests.java new file mode 100644 index 00000000..4d888c0e --- /dev/null +++ b/applications/apps-core/common/stream-apps-metadata-store-common/src/test/java/org/springframework/cloud/stream/app/metadata/MetadataStoreAutoConfigurationTests.java @@ -0,0 +1,158 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; + +import java.beans.Introspector; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.function.Predicate; + +import org.apache.curator.test.TestingServer; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.aws.core.region.RegionProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; +import org.springframework.integration.gemfire.metadata.GemfireMetadataStore; +import org.springframework.integration.hazelcast.metadata.HazelcastMetadataStore; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.integration.metadata.MetadataStore; +import org.springframework.integration.metadata.SimpleMetadataStore; +import org.springframework.integration.mongodb.metadata.MongoDbMetadataStore; +import org.springframework.integration.redis.metadata.RedisMetadataStore; +import org.springframework.integration.zookeeper.metadata.ZookeeperMetadataStore; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync; +import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest; +import com.amazonaws.services.dynamodbv2.model.DescribeTableResult; + +/** + * @author Artem Bilan + * + * @since 2.0.2 + */ +@RunWith(Parameterized.class) +@Ignore +public class MetadataStoreAutoConfigurationTests { + + private final static List> METADATA_STORE_CLASSES = + Arrays.asList( + RedisMetadataStore.class, + MongoDbMetadataStore.class, + GemfireMetadataStore.class, + JdbcMetadataStore.class, + ZookeeperMetadataStore.class, + HazelcastMetadataStore.class, + DynamoDbMetadataStore.class, + SimpleMetadataStore.class + ); + + private static FilteredClassLoader filteredClassLoaderBut(Class classToInclude) { + return new FilteredClassLoader( + METADATA_STORE_CLASSES.stream() + .filter(Predicate.isEqual(classToInclude).negate()) + .toArray(Class[]::new)); + } + + private final ApplicationContextRunner contextRunner; + + private final Class classToInclude; + + + public MetadataStoreAutoConfigurationTests(Class classToInclude) { + this.classToInclude = classToInclude; + this.contextRunner = + new ApplicationContextRunner() + .withUserConfiguration(TestConfiguration.class) + .withClassLoader(filteredClassLoaderBut(classToInclude)); + } + + @Parameterized.Parameters + public static Iterable parameters() { + return METADATA_STORE_CLASSES; + } + + @Test + public void testMetadataStore() { + this.contextRunner + .run(context -> { + assertThat(context.getBeansOfType(MetadataStore.class)).hasSize(1); + + assertThat(context.getBeanNamesForType(this.classToInclude)) + .containsOnlyOnce(Introspector.decapitalize(this.classToInclude.getSimpleName())); + }); + } + + @Configuration + @EnableAutoConfiguration + public static class TestConfiguration { + + @Bean(destroyMethod = "stop") + @ConditionalOnClass(ZookeeperMetadataStore.class) + public static TestingServer zookeeperTestingServer() throws Exception { + TestingServer testingServer = new TestingServer(true); + + System.setProperty("metadata.store.zookeeper.connect-string", testingServer.getConnectString()); + System.setProperty("metadata.store.zookeeper.encoding", StandardCharsets.US_ASCII.name()); + + return testingServer; + } + + @Configuration + @ConditionalOnClass(DynamoDbMetadataStore.class) + protected static class DynamoDbMockConfig { + + @Bean + public static AmazonDynamoDBAsync dynamoDB() { + AmazonDynamoDBAsync dynamoDb = mock(AmazonDynamoDBAsync.class); + willReturn(new DescribeTableResult()) + .given(dynamoDb) + .describeTable(any(DescribeTableRequest.class)); + + return dynamoDb; + } + + @Bean + public static AWSCredentialsProvider awsCredentialsProvider() { + return mock(AWSCredentialsProvider.class); + } + + @Bean + public static RegionProvider regionProvider() { + return mock(RegionProvider.class); + } + + } + + } + +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/pom.xml b/applications/apps-core/common/stream-apps-micrometer-common/pom.xml new file mode 100644 index 00000000..99e9d9ea --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/pom.xml @@ -0,0 +1,35 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + stream-apps-micrometer-common + + + + io.micrometer + micrometer-core + + + io.micrometer + micrometer-registry-influx + test + + + io.pivotal.cfenv + java-cfenv-test-support + test + 2.1.1.RELEASE + + + org.springframework.boot + spring-boot-starter-actuator + + + + diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTags.java b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTags.java new file mode 100644 index 00000000..f4c2eec8 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTags.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import io.micrometer.core.instrument.MeterRegistry; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +/** + * Micrometer common tags for Cloud Foundry deployment properties. Based on the CF application environment variables: + * https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html + * + * Tags are set only if the "cloud" Spring profile is set. The "cloud" profile is activated automatically when an + * application is deployed in CF: https://docs.cloudfoundry.org/buildpacks/java/configuring-service-connections/spring-service-bindings.html#cloud-profiles + * + * Use the spring.cloud.stream.app.metrics.cf.tags.enabled=false property to disable inserting those tags. + * + * @author Christian Tzolov + */ +@Configuration +@Profile("cloud") +@ConditionalOnProperty(name = "spring.cloud.stream.app.metrics.cf.tags.enabled", havingValue = "true", matchIfMissing = true) +public class CloudFoundryMicrometerCommonTags { + + @Value("${vcap.application.org_name:default}") + private String organizationName; + + @Value("${vcap.application.space_id:unknown}") + private String spaceId; + + @Value("${vcap.application.space_name:unknown}") + private String spaceName; + + @Value("${vcap.application.application_name:unknown}") + private String applicationName; + + @Value("${vcap.application.application_id:unknown}") + private String applicationId; + + @Value("${vcap.application.application_version:unknown}") + private String applicationVersion; + + @Value("${vcap.application.instance_index:0}") + private String instanceIndex; + + @Bean + public MeterRegistryCustomizer cloudFoundryMetricsCommonTags() { + return registry -> registry.config() + .commonTags("cf.org.name", organizationName) + .commonTags("cf.space.id", spaceId) + .commonTags("cf.space.name", spaceName) + .commonTags("cf.app.id", applicationId) + .commonTags("cf.app.name", applicationName) + .commonTags("cf.app.version", applicationVersion) + .commonTags("cf.instance.index", instanceIndex); + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTags.java b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTags.java new file mode 100644 index 00000000..c2595fbe --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTags.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.config.MeterFilter; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Auto configuration extends the micrometer metrics with additional tags such as: stream name, application name, + * instance index and guids. Later are necessary to allow discrimination and aggregation of app metrics by external + * metrics collection and visualizaiton tools. + * + * Use the spring.cloud.stream.app.metrics.common.tags.enabled=false property to disable inserting those tags. + * + * @author Christian Tzolov + */ +@Configuration +@ConditionalOnProperty(name = "spring.cloud.stream.app.metrics.common.tags.enabled", havingValue = "true", matchIfMissing = true) +public class SpringCloudStreamMicrometerCommonTags { + + @Value("${spring.cloud.dataflow.stream.name:unknown}") + private String streamName; + + @Value("${spring.cloud.dataflow.stream.app.label:unknown}") + private String applicationName; + + @Value("${spring.cloud.stream.instanceIndex:0}") + private String instanceIndex; + + @Value("${spring.cloud.application.guid:unknown}") + private String applicationGuid; + + @Value("${spring.cloud.dataflow.stream.app.type:unknown}") + private String applicationType; + + @Bean + public MeterRegistryCustomizer metricsCommonTags() { + return registry -> registry.config() + .commonTags("stream.name", streamName) + .commonTags("application.name", applicationName) + .commonTags("application.type", applicationType) + .commonTags("instance.index", instanceIndex) + .commonTags("application.guid", applicationGuid); + } + + @Bean + public MeterRegistryCustomizer renameNameTag() { + return registry -> { + if (registry.getClass().getCanonicalName().contains("AtlasMeterRegistry")) { + registry.config().meterFilter(MeterFilter.renameTag("spring.integration", "name", "aname")); + } + if (registry.getClass().getCanonicalName().contains("InfluxMeterRegistry")) { + registry.config().meterFilter(MeterFilter.replaceTagValues("application.name", + tagValue -> ("time".equalsIgnoreCase(tagValue)) ? "atime" : tagValue)); + } + }; + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessor.java b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessor.java new file mode 100644 index 00000000..94c56743 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/main/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessor.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import java.util.Properties; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertiesPropertySource; + +/** + * Disables all Micrometer Repositories added as App Starters dependencies by default. + * That means disabling Datadog, Influx and Prometheus. + * + * @author Christian Tzolov + */ +public class SpringCloudStreamMicrometerEnvironmentPostProcessor implements EnvironmentPostProcessor { + + protected static final String PROPERTY_SOURCE_KEY_NAME = SpringCloudStreamMicrometerEnvironmentPostProcessor.class.getName(); + + private final static String METRICS_PROPERTY_NAME_TEMPLATE = "management.metrics.export.%s.enabled"; + + private final static String[] METRICS_REPOSITORY_NAMES = + new String[] { "datadog", "influx", "prometheus", "prometheus.rsocket" }; + + @Override + public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { + Properties properties = new Properties(); + + if (!environment.containsProperty("management.endpoints.web.exposure.include")) { + properties.setProperty("management.endpoints.web.exposure.include", "prometheus"); + } + + for (String metricsRepositoryName : METRICS_REPOSITORY_NAMES) { + String propertyKey = String.format(METRICS_PROPERTY_NAME_TEMPLATE, metricsRepositoryName); + + // Back off if the property is already set. + if (!environment.containsProperty(propertyKey)) { + properties.setProperty(propertyKey, "false"); + } + } + + // This post-processor is called multiple times but sets the properties only once. + if (!properties.isEmpty()) { + PropertiesPropertySource propertiesPropertySource = + new PropertiesPropertySource(PROPERTY_SOURCE_KEY_NAME, properties); + environment.getPropertySources().addLast(propertiesPropertySource); + } + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/main/resources/META-INF/spring.factories b/applications/apps-core/common/stream-apps-micrometer-common/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..90d57c3e --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/main/resources/META-INF/spring.factories @@ -0,0 +1,6 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.springframework.cloud.stream.app.micrometer.common.SpringCloudStreamMicrometerCommonTags,\ +org.springframework.cloud.stream.app.micrometer.common.CloudFoundryMicrometerCommonTags +org.springframework.boot.env.EnvironmentPostProcessor=\ +org.springframework.cloud.stream.app.micrometer.common.SpringCloudStreamMicrometerEnvironmentPostProcessor + diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/AbstractMicrometerTagTest.java b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/AbstractMicrometerTagTest.java new file mode 100644 index 00000000..c15b1dd6 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/AbstractMicrometerTagTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import java.io.IOException; +import java.nio.charset.Charset; + +import io.micrometer.core.instrument.Clock; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.simple.SimpleConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import io.pivotal.cfenv.test.CfEnvTestUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.metrics.export.simple.SimpleProperties; +import org.springframework.boot.actuate.autoconfigure.metrics.export.simple.SimplePropertiesConfigAdapter; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.StreamUtils; + +import static org.junit.Assert.assertNotNull; + +/** + * @author Christian Tzolov + * @author Soby Chacko + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = AbstractMicrometerTagTest.AutoConfigurationApplication.class) +public class AbstractMicrometerTagTest { + + @Autowired + protected SimpleMeterRegistry simpleMeterRegistry; + + @Autowired + protected ConfigurableApplicationContext context; + + protected Meter meter; + + @Before + public void before() { + assertNotNull(simpleMeterRegistry); + meter = simpleMeterRegistry.find("jvm.memory.committed").meter(); + assertNotNull("The jvm.memory.committed meter mast be present in SpringBoot apps!", meter); + } + + @BeforeClass + public static void setup() throws IOException { + String serviceJson = StreamUtils.copyToString(new DefaultResourceLoader().getResource( + "classpath:/org/springframework/cloud/stream/app/micrometer/common/pcf-scs-info.json") + .getInputStream(), Charset.forName("UTF-8")); + CfEnvTestUtils.mockVcapServicesFromString(serviceJson); + } + + @SpringBootApplication + @EnableConfigurationProperties(SimpleProperties.class) + public static class AutoConfigurationApplication { + + public static void main(String[] args) { + SpringApplication.run(AutoConfigurationApplication.class, args); + } + + @Bean + public SimpleMeterRegistry simpleMeterRegistry(SimpleConfig config, Clock clock) { + return new SimpleMeterRegistry(config, clock); + } + + @Bean + @ConditionalOnMissingBean + public SimpleConfig simpleConfig(SimpleProperties simpleProperties) { + return new SimplePropertiesConfigAdapter(simpleProperties); + } + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTagsTest.java b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTagsTest.java new file mode 100644 index 00000000..f5b610e0 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/CloudFoundryMicrometerCommonTagsTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import org.hamcrest.Matchers; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; + +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author Christian Tzolov + */ +@RunWith(Enclosed.class) +public class CloudFoundryMicrometerCommonTagsTest { + + @ActiveProfiles("cloud") + public static class ActiveCloudProfileDefaultValues extends AbstractMicrometerTagTest { + @Test + public void testDefaultTagValues() { + assertThat(meter.getId().getTag("cf.org.name"), is("default")); + assertThat(meter.getId().getTag("cf.space.id"), is("unknown")); + assertThat(meter.getId().getTag("cf.space.name"), is("unknown")); + assertThat(meter.getId().getTag("cf.app.name"), is("unknown")); + assertThat(meter.getId().getTag("cf.app.id"), is("unknown")); + assertThat(meter.getId().getTag("cf.app.version"), is("unknown")); + assertThat(meter.getId().getTag("cf.instance.index"), is("0")); + } + } + + @TestPropertySource(properties = { + "vcap.application.org_name=PivotalOrg", + "vcap.application.space_id=SpringSpaceId", + "vcap.application.space_name=SpringSpace", + "vcap.application.application_name=App666", + "vcap.application.application_id=666guid", + "vcap.application.application_version=2.0", + "vcap.application.instance_index=123" }) + @ActiveProfiles("cloud") + public static class ActiveCloudProfile extends AbstractMicrometerTagTest { + + @Test + public void testPresetTagValues() { + assertThat(meter.getId().getTag("cf.org.name"), is("PivotalOrg")); + assertThat(meter.getId().getTag("cf.space.id"), is("SpringSpaceId")); + assertThat(meter.getId().getTag("cf.space.name"), is("SpringSpace")); + assertThat(meter.getId().getTag("cf.app.name"), is("App666")); + assertThat(meter.getId().getTag("cf.app.id"), is("666guid")); + assertThat(meter.getId().getTag("cf.app.version"), is("2.0")); + assertThat(meter.getId().getTag("cf.instance.index"), is("123")); + } + } + + @TestPropertySource(properties = { + "vcap.application.org_name=PivotalOrg", + "vcap.application.space_id=SpringSpaceId", + "vcap.application.space_name=SpringSpace", + "vcap.application.application_name=App666", + "vcap.application.application_id=666guid", + "vcap.application.application_version=2.0", + "vcap.application.instance_index=123" }) + public static class InactiveCloudProfile extends AbstractMicrometerTagTest { + + @Test + public void testDisabledTagValues() { + assertThat(meter.getId().getTag("cf.org.name"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.space.id"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.space.name"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.app.name"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.app.id"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.app.version"), is(Matchers.nullValue())); + assertThat(meter.getId().getTag("cf.instance.index"), is(Matchers.nullValue())); + } + } + + @TestPropertySource(properties = { "spring.cloud.stream.app.metrics.cf.tags.enabled=false" }) + @ActiveProfiles("cloud") + public static class ActiveCloudProfileDisabledProperty extends InactiveCloudProfile { + } +} + diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/InfluxReservedKeywordHandlingTest.java b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/InfluxReservedKeywordHandlingTest.java new file mode 100644 index 00000000..c9d06a51 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/InfluxReservedKeywordHandlingTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.influx.InfluxMeterRegistry; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author Christian Tzolov + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = InfluxReservedKeywordHandlingTest.AutoConfigurationApplication.class, + properties = { + "management.metrics.export.influx.enabled=true", + "spring.cloud.dataflow.stream.app.label=time" }) +public class InfluxReservedKeywordHandlingTest { + + @Autowired + protected InfluxMeterRegistry influxMeterRegistry; + + @Test + public void testPresetTagValues() { + assertNotNull(influxMeterRegistry); + Meter meter = influxMeterRegistry.find("jvm.memory.committed").meter(); + assertNotNull("The jvm.memory.committed meter mast be present in SpringBoot apps!", meter); + + assertThat(meter.getId().getTag("application.name"), is("atime")); + } + + @SpringBootApplication + public static class AutoConfigurationApplication { + public static void main(String[] args) { + SpringApplication.run(AutoConfigurationApplication.class, args); + } + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTagsTest.java b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTagsTest.java new file mode 100644 index 00000000..5fef6c98 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerCommonTagsTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; + +import org.springframework.test.context.TestPropertySource; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; + +/** + * @author Christian Tzolov + */ +@RunWith(Enclosed.class) +public class SpringCloudStreamMicrometerCommonTagsTest { + + public static class TestDefaultTagValues extends AbstractMicrometerTagTest { + + @Test + public void testDefaultTagValues() { + assertThat(meter.getId().getTag("stream.name"), is("unknown")); + assertThat(meter.getId().getTag("application.name"), is("unknown")); + assertThat(meter.getId().getTag("instance.index"), is("0")); + assertThat(meter.getId().getTag("application.type"), is("unknown")); + assertThat(meter.getId().getTag("application.guid"), is("unknown")); + } + } + + @TestPropertySource(properties = { + "spring.cloud.dataflow.stream.name=myStream", + "spring.cloud.dataflow.stream.app.label=myApp", + "spring.cloud.stream.instanceIndex=666", + "spring.cloud.application.guid=666guid", + "spring.cloud.dataflow.stream.app.type=source" }) + public static class TestPresetTagValues extends AbstractMicrometerTagTest { + + @Test + public void testPresetTagValues() { + assertThat(meter.getId().getTag("stream.name"), is("myStream")); + assertThat(meter.getId().getTag("application.name"), is("myApp")); + assertThat(meter.getId().getTag("instance.index"), is("666")); + assertThat(meter.getId().getTag("application.type"), is("source")); + assertThat(meter.getId().getTag("application.guid"), is("666guid")); + } + } + + @TestPropertySource(properties = { "spring.cloud.stream.app.metrics.common.tags.enabled=false" }) + public static class TestDisabledTagValues extends AbstractMicrometerTagTest { + + @Test + public void testDefaultTagValues() { + assertThat(meter.getId().getTag("stream.name"), is(nullValue())); + assertThat(meter.getId().getTag("application.name"), is(nullValue())); + assertThat(meter.getId().getTag("instance.index"), is(nullValue())); + assertThat(meter.getId().getTag("application.type"), is(nullValue())); + assertThat(meter.getId().getTag("application.guid"), is(nullValue())); + } + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessorTest.java b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessorTest.java new file mode 100644 index 00000000..da340292 --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/java/org/springframework/cloud/stream/app/micrometer/common/SpringCloudStreamMicrometerEnvironmentPostProcessorTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.micrometer.common; + +import org.hamcrest.core.Is; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; + +import org.springframework.core.env.PropertySource; +import org.springframework.test.context.TestPropertySource; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author Christian Tzolov + */ +@RunWith(Enclosed.class) +public class SpringCloudStreamMicrometerEnvironmentPostProcessorTest { + + public static class TestDefaultMetricsEnabledProperties extends AbstractMicrometerTagTest { + + @Test + public void testDefaultProperties() { + assertNotNull(context); + + PropertySource propertySource = context.getEnvironment().getPropertySources() + .get(SpringCloudStreamMicrometerEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNotNull("Property source " + + SpringCloudStreamMicrometerEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is null", + propertySource); + + assertThat(propertySource.getProperty("management.metrics.export.influx.enabled"), Is.is("false")); + assertThat(propertySource.getProperty("management.metrics.export.prometheus.enabled"), Is.is("false")); + assertThat(propertySource.getProperty("management.metrics.export.prometheus.rsocket.enabled"), Is.is("false")); + assertThat(propertySource.getProperty("management.metrics.export.datadog.enabled"), Is.is("false")); + + assertThat(propertySource.getProperty("management.endpoints.web.exposure.include"), Is.is("prometheus")); + + assertThat(context.getEnvironment().getProperty("management.metrics.export.influx.enabled"), Is.is("false")); + assertThat(context.getEnvironment().getProperty("management.metrics.export.prometheus.enabled"), Is.is("false")); + assertThat(context.getEnvironment().getProperty("management.metrics.export.datadog.enabled"), Is.is("false")); + + assertThat(context.getEnvironment().getProperty("management.endpoints.web.exposure.include"), Is.is("prometheus")); + + } + } + + @TestPropertySource(properties = { + "management.metrics.export.simple.enabled=true", + "management.metrics.export.influx.enabled=true", + "management.metrics.export.prometheus.enabled=true", + "management.metrics.export.prometheus.rsocket.enabled=true", + "management.metrics.export.datadog.enabled=true", + "management.endpoints.web.exposure.include=info,health"}) + public static class TestOverrideMetricsEnabledProperties extends AbstractMicrometerTagTest { + + @Test + public void testOverrideProperties() { + assertNotNull(context); + + PropertySource propertySource = context.getEnvironment().getPropertySources() + .get(SpringCloudStreamMicrometerEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNull("Property source " + + SpringCloudStreamMicrometerEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is not null", + propertySource); + + assertThat(context.getEnvironment().getProperty("management.metrics.export.influx.enabled"), Is.is("true")); + assertThat(context.getEnvironment().getProperty("management.metrics.export.prometheus.enabled"), Is.is("true")); + assertThat(context.getEnvironment().getProperty("management.metrics.export.prometheus.rsocket.enabled"), Is.is("true")); + assertThat(context.getEnvironment().getProperty("management.metrics.export.datadog.enabled"), Is.is("true")); + + assertThat(context.getEnvironment().getProperty("management.endpoints.web.exposure.include"), Is.is("info,health")); + } + } +} diff --git a/applications/apps-core/common/stream-apps-micrometer-common/src/test/resources/org/springframework/cloud/stream/app/micrometer/common/pcf-scs-info.json b/applications/apps-core/common/stream-apps-micrometer-common/src/test/resources/org/springframework/cloud/stream/app/micrometer/common/pcf-scs-info.json new file mode 100644 index 00000000..adede92b --- /dev/null +++ b/applications/apps-core/common/stream-apps-micrometer-common/src/test/resources/org/springframework/cloud/stream/app/micrometer/common/pcf-scs-info.json @@ -0,0 +1,13 @@ +{ + "sso":[{ + "name": "sso", + "label": "sso", + "plan": "notfree", + "tags": ["configuration"], + "credentials":{ + "uri": "https://pivotal.io", + "client_id": "fakeClientId", + "client_secret": "fakeSecret", + "access_token_uri": "token" + } + }]} diff --git a/applications/apps-core/common/stream-apps-postprocessor-common/pom.xml b/applications/apps-core/common/stream-apps-postprocessor-common/pom.xml new file mode 100644 index 00000000..8107c8b5 --- /dev/null +++ b/applications/apps-core/common/stream-apps-postprocessor-common/pom.xml @@ -0,0 +1,22 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-postprocessor-common + stream-apps-postprocessor-common + + + + org.springframework.integration + spring-integration-test + test + + + + diff --git a/applications/apps-core/common/stream-apps-postprocessor-common/src/main/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessor.java b/applications/apps-core/common/stream-apps-postprocessor-common/src/main/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessor.java new file mode 100644 index 00000000..470f6277 --- /dev/null +++ b/applications/apps-core/common/stream-apps-postprocessor-common/src/main/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessor.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.postprocessor; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.cloud.stream.messaging.Sink; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertiesPropertySource; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * An {@link EnvironmentPostProcessor} to set the {@code spring.cloud.stream.bindings.{input,output}.contentType} + * channel properties to a default of {@code application/octet-stream} if it has not been set already. + * + * Subclasses may extend this class to change the default content type and channel name(s). + * + * @author Chris Schaefer + */ +public class ContentTypeEnvironmentPostProcessor implements EnvironmentPostProcessor { + private Map channelMap = createChannelMap(); + + private Map createChannelMap() { + Map channelMap = new HashMap<>(); + channelMap.put(Sink.INPUT, "application/octet-stream"); + channelMap.put(Source.OUTPUT, "application/octet-stream"); + + return channelMap; + } + + protected static final String PROPERTY_SOURCE_KEY_NAME = ContentTypeEnvironmentPostProcessor.class.getName(); + protected static final String CONTENT_TYPE_PROPERTY_PREFIX = "spring.cloud.stream.bindings."; + protected static final String CONTENT_TYPE_PROPERTY_SUFFIX = ".contentType"; + + public ContentTypeEnvironmentPostProcessor() { + super(); + } + + protected ContentTypeEnvironmentPostProcessor(Map channelMap) { + this.channelMap = channelMap; + } + + protected ContentTypeEnvironmentPostProcessor(String contentType) { + for (Map.Entry channel : channelMap.entrySet()) { + channel.setValue(contentType); + } + } + + protected ContentTypeEnvironmentPostProcessor(String channelName, String contentType) { + channelMap.put(channelName, contentType); + } + + @Override + public void postProcessEnvironment(ConfigurableEnvironment configurableEnvironment, SpringApplication springApplication) { + Properties properties = new Properties(); + + for (Map.Entry channel : channelMap.entrySet()) { + String propertyKey = CONTENT_TYPE_PROPERTY_PREFIX + channel.getKey() + CONTENT_TYPE_PROPERTY_SUFFIX; + + if (!configurableEnvironment.containsProperty(propertyKey)) { + properties.setProperty(propertyKey, channel.getValue()); + } + } + + if (!properties.isEmpty()) { + PropertiesPropertySource propertiesPropertySource = + new PropertiesPropertySource(PROPERTY_SOURCE_KEY_NAME, properties); + configurableEnvironment.getPropertySources().addLast(propertiesPropertySource); + } + } +} diff --git a/applications/apps-core/common/stream-apps-postprocessor-common/src/test/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessorTests.java b/applications/apps-core/common/stream-apps-postprocessor-common/src/test/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessorTests.java new file mode 100644 index 00000000..bde024cf --- /dev/null +++ b/applications/apps-core/common/stream-apps-postprocessor-common/src/test/java/org/springframework/cloud/stream/app/postprocessor/ContentTypeEnvironmentPostProcessorTests.java @@ -0,0 +1,227 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.postprocessor; + +import org.junit.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.cloud.stream.messaging.Sink; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test cases for {@ContentTypeEnvironmentPostProcessor}. + * + * @author Chris Schaefer + */ +public class ContentTypeEnvironmentPostProcessorTests { + @Test + public void testPostProcessorDefaults() { + ConfigurableEnvironment configurableEnvironment = getEnvironment(); + + PropertySource propertySource = configurableEnvironment.getPropertySources() + .get(ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNotNull("Property source " + ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is null", + propertySource); + + assertTrue("Unexpected input content type", propertySource.getProperty(getContentTypeProperty(Sink.INPUT)) + .equals("application/octet-stream")); + + assertTrue("Unexpected output content type", propertySource.getProperty(getContentTypeProperty(Source.OUTPUT)) + .equals("application/octet-stream")); + } + + @Test + public void testUserDefinedOutputContentType() { + PropertiesPropertySource testProperties = buildTestProperties(Source.OUTPUT, "text/plain"); + ConfigurableEnvironment configurableEnvironment = getEnvironment(testProperties); + + assertTrue("Output contentType property key not found", + configurableEnvironment.containsProperty(getContentTypeProperty(Source.OUTPUT))); + + assertTrue("Unexpected output content type", configurableEnvironment.getProperty(getContentTypeProperty(Source.OUTPUT)) + .equals("text/plain")); + } + + @Test + public void testUserDefinedInputContentType() { + PropertiesPropertySource testProperties = buildTestProperties(Sink.INPUT, "text/html"); + ConfigurableEnvironment configurableEnvironment = getEnvironment(testProperties); + + assertTrue("Input contentType property key not found", + configurableEnvironment.containsProperty(getContentTypeProperty(Sink.INPUT))); + + assertTrue("Unexpected input content type", configurableEnvironment.getProperty(getContentTypeProperty(Sink.INPUT)) + .equals("text/html")); + } + + @Test + public void testConfigureCustomChannel() { + ConfigurableEnvironment configurableEnvironment = getEnvironment(new CustomChannel()); + + PropertySource propertySource = configurableEnvironment.getPropertySources() + .get(ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNotNull("Property source " + ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is null", + propertySource); + + assertTrue("myChannelName contentType property key not found", + propertySource.containsProperty(getContentTypeProperty("myChannelName"))); + + assertTrue("Unexpected myChannelName content type", propertySource.getProperty(getContentTypeProperty("myChannelName")) + .equals("application/octet-stream")); + } + + public static class CustomChannel extends ContentTypeEnvironmentPostProcessor { + private static final String CHANNEL_NAME = "myChannelName"; + private static final String CONTENT_TYPE = "application/octet-stream"; + + public CustomChannel() { + super(CHANNEL_NAME, CONTENT_TYPE); + } + } + + @Test + public void testConfigureDefaultChannelsCustomContentType() { + ConfigurableEnvironment configurableEnvironment = getEnvironment(new DefaultChannelsCustomContentTypes()); + + PropertySource propertySource = configurableEnvironment.getPropertySources() + .get(ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNotNull("Property source " + ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is null", + propertySource); + + assertTrue("Output contentType property key not found", + propertySource.containsProperty(getContentTypeProperty(Source.OUTPUT))); + + assertTrue("Unexpected output content type", propertySource.getProperty(getContentTypeProperty(Source.OUTPUT)) + .equals("image/jpeg")); + + assertTrue("Input contentType property key not found", + propertySource.containsProperty(getContentTypeProperty(Sink.INPUT))); + + assertTrue("Unexpected input content type", propertySource.getProperty(getContentTypeProperty(Sink.INPUT)) + .equals("image/gif")); + } + + @Test + public void testConfigureDefaultChannelsSameContentType() { + ConfigurableEnvironment configurableEnvironment = getEnvironment(new ChannelSameContentType()); + + PropertySource propertySource = configurableEnvironment.getPropertySources() + .get(ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME); + + assertNotNull("Property source " + ContentTypeEnvironmentPostProcessor.PROPERTY_SOURCE_KEY_NAME + " is null", + propertySource); + + assertTrue("Output contentType property key not found", + propertySource.containsProperty(getContentTypeProperty(Source.OUTPUT))); + + assertTrue("Unexpected output content type", propertySource.getProperty(getContentTypeProperty(Source.OUTPUT)) + .equals("image/jpeg")); + + assertTrue("Input contentType property key not found", + propertySource.containsProperty(getContentTypeProperty(Sink.INPUT))); + + assertTrue("Unexpected input content type", propertySource.getProperty(getContentTypeProperty(Sink.INPUT)) + .equals("image/jpeg")); + } + + public static class ChannelSameContentType extends ContentTypeEnvironmentPostProcessor { + private static final String CONTENT_TYPE = "image/jpeg"; + + public ChannelSameContentType() { + super(CONTENT_TYPE); + } + } + + public static class DefaultChannelsCustomContentTypes extends ContentTypeEnvironmentPostProcessor { + private static Map CHANNEL_MAP = createChannelMap(); + + private static Map createChannelMap() { + Map channelMap = new HashMap<>(); + channelMap.put(Source.OUTPUT, "image/jpeg"); + channelMap.put(Sink.INPUT, "image/gif"); + + return channelMap; + } + + public DefaultChannelsCustomContentTypes() { + super(CHANNEL_MAP); + } + } + + private static String getContentTypeProperty(String channelName) { + return ContentTypeEnvironmentPostProcessor.CONTENT_TYPE_PROPERTY_PREFIX + channelName + + ContentTypeEnvironmentPostProcessor.CONTENT_TYPE_PROPERTY_SUFFIX; + } + + private PropertiesPropertySource buildTestProperties(String channelName, String contentType) { + Properties testProperties = new Properties(); + testProperties.setProperty(getContentTypeProperty(channelName), contentType); + + return new PropertiesPropertySource("test-properties", testProperties); + } + + private ConfigurableEnvironment getEnvironment() { + return getEnvironment(null, null); + } + + private ConfigurableEnvironment getEnvironment(PropertiesPropertySource propertiesPropertySource) { + return getEnvironment(propertiesPropertySource, null); + } + + private ConfigurableEnvironment getEnvironment(EnvironmentPostProcessor environmentPostProcessor) { + return getEnvironment(null, environmentPostProcessor); + } + + private ConfigurableEnvironment getEnvironment(PropertiesPropertySource propertiesPropertySource, + EnvironmentPostProcessor environmentPostProcessor) { + SpringApplication springApplication = new SpringApplicationBuilder() + .sources(ContentTypeEnvironmentPostProcessorTests.class) + .web(WebApplicationType.NONE).build(); + + ConfigurableApplicationContext context = springApplication.run(); + + if (propertiesPropertySource != null) { + context.getEnvironment().getPropertySources().addFirst(propertiesPropertySource); + } + + if (environmentPostProcessor == null) { + environmentPostProcessor = new ContentTypeEnvironmentPostProcessor(); + } + + environmentPostProcessor.postProcessEnvironment(context.getEnvironment(), springApplication); + + ConfigurableEnvironment configurableEnvironment = context.getEnvironment(); + context.close(); + + return configurableEnvironment; + } +} diff --git a/applications/apps-core/common/stream-apps-security-common/README.adoc b/applications/apps-core/common/stream-apps-security-common/README.adoc new file mode 100644 index 00000000..b58001d6 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/README.adoc @@ -0,0 +1,33 @@ +=== `App Starters Security` Common Module + +Spring Boot auto-configuration to manage the web security of the application starters. + +When the `app-starters-security-common` dependency is on the classpath, the `spring.cloud.streamapp.security.enabled` and `spring.cloud.streamapp.security.csrf-enabled` properties control the application security behavior. + +By default the security is enabled allowing unauthorized access only to the `Info` and `Health` endpoints. + +The `spring.cloud.streamapp.security.enabled = false` completely surpass the application security. + +For secured application setting `spring.cloud.streamapp.security.csrf-enabled = false` disables security for the CSRF access. + +With security enabled (`spring.cloud.streamapp.security.enabled = true`) and `actuator` dependency on the classpath, the `(Reactive)ManagementWebSecurityAutoConfiguration` is activated, providing unauthenticated access to the `HealthEndpoint` and `InfoEndpoint`. + +If the user specifies their own `WebSecurityConfigurerAdapter` (for MVC application), this configuration will back-off completely and the user should specify all the bits that they want to configure as part of the custom security configuration. +For reactive (WebFlux) application the same effect can be achieved with a custom `WebFilterChainProxy` bean. + +=== Configuration +To include app starters security management for a stream app, just include a dependency on this module. + +[source,xml] +---- + + org.springframework.cloud.stream.app + app-starters-security-common + +---- + + +All Spring Cloud Stream app starters that inherit form the `core` pom have the `app-starters-security-common` dependency included by default. + +* `spring.cloud.streamapp.security.enabled` (default: `true`). If set to `false` it surpasses the boot security. +* `spring.cloud.streamapp.security.csrf-enabled` (default: `true`). If set to `false`, for secured applications it enables CQRS. diff --git a/applications/apps-core/common/stream-apps-security-common/pom.xml b/applications/apps-core/common/stream-apps-security-common/pom.xml new file mode 100644 index 00000000..1e3a5271 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/pom.xml @@ -0,0 +1,52 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-security-common + + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-autoconfigure-processor + true + + + javax.servlet + javax.servlet-api + provided + + + org.springframework.boot + spring-boot-starter-actuator + true + + + org.springframework.boot + spring-boot-starter-webflux + true + + + org.springframework.boot + spring-boot-starter-web + true + test + + + org.springframework.boot + spring-boot-starter-security + + + + diff --git a/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java new file mode 100644 index 00000000..7d160091 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import org.springframework.boot.actuate.autoconfigure.security.reactive.ReactiveManagementWebSecurityAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; +import org.springframework.boot.autoconfigure.security.reactive.ReactiveSecurityAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; +import org.springframework.security.config.web.server.ServerHttpSecurity; +import org.springframework.security.web.server.SecurityWebFilterChain; +import org.springframework.security.web.server.WebFilterChainProxy; +import org.springframework.web.reactive.config.WebFluxConfigurer; + +import reactor.core.publisher.Flux; + +/** + * @author Artem Bilan + * + * @since 3.0 + */ +@Conditional(OnHttpCsrfOrSecurityDisabled.class) +@Configuration +@ConditionalOnClass({ Flux.class, EnableWebFluxSecurity.class, WebFilterChainProxy.class, WebFluxConfigurer.class }) +@ConditionalOnMissingBean(WebFilterChainProxy.class) +@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.REACTIVE) +@AutoConfigureBefore(value = { ReactiveManagementWebSecurityAutoConfiguration.class, + ReactiveSecurityAutoConfiguration.class }) +@EnableConfigurationProperties(AppStarterWebSecurityAutoConfigurationProperties.class) +public class AppStarterWebFluxSecurityAutoConfiguration { + + @Bean + public SecurityWebFilterChain springSecurityFilterChain(ServerHttpSecurity http, + AppStarterWebSecurityAutoConfigurationProperties securityProperties) { + if (!securityProperties.isCsrfEnabled()) { + http.csrf().disable(); + } + if (!securityProperties.isEnabled()) { + http.authorizeExchange() + .anyExchange() + .permitAll(); + } + return http.build(); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java new file mode 100644 index 00000000..8e451ceb --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.builders.WebSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 2.1 + */ +@Conditional(OnHttpCsrfOrSecurityDisabled.class) +@Configuration +@ConditionalOnClass(WebSecurityConfigurerAdapter.class) +@ConditionalOnMissingBean(WebSecurityConfigurerAdapter.class) +@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) +@AutoConfigureBefore(value = { ManagementWebSecurityAutoConfiguration.class, SecurityAutoConfiguration.class }) +@EnableConfigurationProperties(AppStarterWebSecurityAutoConfigurationProperties.class) +@EnableWebSecurity +public class AppStarterWebSecurityAutoConfiguration { + + @Bean + WebSecurityConfigurerAdapter appStarterWebSecurityConfigurerAdapter( + AppStarterWebSecurityAutoConfigurationProperties securityProperties) { + + + return new WebSecurityConfigurerAdapter() { + + @Override + protected void configure(HttpSecurity http) throws Exception { + super.configure(http); + if (!securityProperties.isCsrfEnabled()) { + http.csrf().disable(); + } + } + + @Override + public void configure(WebSecurity builder) { + if (!securityProperties.isEnabled()) { + builder.ignoring().antMatchers("/**"); + } + } + + }; + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfigurationProperties.java b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfigurationProperties.java new file mode 100644 index 00000000..7ed36d7a --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfigurationProperties.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * {@code AppStarterWebSecurityAutoConfiguration} properties. + * + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 2.1 + */ +@ConfigurationProperties("spring.cloud.streamapp.security") +public class AppStarterWebSecurityAutoConfigurationProperties { + + + /** + * The security enabling flag. + */ + private boolean enabled = true; + + /** + * The security CSRF enabling flag. Makes sense only if security 'enabled` is `true'. + */ + private boolean csrfEnabled = true; + + public boolean isEnabled() { + return this.enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public boolean isCsrfEnabled() { + return this.csrfEnabled; + } + + public void setCsrfEnabled(boolean csrfEnabled) { + this.csrfEnabled = csrfEnabled; + } +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/OnHttpCsrfOrSecurityDisabled.java b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/OnHttpCsrfOrSecurityDisabled.java new file mode 100644 index 00000000..70e0ff22 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/OnHttpCsrfOrSecurityDisabled.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +/** + * An {@link AnyNestedCondition} to enable app starters-specific security auto-configuration + * overriding out-of-the-box one in Spring Boot. + * + * @author Artem Bilan + * + * @since 3.0 + */ +class OnHttpCsrfOrSecurityDisabled extends AnyNestedCondition { + + OnHttpCsrfOrSecurityDisabled() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(name = "spring.cloud.streamapp.security.enabled", havingValue = "false") + static class SecurityDisabled { + } + + @ConditionalOnProperty(name = "spring.cloud.streamapp.security.csrf-enabled", havingValue = "false") + static class HttpCsrfDisabled { + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/main/resources/META-INF/spring.factories b/applications/apps-core/common/stream-apps-security-common/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..81c908f4 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.springframework.cloud.stream.app.security.common.AppStarterWebSecurityAutoConfiguration,\ +org.springframework.cloud.stream.app.security.common.AppStarterWebFluxSecurityAutoConfiguration diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/AbstractSecurityCommonTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/AbstractSecurityCommonTests.java new file mode 100644 index 00000000..c9cd2230 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/AbstractSecurityCommonTests.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Christian Tzolov + * @author Artem Bilan + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@DirtiesContext +public abstract class AbstractSecurityCommonTests { + + @Autowired + protected TestRestTemplate restTemplate; + + @SpringBootApplication + public static class AutoConfigurationApplication { + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityDisabledManagementSecurityEnabledTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityDisabledManagementSecurityEnabledTests.java new file mode 100644 index 00000000..9474e504 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityDisabledManagementSecurityEnabledTests.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=reactive", + "spring.cloud.streamapp.security.enabled=false", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class ReactiveSecurityDisabledManagementSecurityEnabledTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java new file mode 100644 index 00000000..a899752a --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.SecurityProperties; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.support.BasicAuthenticationInterceptor; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=reactive", + "org.springframework.boot.actuate.autoconfigure.security.reactive.ReactiveManagementWebSecurityAutoConfiguration" + + ",org.springframework.cloud.stream.app.security.common.AppStarterWebFluxSecurityAutoConfiguration", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class ReactiveSecurityEnabledManagementSecurityDisabledAuthorizedAccessTests extends AbstractSecurityCommonTests { + + @Autowired + private SecurityProperties securityProperties; + + @BeforeEach + public void before() { + restTemplate.getRestTemplate().getInterceptors().add(new BasicAuthenticationInterceptor( + securityProperties.getUser().getName(), securityProperties.getUser().getPassword())); + } + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java new file mode 100644 index 00000000..df88435b --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=reactive", + "spring.autoconfigure.exclude=" + + "org.springframework.boot.actuate.autoconfigure.security.reactive.ReactiveManagementWebSecurityAutoConfiguration" + + ",org.springframework.cloud.stream.app.security.common.AppStarterWebFluxSecurityAutoConfiguration", + "management.endpoints.web.exposure.include=health,info" }) +public class ReactiveSecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java new file mode 100644 index 00000000..256281a1 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=reactive", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class ReactiveSecurityEnabledManagementSecurityEnabledTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + // The ManagementWebSecurityAutoConfiguration exposes only Info and Health endpoint not Env! + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityDisabledManagementSecurityEnabledTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityDisabledManagementSecurityEnabledTests.java new file mode 100644 index 00000000..673cd8f5 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityDisabledManagementSecurityEnabledTests.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=servlet", + "spring.cloud.streamapp.security.enabled=false", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class SecurityDisabledManagementSecurityEnabledTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java new file mode 100644 index 00000000..affc6bfb --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledAuthorizedAccessTests.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.SecurityProperties; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.support.BasicAuthenticationInterceptor; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=servlet", + "spring.autoconfigure.exclude=org.springframework.boot.actuate.autoconfigure.security.servlet" + + ".ManagementWebSecurityAutoConfiguration" + + ",org.springframework.cloud.stream.app.security.common.AppStarterWebSecurityAutoConfiguration", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class SecurityEnabledManagementSecurityDisabledAuthorizedAccessTests extends AbstractSecurityCommonTests { + + @Autowired + private SecurityProperties securityProperties; + + @BeforeEach + public void before() { + restTemplate.getRestTemplate().getInterceptors().add(new BasicAuthenticationInterceptor( + securityProperties.getUser().getName(), securityProperties.getUser().getPassword())); + } + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java new file mode 100644 index 00000000..7161bd3e --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=servlet", + "spring.autoconfigure.exclude=" + + "org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration" + + ",org.springframework.cloud.stream.app.security.common.AppStarterWebSecurityAutoConfiguration", + "management.endpoints.web.exposure.include=health,info" }) +public class SecurityEnabledManagementSecurityDisabledUnauthorizedAccessTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("Unauthorized", health.get("error")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("Unauthorized", info.get("error")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityEnabledTests.java b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityEnabledTests.java new file mode 100644 index 00000000..155eaaa8 --- /dev/null +++ b/applications/apps-core/common/stream-apps-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/SecurityEnabledManagementSecurityEnabledTests.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.security.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Christian Tzolov + * @author Artem Bilan + * + * @since 3.0 + */ +@TestPropertySource(properties = { + "spring.main.web-application-type=servlet", + "management.endpoints.web.exposure.include=health,info,env", + "info.name=MY TEST APP" }) +public class SecurityEnabledManagementSecurityEnabledTests extends AbstractSecurityCommonTests { + + @Test + @SuppressWarnings("rawtypes") + public void testHealthEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map health = response.getBody(); + assertEquals("UP", health.get("status")); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInfoEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/info", Map.class); + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertTrue(response.hasBody()); + Map info = response.getBody(); + assertEquals("MY TEST APP", info.get("name")); + } + + // The ManagementWebSecurityAutoConfiguration exposes only Info and Health endpoint not Env! + @Test + @SuppressWarnings("rawtypes") + public void testEnvEndpoint() { + ResponseEntity response = this.restTemplate.getForEntity("/actuator/env", Map.class); + assertEquals(HttpStatus.UNAUTHORIZED, response.getStatusCode()); + assertTrue(response.hasBody()); + } + +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/README.adoc b/applications/apps-core/common/stream-apps-task-launch-request-common/README.adoc new file mode 100644 index 00000000..3e026aa9 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/README.adoc @@ -0,0 +1,52 @@ +=== `Data Flow Task Launch Request Support` Common Module + +This artifact contains a Spring Boot auto-configuration providing components to produce a Task Launch Request payload. +In theory any message source can be used to launch a Task. The task launch request is compatible with the +task-launcher-dataflow sink. + + +==== Data Flow Task Launch Request + +Data Flow Task Launch Request requests require a Data Flow server with an existing task definition. +The task launch request contains the name of the task to launch. This must the name of a defined task in Data Flow. +You may optionally provide command line arguments and deployment properties. + +===== Task Name + +The task name is a required field. This may be statically configured by setting `task.launch.request.task-name`, +extracted from the Message by setting `task.launch.request.task-name-expression`. +You may also override the default implementation of the `TaskNameMessageMapper` bean to enable more complex runtime task name mappings. + + +===== Task Command Line Arguments + +The launched task often requires additional data which may be passed as command line arguments. +The `task.launch.request.args` property accepts a comma delimited string of key-value pairs, for example +`key1=val1,key2=val2`. In addition, a `task.launch.request.arg-expressions` allows you to use SpEL expressions to evaluate +message contents to provide command line arguments. +For example, `task.launch.request.arg-expressions=foo=payload.toUpperCase(),bar=payload.substring(0,2)`. + +You may also provide override implementation of the `CommandLineArgumentsMessageMapper` bean to implement more complex logic. + +===== Task Deployment Properties + +Deployment properties are platform-specific configuration used by the `TaskLauncher` and are always statically configured by +setting `task.launch.request.deployment-properties` and apply to every task launch request. + +=== Configuration +To enable any stream app to transform its output to a Task Launch Request, include a dependency on this module + +[source,xml] +---- + + org.springframework.cloud.stream.app + app-starters-task-launch-request-common + +---- + +Setting the application property `spring.cloud.stream.function.definition=taskLaunchRequest` is required to execute the transformation. +You may safely add the above dependency to applications which optionally produce a task launch request. + + +`TaskLaunchRequestIntegrationTests` provides some configuration examples. + diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/pom.xml b/applications/apps-core/common/stream-apps-task-launch-request-common/pom.xml new file mode 100644 index 00000000..4e785243 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/pom.xml @@ -0,0 +1,45 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-task-launch-request-common + stream-apps-task-launch-request-common + + + + org.springframework.cloud.stream.app + stream-apps-file-common + ${project.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream + test-jar + test + test-binder + + + org.springframework.cloud + spring-cloud-stream + + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + + diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequest.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequest.java new file mode 100644 index 00000000..199106c7 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DataFlowTaskLaunchRequest { + @JsonProperty("args") + private List commandlineArguments = new ArrayList<>(); + + @JsonProperty("deploymentProps") + private Map deploymentProperties = new HashMap<>(); + + @JsonProperty("name") + private String taskName; + + public void setCommandlineArguments(List commandlineArguments) { + this.commandlineArguments = new ArrayList<>(commandlineArguments); + } + + public List getCommandlineArguments() { + return this.commandlineArguments; + } + + public void setDeploymentProperties(Map deploymentProperties) { + this.deploymentProperties = deploymentProperties; + } + + public Map getDeploymentProperties() { + return this.deploymentProperties; + } + + public void setTaskName(String taskName) { + this.taskName = taskName; + } + + public String getTaskName() { + return this.taskName; + } + + public DataFlowTaskLaunchRequest addCommmandLineArguments(Collection args) { + this.commandlineArguments.addAll(args); + return this; + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequestAutoConfiguration.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequestAutoConfiguration.java new file mode 100644 index 00000000..06b72528 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataFlowTaskLaunchRequestAutoConfiguration.java @@ -0,0 +1,162 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.CommandLineArgumentsMessageMapper; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskLaunchRequestSupplier; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskNameMessageMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.messaging.Message; +import org.springframework.util.StringUtils; + + +/** + * @author David Turanski + **/ +@Configuration +@EnableConfigurationProperties(DataflowTaskLaunchRequestProperties.class) +public class DataFlowTaskLaunchRequestAutoConfiguration { + + @Autowired + private BeanFactory beanFactory; + + private EvaluationContext evaluationContext; + + public final static String TASK_LAUNCH_REQUEST_FUNCTION_NAME = "taskLaunchRequest"; + + /** + * A {@link java.util.function.Function} to transform a {@link Message} payload to a {@link DataFlowTaskLaunchRequest}. + * + * @param taskLaunchRequestMessageProcessor a {@link TaskLaunchRequestMessageProcessor}. + * + * @return a {code DataFlowTaskLaunchRequest} Message. + */ + @Bean(name = TASK_LAUNCH_REQUEST_FUNCTION_NAME) + @ConditionalOnMissingBean(TaskLaunchRequestFunction.class) + public TaskLaunchRequestFunction taskLaunchRequest(TaskLaunchRequestMessageProcessor taskLaunchRequestMessageProcessor) { + return message -> taskLaunchRequestMessageProcessor.postProcessMessage(message); + } + + @Bean + @ConditionalOnMissingBean(TaskNameMessageMapper.class) + public TaskNameMessageMapper taskNameMessageMapper(DataflowTaskLaunchRequestProperties taskLaunchRequestProperties) { + if (StringUtils.hasText(taskLaunchRequestProperties.getTaskNameExpression())) { + SpelExpressionParser expressionParser = new SpelExpressionParser(); + Expression taskNameExpression = expressionParser.parseExpression(taskLaunchRequestProperties.getTaskNameExpression()); + return new ExpressionEvaluatingTaskNameMessageMapper(taskNameExpression, this.evaluationContext); + } + + return message -> taskLaunchRequestProperties.getTaskName(); + } + + @Bean + @ConditionalOnMissingBean(CommandLineArgumentsMessageMapper.class) + public CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper( + DataflowTaskLaunchRequestProperties dataflowTaskLaunchRequestProperties){ + + return new ExpressionEvaluatingCommandLineArgsMapper(dataflowTaskLaunchRequestProperties.getArgExpressions(), + this.evaluationContext); + } + + @Bean + public TaskLaunchRequestSupplier taskLaunchRequestInitializer( + DataflowTaskLaunchRequestProperties taskLaunchRequestProperties){ + return new DataflowTaskLaunchRequestPropertiesInitializer(taskLaunchRequestProperties); + } + + @Bean + public TaskLaunchRequestMessageProcessor taskLaunchRequestMessageProcessor( + TaskLaunchRequestSupplier taskLaunchRequestInitializer, + TaskNameMessageMapper taskNameMessageMapper, + CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper) { + + return new TaskLaunchRequestMessageProcessor(taskLaunchRequestInitializer, + taskNameMessageMapper, + commandLineArgumentsMessageMapper); + } + + static class DataflowTaskLaunchRequestPropertiesInitializer extends TaskLaunchRequestSupplier { + DataflowTaskLaunchRequestPropertiesInitializer( + DataflowTaskLaunchRequestProperties taskLaunchRequestProperties){ + + this.commandLineArgumentSupplier( + () -> new ArrayList<>(taskLaunchRequestProperties.getArgs()) + ); + + this.deploymentPropertiesSupplier( + () -> KeyValueListParser.parseCommaDelimitedKeyValuePairs( + taskLaunchRequestProperties.getDeploymentProperties()) + ); + + this.taskNameSupplier(()->taskLaunchRequestProperties.getTaskName()); + } + } + + static class ExpressionEvaluatingCommandLineArgsMapper implements CommandLineArgumentsMessageMapper { + private final Map argExpressionsMap; + private final EvaluationContext evaluationContext; + + ExpressionEvaluatingCommandLineArgsMapper(String argExpressions, EvaluationContext evaluationContext) { + this.evaluationContext = evaluationContext; + this.argExpressionsMap = new HashMap<>(); + if (StringUtils.hasText(argExpressions)) { + SpelExpressionParser expressionParser = new SpelExpressionParser(); + + KeyValueListParser.parseCommaDelimitedKeyValuePairs(argExpressions).forEach( + (k,v)-> argExpressionsMap.put(k, expressionParser.parseExpression(v))); + } + + } + + @Override + public Collection processMessage(Message message) { + return evaluateArgExpressions(message); + } + + private Collection evaluateArgExpressions(Message message) { + List results = new LinkedList<>(); + this.argExpressionsMap.forEach((k, expression) -> + results.add(String.format("%s=%s", k, expression.getValue(this.evaluationContext, message)))); + return results; + } + } + + @PostConstruct + public void createEvaluationContext(){ + this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.beanFactory); + } + +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataflowTaskLaunchRequestProperties.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataflowTaskLaunchRequestProperties.java new file mode 100644 index 00000000..2003c530 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/DataflowTaskLaunchRequestProperties.java @@ -0,0 +1,111 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.ArrayList; +import java.util.List; +import javax.validation.constraints.AssertFalse; +import javax.validation.constraints.NotNull; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; + +/** + * Base Properties to create a {@link DataFlowTaskLaunchRequest}. + * + * @author Chris Schaefer + * @author David Turanski + */ +@Validated +@ConfigurationProperties("task.launch.request") +public class DataflowTaskLaunchRequestProperties { + + /** + * Comma separated list of optional args in key=value format. + */ + private List args = new ArrayList<>(); + + /** + * Comma separated list of option args as SpEL expressions in key=value format. + */ + private String argExpressions = ""; + + /** + * Comma delimited list of deployment properties to be applied to the + * TaskLaunchRequest. + */ + private String deploymentProperties = ""; + + /** + * The Data Flow task name. + */ + private String taskName; + + + /** + * A SpEL expression to extract the task name from each Message, using the Message as the evaluation context. + */ + private String taskNameExpression; + + @NotNull + public List getArgs() { + return this.args; + } + + public void setArgs(List args) { + this.args = new ArrayList<>(args); + } + + @NotNull + public String getDeploymentProperties() { + return this.deploymentProperties; + } + + public void setDeploymentProperties(String deploymentProperties) { + this.deploymentProperties = deploymentProperties; + } + + public String getTaskName() { + return taskName; + } + + public void setTaskName(String taskName) { + this.taskName = taskName; + } + + public String getTaskNameExpression() { + return taskNameExpression; + } + + public void setTaskNameExpression(String taskNameExpression) { + this.taskNameExpression = taskNameExpression; + } + + public String getArgExpressions() { + return argExpressions; + } + + public void setArgExpressions(String argExpressions) { + this.argExpressions = argExpressions; + } + + @AssertFalse(message = "Cannot specify both 'taskName' and 'taskNameExpression'.") + public boolean isTaskNameAndTaskNameExpressionSet() { + return StringUtils.hasText(this.taskName) && StringUtils.hasText(this.taskNameExpression); + } + +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/ExpressionEvaluatingTaskNameMessageMapper.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/ExpressionEvaluatingTaskNameMessageMapper.java new file mode 100644 index 00000000..b38d35e3 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/ExpressionEvaluatingTaskNameMessageMapper.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskNameMessageMapper; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.messaging.Message; + +public class ExpressionEvaluatingTaskNameMessageMapper implements TaskNameMessageMapper { + + private final Expression expression; + private final EvaluationContext evaluationContext; + + public ExpressionEvaluatingTaskNameMessageMapper(Expression expression, EvaluationContext evaluationContext) { + this.evaluationContext = evaluationContext; + this.expression = expression; + } + + @Override + public String processMessage(Message message) { + return expression.getValue(evaluationContext, message).toString(); + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParser.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParser.java new file mode 100644 index 00000000..ee71ebba --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParser.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.springframework.util.StringUtils; + +/** + * Parses a comma delimited list of key value pairs in which the values can contain commas as well. + * + * @author Chris Schaeffer + * @author David Turanski + **/ +abstract class KeyValueListParser { + + static Map parseCommaDelimitedKeyValuePairs(String value) { + Map keyValuePairs = new HashMap<>(); + + if (StringUtils.isEmpty(value)) { + return keyValuePairs; + } + + ArrayList pairs = new ArrayList<>(); + + String[] candidates = StringUtils.commaDelimitedListToStringArray(value); + + for (int i = 0; i < candidates.length; i++) { + if (i > 0 && !candidates[i].contains("=")) { + pairs.add(pairs.get(pairs.size() - 1) + "," + candidates[i]); + } + else { + pairs.add(candidates[i]); + } + } + + for (String pair : pairs) { + addKeyValuePair(pair, keyValuePairs); + } + + return keyValuePairs; + } + + private static void addKeyValuePair(String pair, Map properties) { + int firstEquals = pair.indexOf('='); + if (firstEquals != -1) { + properties.put(pair.substring(0, firstEquals).trim(), pair.substring(firstEquals + 1).trim()); + } + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestFunction.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestFunction.java new file mode 100644 index 00000000..611dc9c1 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestFunction.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.function.Function; + +import org.springframework.messaging.Message; + +/** + * A marker interface useful for unambiguous dependency injection of this Function. + * + * @author David Turanski + **/ +@FunctionalInterface +public interface TaskLaunchRequestFunction extends Function, Message> { + +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestMessageProcessor.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestMessageProcessor.java new file mode 100644 index 00000000..f53209c8 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestMessageProcessor.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import org.springframework.cloud.stream.app.tasklaunchrequest.support.CommandLineArgumentsMessageMapper; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskLaunchRequestSupplier; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskNameMessageMapper; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.core.MessagePostProcessor; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.Assert; +import org.springframework.util.MimeTypeUtils; +import org.springframework.util.StringUtils; + +public class TaskLaunchRequestMessageProcessor implements MessagePostProcessor { + + private final TaskNameMessageMapper taskNameMessageMapper; + private final CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper; + private final TaskLaunchRequestSupplier taskLaunchRequestInitializer; + + public TaskLaunchRequestMessageProcessor(TaskLaunchRequestSupplier taskLaunchRequestInitializer, + TaskNameMessageMapper taskNameMessageMapper, + CommandLineArgumentsMessageMapper commandLIneArgumentsMessageMapper) { + + this.taskLaunchRequestInitializer = taskLaunchRequestInitializer; + + this.taskNameMessageMapper = taskNameMessageMapper; + + this.commandLineArgumentsMessageMapper = commandLIneArgumentsMessageMapper; + + } + + @Override + public Message postProcessMessage(Message message) { + DataFlowTaskLaunchRequest taskLaunchRequest = taskLaunchRequestInitializer.get(); + + if (!StringUtils.hasText(taskLaunchRequest.getTaskName())) { + taskLaunchRequest.setTaskName(taskNameMessageMapper.processMessage(message)); + Assert.hasText(taskLaunchRequest.getTaskName(), ()-> + "'taskName' is required in " + DataFlowTaskLaunchRequest.class.getName()); + } + + taskLaunchRequest.addCommmandLineArguments(commandLineArgumentsMessageMapper.processMessage(message)); + + MessageBuilder builder + = MessageBuilder.withPayload(taskLaunchRequest).copyHeaders(message.getHeaders()); + return adjustHeaders(builder).build(); + } + + + private MessageBuilder adjustHeaders(MessageBuilder builder) { + builder.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON); + return builder; + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/CommandLineArgumentsMessageMapper.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/CommandLineArgumentsMessageMapper.java new file mode 100644 index 00000000..12f4a5af --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/CommandLineArgumentsMessageMapper.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest.support; + +import java.util.Collection; +import org.springframework.integration.handler.MessageProcessor; + +public interface CommandLineArgumentsMessageMapper extends MessageProcessor> { +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskLaunchRequestSupplier.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskLaunchRequestSupplier.java new file mode 100644 index 00000000..9f8a3bb2 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskLaunchRequestSupplier.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest.support; + +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import org.springframework.cloud.stream.app.tasklaunchrequest.DataFlowTaskLaunchRequest; +import org.springframework.util.Assert; + +public class TaskLaunchRequestSupplier implements Supplier { + + private Supplier taskNameSupplier; + private Supplier> commandLineArgumentsSupplier; + private Supplier> deploymentPropertiesSupplier; + + + public TaskLaunchRequestSupplier taskNameSupplier(Supplier taskNameSupplier) { + this.taskNameSupplier = taskNameSupplier; + return this; + } + + public TaskLaunchRequestSupplier commandLineArgumentSupplier(Supplier> commandLineArgumentsSupplier) { + this.commandLineArgumentsSupplier = commandLineArgumentsSupplier; + return this; + } + + public TaskLaunchRequestSupplier deploymentPropertiesSupplier(Supplier> deploymentPropertiesSupplier) { + this.deploymentPropertiesSupplier = deploymentPropertiesSupplier; + return this; + } + + @Override + public DataFlowTaskLaunchRequest get() { + + Assert.notNull(this.taskNameSupplier, "'taskNameSupplier' is required."); + + DataFlowTaskLaunchRequest dataFlowTaskLaunchRequest = new DataFlowTaskLaunchRequest(); + dataFlowTaskLaunchRequest.setTaskName(this.taskNameSupplier.get()); + + if (this.commandLineArgumentsSupplier != null) { + dataFlowTaskLaunchRequest.setCommandlineArguments(this.commandLineArgumentsSupplier.get()); + } + + if (this.deploymentPropertiesSupplier != null) { + dataFlowTaskLaunchRequest.setDeploymentProperties(this.deploymentPropertiesSupplier.get()); + } + + return dataFlowTaskLaunchRequest; + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskNameMessageMapper.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskNameMessageMapper.java new file mode 100644 index 00000000..0997a72b --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/java/org/springframework/cloud/stream/app/tasklaunchrequest/support/TaskNameMessageMapper.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest.support; + +import org.springframework.integration.handler.MessageProcessor; + +@FunctionalInterface +public interface TaskNameMessageMapper extends MessageProcessor { +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..0fbbde32 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring-configuration-metadata-whitelist.properties @@ -0,0 +1,2 @@ +configuration-properties.classes=\ + org.springframework.cloud.stream.app.tasklaunchrequest.DataflowTaskLaunchRequestProperties diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring.factories b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..3e0260f2 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.springframework.cloud.stream.app.tasklaunchrequest.DataFlowTaskLaunchRequestAutoConfiguration diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParserTests.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParserTests.java new file mode 100644 index 00000000..abb59d0a --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/KeyValueListParserTests.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.Map; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Chris Schaefer + * @author David Turanski + */ +public class KeyValueListParserTests { + + @Test + public void testParseSimpleDeploymentProperty() { + Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( + "app.sftp.param=value"); + assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), + deploymentProperties.size() == 1); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); + assertEquals("Invalid deployment value", "value", deploymentProperties.get("app.sftp.param")); + } + + @Test + public void testParseSimpleDeploymentPropertyMultipleValues() { + Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( + "app.sftp.param=value1,value2,value3"); + + assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), + deploymentProperties.size() == 1); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); + assertEquals("Invalid deployment value", "value1,value2,value3", deploymentProperties.get("app.sftp.param")); + } + + @Test + public void testParseSpelExpressionMultipleValues() { + Map argExpressions = KeyValueListParser.parseCommaDelimitedKeyValuePairs( + "arg1=payload.substr(0,2),arg2=headers['foo'],arg3=headers['bar']==false"); + + assertTrue("Invalid number of deployment properties: " + argExpressions.size(), + argExpressions.size() == 3); + assertTrue("Expected deployment key not found", argExpressions.containsKey("arg1")); + assertEquals("Invalid deployment value", "payload.substr(0,2)", argExpressions.get("arg1")); + + assertTrue("Expected deployment key not found", argExpressions.containsKey("arg2")); + assertEquals("Invalid deployment value", "headers['foo']", argExpressions.get("arg2")); + + assertTrue("Expected deployment key not found", argExpressions.containsKey("arg3")); + assertEquals("Invalid deployment value", "headers['bar']==false", argExpressions.get("arg3")); + } + + @Test + public void testParseMultipleDeploymentPropertiesSingleValue() { + Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( + "app.sftp.param=value1,app.sftp.other.param=value2"); + + assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), + deploymentProperties.size() == 2); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); + assertEquals("Invalid deployment value", "value1", deploymentProperties.get("app.sftp.param")); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.other.param")); + assertEquals("Invalid deployment value", "value2", deploymentProperties.get("app.sftp.other.param")); + } + + @Test + public void testParseMultipleDeploymentPropertiesMultipleValues() { + DataflowTaskLaunchRequestProperties taskLaunchRequestProperties = new DataflowTaskLaunchRequestProperties(); + + Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( + "app.sftp.param=value1,value2,app.sftp.other.param=other1,other2"); + + assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), + deploymentProperties.size() == 2); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); + assertEquals("Invalid deployment value", "value1,value2", deploymentProperties.get("app.sftp.param")); + assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.other.param")); + assertEquals("Invalid deployment value", "other1,other2", deploymentProperties.get("app.sftp.other.param")); + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestIntegrationTests.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestIntegrationTests.java new file mode 100644 index 00000000..0aaaa167 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestIntegrationTests.java @@ -0,0 +1,313 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +import java.io.IOException; +import java.util.Collections; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.CommandLineArgumentsMessageMapper; +import org.springframework.cloud.stream.app.tasklaunchrequest.support.TaskNameMessageMapper; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.cloud.stream.test.binder.MessageCollectorAutoConfiguration; +import org.springframework.cloud.stream.test.binder.TestSupportBinderAutoConfiguration; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.MessageBuilder; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * @author David Turanski + **/ +public class TaskLaunchRequestIntegrationTests { + + private ApplicationContextRunner applicationContextRunner; + + @Before + public void setUp() { + applicationContextRunner = + new ApplicationContextRunner().withUserConfiguration(TestChannelBinderConfiguration.class, TestApp.class); + } + + @Test + public void noTaskLaunchRequestPropertiesAreRequired() { + + applicationContextRunner.withPropertyValues("spring.jmx.enabled=false") + .run(context -> { + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + Message message = + MessageBuilder.withPayload("hello".getBytes()).build(); + input.send(message); + + Message response = target.receive(1000); + assertThat(response.getPayload()).isEqualTo(message.getPayload()); + }); + } + + @Test + public void simpleDataflowTaskLaunchRequest() { + + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", + "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name=foo") + .run(context -> { + DataFlowTaskLaunchRequest dataFlowTaskLaunchRequest = verifyAndreceiveDataFlowTaskLaunchRequest(context); + + assertThat(dataFlowTaskLaunchRequest.getTaskName()).isEqualTo("foo"); + assertThat(dataFlowTaskLaunchRequest.getCommandlineArguments()).hasSize(0); + assertThat(dataFlowTaskLaunchRequest.getDeploymentProperties()).hasSize(0); + }); + } + + @Test + public void dataflowTaskLaunchRequestWithArgsAndDeploymentProperties() { + + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name=foo", "task.launch.request.args=foo=bar,baz=boo", + "task.launch.request.deploymentProperties=count=3") + .run(context -> { + DataFlowTaskLaunchRequest dataFlowTaskLaunchRequest = verifyAndreceiveDataFlowTaskLaunchRequest(context); + + assertThat(dataFlowTaskLaunchRequest.getTaskName()).isEqualTo("foo"); + assertThat(dataFlowTaskLaunchRequest.getCommandlineArguments()).containsExactlyInAnyOrder("foo=bar", + "baz=boo"); + assertThat(dataFlowTaskLaunchRequest.getDeploymentProperties()).containsOnly(entry("count", "3")); + }); + } + + @Test + public void dataflowTaskLaunchRequestWithCommandLineArgsMessageMapper() { + + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name=foo", "enhanceTLRArgs=true") + .run(context -> { + + DataFlowTaskLaunchRequest dataFlowTaskLaunchRequest = verifyAndreceiveDataFlowTaskLaunchRequest(context); + + assertThat(dataFlowTaskLaunchRequest.getTaskName()).isEqualTo("foo"); + assertThat(dataFlowTaskLaunchRequest.getCommandlineArguments()).hasSize(1); + assertThat(dataFlowTaskLaunchRequest.getCommandlineArguments()).containsExactly("runtimeArg"); + }); + } + + @Test + public void taskLaunchRequestWithArgExpressions() { + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", + "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name=foo", + "task.launch.request.arg-expressions=foo=payload.toUpperCase(),bar=payload.substring(0,2)") + .run(context -> { + + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + Message message = MessageBuilder.withPayload("hello").build(); + + input.send(message); + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + Message response = target.receive(1000); + + assertThat(response).isNotNull(); + + DataFlowTaskLaunchRequest request = objectMapper.readValue(response.getPayload(), + DataFlowTaskLaunchRequest.class); + + assertThat(request.getCommandlineArguments()).containsExactlyInAnyOrder("foo=HELLO", "bar=he"); + + }); + } + + @Test + public void taskLaunchRequestWithIntPayload() { + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name=foo", + "task.launch.request.arg-expressions=i=payload") + .run(context -> { + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + Message message = + MessageBuilder.withPayload(123).build(); + + input.send(message); + + Message response = target.receive(1000); + + assertThat(response).isNotNull(); + + DataFlowTaskLaunchRequest request = objectMapper.readValue(response.getPayload(), + DataFlowTaskLaunchRequest.class); + + assertThat(request.getCommandlineArguments()).containsExactly("i=123"); + + }); + } + + @Test + public void taskNameExpression() { + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "task.launch.request.task-name-expression=payload+'_task'") + .run(context -> { + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + Message message = MessageBuilder.withPayload("foo").build(); + input.send(message); + + Message response = target.receive(1000); + assertThat(response).isNotNull(); + + DataFlowTaskLaunchRequest request = objectMapper.readValue(response.getPayload(), + DataFlowTaskLaunchRequest.class); + + assertThat(request.getTaskName()).isEqualTo("foo_task"); + }); + } + + @Test + public void customTaskNameExtractor() { + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "customTaskNameExtractor=true") + .run(context -> { + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + Message message = MessageBuilder.withPayload("foo").build(); + input.send(message); + + Message response = target.receive(1000); + assertThat(response).isNotNull(); + + DataFlowTaskLaunchRequest request = objectMapper.readValue(response.getPayload(), + DataFlowTaskLaunchRequest.class); + + assertThat(request.getTaskName()).isEqualTo("fooTask"); + }); + //TODO: Workaround for https://github.com/spring-cloud/spring-cloud-stream/issues/1876 + applicationContextRunner.withPropertyValues( + "spring.jmx.enabled=false", "spring.cloud.stream.function.definition=taskLaunchRequest", + "customTaskNameExtractor=true") + .run(context -> { + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + Message message = MessageBuilder.withPayload("bar").build(); + input.send(message); + + Message response = target.receive(1000); + assertThat(response).isNotNull(); + + DataFlowTaskLaunchRequest request = objectMapper.readValue(response.getPayload(), + DataFlowTaskLaunchRequest.class); + + assertThat(request.getTaskName()).isEqualTo("defaultTask"); + }); + } + + private DataFlowTaskLaunchRequest verifyAndreceiveDataFlowTaskLaunchRequest(ApplicationContext context) + throws IOException { + MessageChannel input = context.getBean("input", MessageChannel.class); + + OutputDestination target = context.getBean(OutputDestination.class); + + MessageBuilder builder = MessageBuilder.withPayload(new byte[] {}); + + ObjectMapper objectMapper = context.getBean(ObjectMapper.class); + + input.send(builder.build()); + + Message message = target.receive(1000); + + assertThat(message).isNotNull(); + + return objectMapper.readValue(message.getPayload(), + DataFlowTaskLaunchRequest.class); + } + + @EnableAutoConfiguration(exclude = { TestSupportBinderAutoConfiguration.class, + MessageCollectorAutoConfiguration.class }) + @EnableBinding(Processor.class) + static class TestApp { + + @Bean + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + @ConditionalOnProperty("customTaskNameExtractor") + TaskNameMessageMapper taskNameExtractor() { + return message -> ((String)(message.getPayload())).equalsIgnoreCase("foo") ? + "fooTask" : + "defaultTask"; + } + + @Bean + @ConditionalOnProperty("enhanceTLRArgs") + CommandLineArgumentsMessageMapper commandLineArgumentsProvider(){ + return message -> Collections.singletonList("runtimeArg"); + } + + @Bean + public IntegrationFlow flow() { + + return IntegrationFlows.from(Processor.INPUT) + .channel(Processor.OUTPUT).get(); + } + } +} diff --git a/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestPropertiesTests.java b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestPropertiesTests.java new file mode 100644 index 00000000..0753e5e7 --- /dev/null +++ b/applications/apps-core/common/stream-apps-task-launch-request-common/src/test/java/org/springframework/cloud/stream/app/tasklaunchrequest/TaskLaunchRequestPropertiesTests.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.tasklaunchrequest; + +import java.util.List; + +import org.junit.Test; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.integration.config.EnableIntegration; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author David Turanski + **/ +public class TaskLaunchRequestPropertiesTests { + + @Test + public void deploymentPropertiesCanBeCustomized() { + DataflowTaskLaunchRequestProperties properties = getBatchProperties( + "task.launch.request.deploymentProperties:prop1=val1,prop2=val2"); + assertThat(properties.getDeploymentProperties()).isEqualTo("prop1=val1,prop2=val2"); + } + + @Test + public void parametersCanBeCustomized() { + DataflowTaskLaunchRequestProperties properties = getBatchProperties( + "task.launch.request.args:jp1=jpv1,jp2=jpv2"); + List args = properties.getArgs(); + + assertThat(args).isNotNull(); + assertThat(args).hasSize(2); + assertThat(args.get(0)).isEqualTo("jp1=jpv1"); + assertThat(args.get(1)).isEqualTo("jp2=jpv2"); + } + + private DataflowTaskLaunchRequestProperties getBatchProperties(String... var) { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + + if (var != null) { + TestPropertyValues.of(var).applyTo(context); + } + + context.register(Conf.class); + context.refresh(); + + return context.getBean(DataflowTaskLaunchRequestProperties.class); + } + + + @Configuration + @EnableIntegration + @EnableConfigurationProperties(DataflowTaskLaunchRequestProperties.class) + @Import(DataFlowTaskLaunchRequestAutoConfiguration.class) + static class Conf { + + } +} diff --git a/applications/apps-core/common/stream-apps-test-support/pom.xml b/applications/apps-core/common/stream-apps-test-support/pom.xml new file mode 100644 index 00000000..975777a2 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/pom.xml @@ -0,0 +1,29 @@ + + + + stream-apps-common + org.springframework.cloud.stream.app + 3.0.0.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-test-support + + + + org.springframework.integration + spring-integration-test + + + org.springframework.integration + spring-integration-test-support + + + org.springframework.boot + spring-boot-starter-data-redis + true + + + + + diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/BinderTestPropertiesInitializer.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/BinderTestPropertiesInitializer.java new file mode 100644 index 00000000..e49ca6e8 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/BinderTestPropertiesInitializer.java @@ -0,0 +1,58 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.test; + +import java.util.Properties; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.PropertiesPropertySource; + +/** + * Unlike the {@code PropertiesInitializer}, this does not require boot infrastructure + * to add properties to the context. Used for testing generated apps where the + * {@code ApplicationContextInitializer} can't be used. Since it's a BDRPP, it runs + * before any BFPPs - i.e. as early as possible. + * + * @author Gary Russell + * + */ +public class BinderTestPropertiesInitializer implements BeanDefinitionRegistryPostProcessor { + + private final ConfigurableApplicationContext context; + + private final Properties properties; + + public BinderTestPropertiesInitializer(ConfigurableApplicationContext context, Properties properties) { + this.context = context; + this.properties = properties; + } + + @Override + public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) throws BeansException { + } + + @Override + public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException { + this.context.getEnvironment().getPropertySources() + .addLast(new PropertiesPropertySource("scsAppProperties", properties)); + } + +} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/PropertiesInitializer.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/PropertiesInitializer.java new file mode 100644 index 00000000..145adc5d --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/PropertiesInitializer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014-2016 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.test; + +import java.util.Properties; + +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.PropertiesPropertySource; + +/** + * @author David Turanski + */ +public class PropertiesInitializer implements ApplicationContextInitializer { + + public static Properties PROPERTIES; + + @Override + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + configurableApplicationContext.getEnvironment().getPropertySources().addLast(new + PropertiesPropertySource("applicationOptions", PROPERTIES)); + } +} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/file/remote/RemoteFileTestSupport.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/file/remote/RemoteFileTestSupport.java new file mode 100644 index 00000000..dad41cd9 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/file/remote/RemoteFileTestSupport.java @@ -0,0 +1,149 @@ +/* + * Copyright 2015-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.stream.app.test.file.remote; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.TemporaryFolder; + +/** + * Abstract base class for tests requiring remote file servers, e.g. (S)FTP. + * + * @author Gary Russell + * + */ +public abstract class RemoteFileTestSupport { + + protected static final int port = 0; + + @ClassRule + public static final TemporaryFolder remoteTemporaryFolder = new TemporaryFolder(); + + @ClassRule + public static final TemporaryFolder localTemporaryFolder = new TemporaryFolder(); + + protected volatile File sourceRemoteDirectory; + + protected volatile File targetRemoteDirectory; + + protected volatile File sourceLocalDirectory; + + protected volatile File targetLocalDirectory; + + public File getSourceRemoteDirectory() { + return sourceRemoteDirectory; + } + + public File getTargetRemoteDirectory() { + return targetRemoteDirectory; + } + + public File getSourceLocalDirectory() { + return sourceLocalDirectory; + } + + public File getTargetLocalDirectory() { + return targetLocalDirectory; + } + + /** + * Default implementation creates the following folder structures: + * + * 
+	 *  $ tree remoteSource/
+	 *  remoteSource/
+	 *  ├── remoteSource1.txt - contains 'source1'
+	 *  ├── remoteSource2.txt - contains 'source2'
+	 *  remoteTarget/
+	 *  $ tree localSource/
+	 *  localSource/
+	 *  ├── localSource1.txt - contains 'local1'
+	 *  ├── localSource2.txt - contains 'local2'
+	 *  localTarget/
+	 *
+ * + * The intent is tests retrieve from remoteSource and verify arrival in localTarget or send from localSource and verify + * arrival in remoteTarget. + *

+ * Subclasses can change 'remote' in these names by overriding {@link #prefix()} or override this method completely to + * create a different structure. + *

+ * While a single server exists for all tests, the directory structure is rebuilt for each test. + * @throws IOException IO Exception. + */ + @Before + public void setupFolders() throws IOException { + String prefix = prefix(); + recursiveDelete(new File(remoteTemporaryFolder.getRoot(), prefix + "Source")); + this.sourceRemoteDirectory = remoteTemporaryFolder.newFolder(prefix + "Source"); + recursiveDelete(new File(remoteTemporaryFolder.getRoot(), prefix + "Target")); + this.targetRemoteDirectory = remoteTemporaryFolder.newFolder(prefix + "Target"); + recursiveDelete(new File(localTemporaryFolder.getRoot(), "localSource")); + this.sourceLocalDirectory = localTemporaryFolder.newFolder("localSource"); + recursiveDelete(new File(localTemporaryFolder.getRoot(), "localTarget")); + this.targetLocalDirectory = localTemporaryFolder.newFolder("localTarget"); + File file = new File(sourceRemoteDirectory, prefix + "Source1.txt"); + file.createNewFile(); + FileOutputStream fos = new FileOutputStream(file); + fos.write("source1".getBytes()); + fos.close(); + file = new File(sourceRemoteDirectory, prefix + "Source2.txt"); + file.createNewFile(); + fos = new FileOutputStream(file); + fos.write("source2".getBytes()); + fos.close(); + file = new File(sourceLocalDirectory, "localSource1.txt"); + file.createNewFile(); + fos = new FileOutputStream(file); + fos.write("local1".getBytes()); + fos.close(); + file = new File(sourceLocalDirectory, "localSource2.txt"); + file.createNewFile(); + fos = new FileOutputStream(file); + fos.write("local2".getBytes()); + fos.close(); + } + + public void recursiveDelete(File file) { + if (file != null && file.exists()) { + File[] files = file.listFiles(); + if (files != null) { + for (File fyle : files) { + if (fyle.isDirectory()) { + recursiveDelete(fyle); + } + else { + fyle.delete(); + } + } + } + file.delete(); + } + } + + /** + * Prefix for directory/file structure; default 'remote'. + * @return the prefix. + */ + protected String prefix() { + return "remote"; + } + +} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSinkTestConfiguration.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSinkTestConfiguration.java new file mode 100644 index 00000000..e1de6007 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSinkTestConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.test.ip; + +import java.util.Properties; + +import org.springframework.cloud.stream.app.test.BinderTestPropertiesInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Generated app test configuration for the IP (TCP) sink. + * + * @author Gary Russell + * + */ +@Configuration +public class IpSinkTestConfiguration { + + @Bean + public static BinderTestPropertiesInitializer loadProps(ConfigurableApplicationContext context) { + // minimal properties for the context to load + Properties properties = new Properties(); + properties.put("host", "localhost"); + return new BinderTestPropertiesInitializer(context, properties); + } + +} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSourceTestConfiguration.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSourceTestConfiguration.java new file mode 100644 index 00000000..babe0127 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/ip/IpSourceTestConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.test.ip; + +import java.util.Properties; + +import org.springframework.cloud.stream.app.test.BinderTestPropertiesInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Generated app test configuration for IP (TCP, UDP) sources. + * + * @author Gary Russell + * + */ +@Configuration +public class IpSourceTestConfiguration { + + @Bean + public static BinderTestPropertiesInitializer loadProps(ConfigurableApplicationContext context) { + // minimal properties for the context to load + Properties properties = new Properties(); + properties.put("port", 0); + return new BinderTestPropertiesInitializer(context, properties); + } + +} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/redis/RedisTestSupport.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/redis/RedisTestSupport.java new file mode 100644 index 00000000..321d9499 --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/redis/RedisTestSupport.java @@ -0,0 +1,43 @@ +///* +// * Copyright 2016 the original author or authors. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * https://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.springframework.cloud.stream.app.test.redis; +// +//import org.springframework.cloud.stream.test.junit.AbstractExternalResourceTestSupport; +//import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +// +///** +// * Porting from https://github.com/spring-cloud/spring-cloud-stream/blob/1.0.x/spring-cloud-stream-test-support-internal +// * +// * @author Soby Chacko +// */ +//public class RedisTestSupport extends AbstractExternalResourceTestSupport { +// +// public RedisTestSupport() { +// super("REDIS"); +// } +// @Override +// protected void cleanupResource() throws Exception { +// resource.destroy(); +// } +// +// @Override +// protected void obtainResource() throws Exception { +// resource = new LettuceConnectionFactory(); +// resource.afterPropertiesSet(); +// resource.getConnection().close(); +// } +//} diff --git a/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/script/ScriptableTestConfiguration.java b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/script/ScriptableTestConfiguration.java new file mode 100644 index 00000000..c677dc5d --- /dev/null +++ b/applications/apps-core/common/stream-apps-test-support/src/main/java/org/springframework/cloud/stream/app/test/script/ScriptableTestConfiguration.java @@ -0,0 +1,44 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.test.script; + +import java.util.Properties; + +import org.springframework.cloud.stream.app.test.BinderTestPropertiesInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Test configuration for generated scriptable apps. + * + * @author Gary Russell + * + */ +@Configuration +public class ScriptableTestConfiguration { + + @Bean + public BinderTestPropertiesInitializer loadProps(ConfigurableApplicationContext context) { + // minimal properties for the context to load + Properties properties = new Properties(); + properties.put("script", "foo"); + properties.put("language", "ruby"); + return new BinderTestPropertiesInitializer(context, properties); + } + +} \ No newline at end of file diff --git a/applications/apps-core/pom.xml b/applications/apps-core/pom.xml new file mode 100644 index 00000000..37f50ab8 --- /dev/null +++ b/applications/apps-core/pom.xml @@ -0,0 +1,348 @@ + + + 4.0.0 + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + stream-apps-parent + Infrastructure for stream apps + pom + + + org.springframework.boot + spring-boot-starter-parent + 2.3.0.M4 + + + + 1.8 + 3.0.0.M2 + 2.3.0.M4 + 2.0.1.M1 + 2.0.0.M2 + 3.0.0.M1 + 3.0.2.RELEASE + 3.0.2.RELEASE + Hoxton.SR2 + Horsham.SR2 + 2.1.1.RELEASE + 1.0.0.BUILD-SNAPSHOT + 0.9.0 + + + + common + + + + + + org.springframework.cloud + spring-cloud-stream-dependencies + ${spring-cloud-stream-dependencies.version} + pom + import + + + + + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream + test-jar + test-binder + test + + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + Copyright 2014-2020 the original author or authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. + + See the License for the specific language governing permissions and + limitations under the License. + + + + scm:git:git://github.com/pivotal/java-functions.git + scm:git:ssh://git@github.com/pivotal/java-functions.git + https://github.com/pivotal/java-functions + + + + + repo.spring.io + Spring Release Repository + https://repo.spring.io/libs-release-local + + + repo.spring.io + Spring Snapshot Repository + https://repo.spring.io/libs-snapshot-local + + + + + + milestone + + + repo.spring.io + Spring Milestone Repository + https://repo.spring.io/libs-milestone-local + + + + + central + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + + + + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + **/*Tests.java + **/*Test.java + + + **/Abstract*.java + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + ${stream-apps-docs-plugin.version} + + + generate-documentation + verify + + generate-documentation + + + + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + ${scst-app-maven-plugin.version} + + + app-gen + package + + generate-app + + + + + ${basedir}/apps + 1.8 + ${spring-boot.version} + + + kafka + rabbit + + + + org.springframework.cloud + spring-cloud-stream-dependencies + ${spring-cloud-stream-dependencies.version} + + + org.springframework.cloud + spring-cloud-function-dependencies + ${spring-cloud-function-dependencies.version} + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud-dependencies.version} + + + + + org.springframework.cloud.stream.app + stream-apps-security-common + ${stream-apps-core.version} + + + org.springframework.cloud.stream.app + stream-apps-micrometer-common + ${stream-apps-core.version} + + + io.micrometer + micrometer-registry-influx + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer.prometheus + prometheus-rsocket-spring + ${prometheus-rsocket.version} + + + io.micrometer + micrometer-registry-datadog + + + io.pivotal.cfenv + java-cfenv-boot + ${java-cfenv-boot.version} + + + org.springframework.boot + spring-boot-configuration-processor + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-logging + + + org.springframework.boot + spring-boot-starter-security + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + false + + spring-releases + Spring Releases + https://repo.spring.io/release + + + + false + + spring-libs-release + Spring Libs Release + https://repo.spring.io/libs-release + + + + false + + spring-milestone-release + Spring Milestone Release + https://repo.spring.io/libs-milestone + + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/apps-metadata/CODE_OF_CONDUCT.adoc b/applications/apps-metadata/CODE_OF_CONDUCT.adoc new file mode 100644 index 00000000..17783c7c --- /dev/null +++ b/applications/apps-metadata/CODE_OF_CONDUCT.adoc @@ -0,0 +1,44 @@ += Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of fostering an open +and welcoming community, we pledge to respect all people who contribute through reporting +issues, posting feature requests, updating documentation, submitting pull requests or +patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for +everyone, regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, body size, race, ethnicity, age, +religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic addresses, + without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or reject comments, +commits, code, wiki edits, issues, and other contributions that are not aligned to this +Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors +that they deem inappropriate, threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to fairly and +consistently applying these principles to every aspect of managing this project. Project +maintainers who do not follow or enforce the Code of Conduct may be permanently removed +from the project team. + +This Code of Conduct applies both within project spaces and in public spaces when an +individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by +contacting a project maintainer at spring-code-of-conduct@pivotal.io . All complaints will +be reviewed and investigated and will result in a response that is deemed necessary and +appropriate to the circumstances. Maintainers are obligated to maintain confidentiality +with regard to the reporter of an incident. + +This Code of Conduct is adapted from the +https://contributor-covenant.org[Contributor Covenant], version 1.3.0, available at +https://contributor-covenant.org/version/1/3/0/[contributor-covenant.org/version/1/3/0/] diff --git a/applications/apps-metadata/LICENSE b/applications/apps-metadata/LICENSE new file mode 100644 index 00000000..9b259bdf --- /dev/null +++ b/applications/apps-metadata/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/applications/apps-metadata/pom.xml b/applications/apps-metadata/pom.xml new file mode 100644 index 00000000..68a2325a --- /dev/null +++ b/applications/apps-metadata/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + apps-metadata + Fahrenheit.BUILD-SNAPSHOT + pom + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + stream-apps-docs + stream-apps-descriptor + + + + + spring + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + spring-libs-release + Spring Libs Release + https://repo.spring.io/libs-release + + false + + + + + false + + spring-milestone-release + Spring Milestone Release + https://repo.spring.io/libs-milestone + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + + + + diff --git a/applications/apps-metadata/release-tools/core-tag-next-version.sh b/applications/apps-metadata/release-tools/core-tag-next-version.sh new file mode 100755 index 00000000..6f9a086a --- /dev/null +++ b/applications/apps-metadata/release-tools/core-tag-next-version.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# The script takes two arguments - currently released version for tagging and next version + +if [ "$#" -ne 2 ]; then + echo "Please specify the released version and the next version" + exit +fi + +pushd /tmp + +git clone git@github.com:spring-cloud-stream-app-starters/core.git +cd core + +git tag v$1 +git push origin v$1 + +./mvnw versions:set -DnewVersion=$2 -DgenerateBackupPoms=false +./mvnw versions:set -DnewVersion=$2 -DgenerateBackupPoms=false -pl :app-starters-core-dependencies + +sed -i '' 's/.*/'"$2"'<\/app-starters-core-dependencies.version>/g' pom.xml + +git commit -am"Next version - $2" +git push origin master + +cd .. +rm -rf core + +popd diff --git a/applications/apps-metadata/release-tools/core-version-check.sh b/applications/apps-metadata/release-tools/core-version-check.sh new file mode 100755 index 00000000..7c883647 --- /dev/null +++ b/applications/apps-metadata/release-tools/core-version-check.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# The script takes one argument - release version + +if [ "$#" -ne 1 ]; then + echo "Please specify the release version" + exit +fi + +pushd /tmp + +git clone git@github.com:spring-cloud-stream-app-starters/core.git +cd core + +./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false -U +./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false -pl :app-starters-core-dependencies -U + +sed -i '' 's/.*/'"$1"'<\/app-starters-core-dependencies.version>/g' pom.xml + +snapshotlines=$(find . -type f -name pom.xml | xargs grep SNAPSHOT | wc -l) +rclines=$(find . -type f -name pom.xml | xargs grep .RC | wc -l) +milestonelines=$(find . -type f -name pom.xml | xargs grep version | grep .M | wc -l) + +if [ $snapshotlines -eq 0 ] && [ $rclines -eq 0 ] && [$milestonelines -eq 0 ]; then + echo "All clear" +else + echo "Snapshots found." + find . -type f -name pom.xml | xargs grep SNAPSHOT + echo "SNAPSHOTS: " $snapshotlines + exit 1 +fi + +cd .. +rm -rf core + +popd + + diff --git a/applications/apps-metadata/release-tools/core-version-upgrade.sh b/applications/apps-metadata/release-tools/core-version-upgrade.sh new file mode 100755 index 00000000..60c4a0a3 --- /dev/null +++ b/applications/apps-metadata/release-tools/core-version-upgrade.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# The script takes one argument - release version + +if [ "$#" -ne 1 ]; then + echo "Please specify the release version" + exit +fi + +pushd /tmp + +git clone git@github.com:spring-cloud-stream-app-starters/core.git +cd core + +./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false -U +./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false -pl :app-starters-core-dependencies -U + +sed -i '' 's/.*/'"$1"'<\/app-starters-core-dependencies.version>/g' pom.xml + +snapshotlines=$(find . -type f -name pom.xml | xargs grep SNAPSHOT | wc -l) +rclines=$(find . -type f -name pom.xml | xargs grep .RC | wc -l) +milestonelines=$(find . -type f -name pom.xml | xargs grep version | grep .M | wc -l) + +if [ $snapshotlines -eq 0 ] && [ $rclines -eq 0 ] && [$milestonelines -eq 0 ]; then + echo "All clear" + git commit -am"Update version to $1" + git push origin master +else + echo "Snapshots found." + echo "SNAPSHOTS: " $snapshotlines + echo "Milestones: " $milestonelines + echo "RC: " $rclines + exit 1 +fi + +cd .. +rm -rf core + +popd diff --git a/applications/apps-metadata/stream-apps-descriptor/pom.xml b/applications/apps-metadata/stream-apps-descriptor/pom.xml new file mode 100644 index 00000000..2206d721 --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/pom.xml @@ -0,0 +1,140 @@ + + + + stream-apps-release-train + org.springframework.cloud.stream.app + Fahrenheit.BUILD-SNAPSHOT + + 4.0.0 + + stream-apps-descriptor + stream-apps-descriptor + jar + + + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + 3.0.0.BUILD-SNAPSHOT + + + + + + src/main/resources + true + + META-INF/kafka-apps-maven.properties + META-INF/rabbit-apps-maven.properties + META-INF/kafka-apps-docker.properties + META-INF/rabbit-apps-docker.properties + META-INF/kafka-apps-maven-repo-url.properties + META-INF/rabbit-apps-maven-repo-url.properties + + + + + + + org.codehaus.gmaven + gmaven-plugin + 1.5 + + + validate + + execute + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.8 + + + attach-artifacts + package + + attach-artifact + + + + + target/classes/META-INF/kafka-apps-maven.properties + stream-apps-kafka-maven + + + target/classes/META-INF/kafka-apps-docker.properties + stream-apps-kafka-docker + + + target/classes/META-INF/rabbit-apps-maven.properties + stream-apps-rabbit-maven + + + target/classes/META-INF/rabbit-apps-docker.properties + stream-apps-rabbit-docker + + + target/classes/META-INF/kafka-apps-maven-repo-url.properties + kafka-apps-maven-repo-url.properties + + + target/classes/META-INF/rabbit-apps-maven-repo-url.properties + rabbit-apps-maven-repo-url.properties + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties new file mode 100644 index 00000000..ceeef29d --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties @@ -0,0 +1,13 @@ +sink.cassandra=docker:springcloudstream/cassandra-sink-kafka:@cassandra-sink-docker.tag@ +sink.counter=docker:springcloudstream/counter-sink-kafka:@counter-sink-docker.tag@ +sink.jdbc=docker:springcloudstream/jdbc-sink-kafka:@jdbc-sink-docker.tag@ +sink.log=docker:springcloudstream/log-sink-kafka:@log-sink-docker.tag@ +sink.mongodb=docker:springcloudstream/mongodb-sink-kafka:@mongodb-sink-docker.tag@ +sink.rabbit=docker:springcloudstream/rabbit-sink-kafka:@rabbit-sink-docker.tag@ +source.http=docker:springcloudstream/http-source-kafka:@http-source-docker.tag@ +source.jdbc=docker:springcloudstream/jdbc-source-kafka:@jdbc-source-docker.tag@ +source.mongodb=docker:springcloudstream/mongodb-source-kafka:@mongodb-source-docker.tag@ +source.time=docker:springcloudstream/time-source-kafka:@time-source-docker.tag@ +processor.filter=docker:springcloudstream/filter-processor-kafka:@filter-processor-docker.tag@ +processor.splitter=docker:springcloudstream/splitter-processor-kafka:@splitter-processor-docker.tag@ +processor.transform=docker:springcloudstream/transform-processor-kafka:@transform-processor-docker.tag@ diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties new file mode 100644 index 00000000..140d9a87 --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties @@ -0,0 +1,26 @@ +sink.cassandra=https://@repo-spring-io@/org/springframework/cloud/stream/app/cassandra-sink-kafka/@cassandra-sink.version@/cassandra-sink-kafka-@cassandra-sink.version@.jar +sink.cassandra.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/cassandra-sink-kafka/@cassandra-sink.version@/cassandra-sink-kafka-@cassandra-sink.version@-metadata.jar +sink.counter=https://@repo-spring-io@/org/springframework/cloud/stream/app/counter-sink-kafka/@counter-sink.version@/counter-sink-kafka-@counter-sink.version@.jar +sink.counter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/counter-sink-kafka/@counter-sink.version@/counter-sink-kafka-@counter-sink.version@-metadata.jar +sink.jdbc=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-sink-kafka/@jdbc-sink.version@/jdbc-sink-kafka-@jdbc-sink.version@.jar +sink.jdbc.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-sink-kafka/@jdbc-sink.version@/jdbc-sink-kafka-@jdbc-sink.version@-metadata.jar +sink.log=https://@repo-spring-io@/org/springframework/cloud/stream/app/log-sink-kafka/@log-sink.version@/log-sink-kafka-@log-sink.version@.jar +sink.log.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/log-sink-kafka/@log-sink.version@/log-sink-kafka-@log-sink.version@-metadata.jar +sink.mongodb=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-sink-kafka/@mongodb-sink.version@/mongodb-sink-kafka-@mongodb-sink.version@.jar +sink.mongodb.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-sink-kafka/@mongodb-sink.version@/mongodb-sink-kafka-@mongodb-sink.version@-metadata.jar +sink.rabbit=https://@repo-spring-io@/org/springframework/cloud/stream/app/rabbit-sink-kafka/@rabbit-sink.version@/rabbit-sink-kafka-@rabbit-sink.version@.jar +sink.rabbit.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/rabbit-sink-kafka/@rabbit-sink.version@/rabbit-sink-kafka-@rabbit-sink.version@-metadata.jar +source.http=https://@repo-spring-io@/org/springframework/cloud/stream/app/http-source-kafka/@http-source.version@/http-source-kafka-@http-source.version@.jar +source.http.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/http-source-kafka/@http-source.version@/http-source-kafka-@http-source.version@-metadata.jar +source.jdbc=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-source-kafka/@jdb-source.version@/jdbc-source-kafka-@jdbc-source.version@.jar +source.jdbc.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-source-kafka/@jdb-source.version@/jdbc-source-kafka-@jdbc-source.version@-metadata.jar +source.mongodb=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-source-kafka/@mongodb-source.version@/mongodb-source-kafka-@mongodb-source.version@.jar +source.mongodb.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-source-kafka/@mongodb-source.version@/mongodb-source-kafka-@mongodb-source.version@-metadata.jar +source.time=https://@repo-spring-io@/org/springframework/cloud/stream/app/time-source-kafka/@time-source.version@/time-source-kafka-@time-source.version@.jar +source.time.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/time-source-kafka/@time-source.version@/time-source-kafka-@time-source.version@-metadata.jar +processor.filter=https://@repo-spring-io@/org/springframework/cloud/stream/app/filter-processor-kafka/@filter-processor.version@/filter-processor-kafka-@filter-processor.version@.jar +processor.filter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/filter-processor-kafka/@filter-processor.version@/filter-processor-kafka-@filter-processor.version@-metadata.jar +processor.splitter=https://@repo-spring-io@/org/springframework/cloud/stream/app/splitter-processor-kafka/@splitter-processor.version@/splitter-processor-kafka-@splitter-processor.version@.jar +processor.splitter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/splitter-processor-kafka/@splitter-processor.version@/splitter-processor-kafka-@splitter-processor.version@-metadata.jar +processor.transform=https://@repo-spring-io@/org/springframework/cloud/stream/app/transform-processor-kafka/@transform-processor.version@/transform-processor-kafka-@transform-processor.version@.jar +processor.transform.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/transform-processor-kafka/@transform-processor.version@/transform-processor-kafka-@transform-processor.version@-metadata.jar \ No newline at end of file diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties new file mode 100644 index 00000000..8655eac1 --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties @@ -0,0 +1,27 @@ +sink.cassandra=maven://org.springframework.cloud.stream.app:cassandra-sink-kafka:@cassandra-sink.version@ +sink.cassandra.metadata=maven://org.springframework.cloud.stream.app:cassandra-sink-kafka:jar:metadata:@cassandra-sink.version@ +sink.counter=maven://org.springframework.cloud.stream.app:counter-sink-kafka:@counter-sink.version@ +sink.counter.metadata=maven://org.springframework.cloud.stream.app:counter-sink-kafka:jar:metadata:@counter-sink.version@ +sink.jdbc=maven://org.springframework.cloud.stream.app:jdbc-sink-kafka:@jdbc-sink.version@ +sink.jdbc.metadata=maven://org.springframework.cloud.stream.app:jdbc-sink-kafka:jar:metadata:@jdbc-sink.version@ +sink.log=maven://org.springframework.cloud.stream.app:log-sink-kafka:@log-sink.version@ +sink.log.metadata=maven://org.springframework.cloud.stream.app:log-sink-kafka:jar:metadata:@log-sink.version@ +sink.mongodb=maven://org.springframework.cloud.stream.app:mongodb-sink-kafka:@mongodb-sink.version@ +sink.mongodb.metadata=maven://org.springframework.cloud.stream.app:mongodb-sink-kafka:jar:metadata:@mongodb-sink.version@ +sink.rabbit=maven://org.springframework.cloud.stream.app:rabbit-sink-kafka:@rabbit-sink.version@ +sink.rabbit.metadata=maven://org.springframework.cloud.stream.app:rabbit-sink-kafka:jar:metadata:@rabbit-sink.version@ +source.http=maven://org.springframework.cloud.stream.app:http-source-kafka:@http-source.version@ +source.http.metadata=maven://org.springframework.cloud.stream.app:http-source-kafka:jar:metadata:@http-source.version@ +source.jdbc=maven://org.springframework.cloud.stream.app:jdbc-source-kafka:@jdbc-source.version@ +source.jdbc.metadata=maven://org.springframework.cloud.stream.app:jdbc-source-kafka:jar:metadata:@jdbc-source.version@ +source.mongodb=maven://org.springframework.cloud.stream.app:mongodb-source-kafka:@mongodb-source.version@ +source.mongodb.metadata=maven://org.springframework.cloud.stream.app:mongodb-source-kafka:jar:metadata:@mongodb-source.version@ +source.time=maven://org.springframework.cloud.stream.app:time-source-kafka:@time-source.version@ +source.time.metadata=maven://org.springframework.cloud.stream.app:time-source-kafka:jar:metadata:@time-source.version@ +processor.filter=maven://org.springframework.cloud.stream.app:filter-processor-kafka:@filter-processor.version@ +processor.filter.metadata=maven://org.springframework.cloud.stream.app:filter-processor-kafka:jar:metadata:@filter-processor.version@ +processor.splitter=maven://org.springframework.cloud.stream.app:splitter-processor-kafka:@splitter-processor.version@ +processor.splitter.metadata=maven://org.springframework.cloud.stream.app:splitter-processor-kafka:jar:metadata:@splitter-processor.version@ +processor.transform=maven://org.springframework.cloud.stream.app:transform-processor-kafka:@transform-processor.version@ +processor.transform.metadata=maven://org.springframework.cloud.stream.app:transform-processor-kafka:jar:metadata:@transform-processor.version@ + diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties new file mode 100644 index 00000000..5af47680 --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties @@ -0,0 +1,13 @@ +sink.cassandra=docker:springcloudstream/cassandra-sink-rabbit:@cassandra-sink-docker.tag@ +sink.counter=docker:springcloudstream/counter-sink-rabbit:@counter-sink-docker.tag@ +sink.jdbc=docker:springcloudstream/jdbc-sink-rabbit:@jdbc-sink-docker.tag@ +sink.log=docker:springcloudstream/log-sink-rabbit:@log-sink-docker.tag@ +sink.mongodb=docker:springcloudstream/mongodb-sink-rabbit:@mongodb-sink-docker.tag@ +sink.rabbit=docker:springcloudstream/rabbit-sink-rabbit:@rabbit-sink-docker.tag@ +source.http=docker:springcloudstream/http-source-rabbit:@http-source-docker.tag@ +source.jdbc=docker:springcloudstream/jdbc-source-rabbit:@jdbc-source-docker.tag@ +source.mongodb=docker:springcloudstream/mongodb-source-rabbit:@mongodb-source-docker.tag@ +source.time=docker:springcloudstream/time-source-rabbit:@time-source-docker.tag@ +processor.filter=docker:springcloudstream/filter-processor-rabbit:@filter-processor-docker.tag@ +processor.splitter=docker:springcloudstream/splitter-processor-rabbit:@splitter-processor-docker.tag@ +processor.transform=docker:springcloudstream/transform-processor-rabbit:@transform-processor-docker.tag@ diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties new file mode 100644 index 00000000..b67e0dd8 --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties @@ -0,0 +1,26 @@ +sink.cassandra=https://@repo-spring-io@/org/springframework/cloud/stream/app/cassandra-sink-rabbit/@cassandra-sink.version@/cassandra-sink-rabbit-@cassandra-sink.version@.jar +sink.cassandra.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/cassandra-sink-rabbit/@cassandra-sink.version@/cassandra-sink-rabbit-@cassandra-sink.version@-metadata.jar +sink.counter=https://@repo-spring-io@/org/springframework/cloud/stream/app/counter-sink-rabbit/@counter-sink.version@/counter-sink-rabbit-@counter-sink.version@.jar +sink.counter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/counter-sink-rabbit/@counter-sink.version@/counter-sink-rabbit-@counter-sink.version@-metadata.jar +sink.jdbc=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-sink-rabbit/@jdbc-sink.version@/jdbc-sink-rabbit-@jdbc-sink.version@.jar +sink.jdbc.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-sink-rabbit/@jdbc-sink.version@/jdbc-sink-rabbit-@jdbc-sink.version@-metadata.jar +sink.log=https://@repo-spring-io@/org/springframework/cloud/stream/app/log-sink-rabbit/@log-sink.version@/log-sink-rabbit-@log-sink.version@.jar +sink.log.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/log-sink-rabbit/@log-sink.version@/log-sink-rabbit-@log-sink.version@-metadata.jar +sink.mongodb=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-sink-rabbit/@mongodb-sink.version@/mongodb-sink-rabbit-@mongodb-sink.version@.jar +sink.mongodb.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-sink-rabbit/@mongodb-sink.version@/mongodb-sink-rabbit-@mongodb-sink.version@-metadata.jar +sink.rabbit=https://@repo-spring-io@/org/springframework/cloud/stream/app/rabbit-sink-rabbit/@rabbit-sink.version@/rabbit-sink-rabbit-@rabbit-sink.version@.jar +sink.rabbit.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/rabbit-sink-rabbit/@rabbit-sink.version@/rabbit-sink-rabbit-@rabbit-sink.version@-metadata.jar +source.http=https://@repo-spring-io@/org/springframework/cloud/stream/app/http-source-rabbit/@http-source.version@/http-source-rabbit-@http-source.version@.jar +source.http.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/http-source-rabbit/@http-source.version@/http-source-rabbit-@http-source.version@-metadata.jar +source.jdbc=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-source-rabbit/@jdb-source.version@/jdbc-source-rabbit-@jdbc-source.version@.jar +source.jdbc.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/jdbc-source-rabbit/@jdb-source.version@/jdbc-source-rabbit-@jdbc-source.version@-metadata.jar +source.mongodb=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-source-rabbit/@mongodb-source.version@/mongodb-source-rabbit-@mongodb-source.version@.jar +source.mongodb.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/mongodb-source-rabbit/@mongodb-source.version@/mongodb-source-rabbit-@mongodb-source.version@-metadata.jar +source.time=https://@repo-spring-io@/org/springframework/cloud/stream/app/time-source-rabbit/@time-source.version@/time-source-rabbit-@time-source.version@.jar +source.time.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/time-source-rabbit/@time-source.version@/time-source-rabbit-@time-source.version@-metadata.jar +processor.filter=https://@repo-spring-io@/org/springframework/cloud/stream/app/filter-processor-rabbit/@filter-processor.version@/filter-processor-rabbit-@filter-processor.version@.jar +processor.filter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/filter-processor-rabbit/@filter-processor.version@/filter-processor-rabbit-@filter-processor.version@-metadata.jar +processor.splitter=https://@repo-spring-io@/org/springframework/cloud/stream/app/splitter-processor-rabbit/@splitter-processor.version@/splitter-processor-rabbit-@splitter-processor.version@.jar +processor.splitter.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/splitter-processor-rabbit/@splitter-processor.version@/splitter-processor-rabbit-@splitter-processor.version@-metadata.jar +processor.transform=https://@repo-spring-io@/org/springframework/cloud/stream/app/transform-processor-rabbit/@transform-processor.version@/transform-processor-rabbit-@transform-processor.version@.jar +processor.transform.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/transform-processor-rabbit/@transform-processor.version@/transform-processor-rabbit-@transform-processor.version@-metadata.jar \ No newline at end of file diff --git a/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties new file mode 100644 index 00000000..f6bfaa8b --- /dev/null +++ b/applications/apps-metadata/stream-apps-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties @@ -0,0 +1,27 @@ +sink.cassandra=maven://org.springframework.cloud.stream.app:cassandra-sink-rabbit:@cassandra-sink.version@ +sink.cassandra.metadata=maven://org.springframework.cloud.stream.app:cassandra-sink-rabbit:jar:metadata:@cassandra-sink.version@ +sink.counter=maven://org.springframework.cloud.stream.app:counter-sink-rabbit:@counter-sink.version@ +sink.counter.metadata=maven://org.springframework.cloud.stream.app:counter-sink-rabbit:jar:metadata:@counter-sink.version@ +sink.jdbc=maven://org.springframework.cloud.stream.app:jdbc-sink-rabbit:@jdbc-sink.version@ +sink.jdbc.metadata=maven://org.springframework.cloud.stream.app:jdbc-sink-rabbit:jar:metadata:@jdbc-sink.version@ +sink.log=maven://org.springframework.cloud.stream.app:log-sink-rabbit:@log-sink.version@ +sink.log.metadata=maven://org.springframework.cloud.stream.app:log-sink-rabbit:jar:metadata:@log-sink.version@ +sink.mongodb=maven://org.springframework.cloud.stream.app:mongodb-sink-rabbit:@mongodb-sink.version@ +sink.mongodb.metadata=maven://org.springframework.cloud.stream.app:mongodb-sink-rabbit:jar:metadata:@mongodb-sink.version@ +sink.rabbit=maven://org.springframework.cloud.stream.app:rabbit-sink-rabbit:@rabbit-sink.version@ +sink.rabbit.metadata=maven://org.springframework.cloud.stream.app:rabbit-sink-rabbit:jar:metadata:@rabbit-sink.version@ +source.http=maven://org.springframework.cloud.stream.app:http-source-rabbit:@http-source.version@ +source.http.metadata=maven://org.springframework.cloud.stream.app:http-source-rabbit:jar:metadata:@http-source.version@ +source.jdbc=maven://org.springframework.cloud.stream.app:jdbc-source-rabbit:@jdbc-source.version@ +source.jdbc.metadata=maven://org.springframework.cloud.stream.app:jdbc-source-rabbit:jar:metadata:@jdbc-source.version@ +source.mongodb=maven://org.springframework.cloud.stream.app:mongodb-source-rabbit:@mongodb-source.version@ +source.mongodb.metadata=maven://org.springframework.cloud.stream.app:mongodb-source-rabbit:jar:metadata:@mongodb-source.version@ +source.time=maven://org.springframework.cloud.stream.app:time-source-rabbit:@time-source.version@ +source.time.metadata=maven://org.springframework.cloud.stream.app:time-source-rabbit:jar:metadata:@time-source.version@ +processor.filter=maven://org.springframework.cloud.stream.app:filter-processor-rabbit:@filter-processor.version@ +processor.filter.metadata=maven://org.springframework.cloud.stream.app:filter-processor-rabbit:jar:metadata:@filter-processor.version@ +processor.splitter=maven://org.springframework.cloud.stream.app:splitter-processor-rabbit:@splitter-processor.version@ +processor.splitter.metadata=maven://org.springframework.cloud.stream.app:splitter-processor-rabbit:jar:metadata:@splitter-processor.version@ +processor.transform=maven://org.springframework.cloud.stream.app:transform-processor-rabbit:@transform-processor.version@ +processor.transform.metadata=maven://org.springframework.cloud.stream.app:transform-processor-rabbit:jar:metadata:@transform-processor.version@ + diff --git a/applications/apps-metadata/stream-apps-docs/README.md b/applications/apps-metadata/stream-apps-docs/README.md new file mode 100644 index 00000000..9ce4320c --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/README.md @@ -0,0 +1,2 @@ +# docs +app starter docs aggregator diff --git a/applications/apps-metadata/stream-apps-docs/pom.xml b/applications/apps-metadata/stream-apps-docs/pom.xml new file mode 100644 index 00000000..e72e8bd3 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/pom.xml @@ -0,0 +1,325 @@ + + + 4.0.0 + stream-apps-docs + stream-apps-docs + Stream Apps Docs + + org.springframework.cloud.stream.app + stream-apps-release-train + Fahrenheit.BUILD-SNAPSHOT + + + + 0.2.1.RELEASE + + + + + io.spring.docresources + spring-doc-resources + ${docs.resources.version} + zip + true + + + io.pivotal.java.function + splitter-function + ${java-functions.version} + + + io.pivotal.java.function + spel-function + ${java-functions.version} + + + io.pivotal.java.function + counter-consumer + ${java-functions.version} + + + io.pivotal.java.function + jdbc-consumer + ${java-functions.version} + + + io.pivotal.java.function + mongodb-consumer + ${java-functions.version} + + + io.pivotal.java.function + cassandra-consumer + ${java-functions.version} + + + io.pivotal.java.function + rabbit-consumer + ${java-functions.version} + + + io.pivotal.java.function + log-consumer + ${java-functions.version} + + + io.pivotal.java.function + time-supplier + ${java-functions.version} + + + io.pivotal.java.function + jdbc-supplier + ${java-functions.version} + + + io.pivotal.java.function + http-supplier + ${java-functions.version} + + + io.pivotal.java.function + mongodb-supplier + ${java-functions.version} + + + + + full + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + prepare-package + + true + + io.pivotal.java.function:* + + false + true + ${basedir}/src/main/javadoc/spring-javadoc.css + + https://docs.spring.io/spring-framework/docs/${spring.version}/javadoc-api/ + https://docs.spring.io/spring-shell/docs/current/api/ + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-doc-resources + + unpack-dependencies + + generate-resources + + io.spring.docresources + spring-doc-resources + zip + true + ${project.build.directory}/refdocs/ + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-asciidoc-resources + generate-resources + + copy-resources + + + ${project.build.directory}/refdocs/ + + + src/main/asciidoc + false + + + + + + + + org.asciidoctor + asciidoctor-maven-plugin + 1.5.6 + + + org.asciidoctor + asciidoctorj-pdf + 1.5.0-alpha.16 + + + + + + html + generate-resources + + process-asciidoc + + + html5 + highlight.js + ${project.build.directory}/contents/reference/html + false + + ${project.version} + shared + true + font + true + css/ + spring.css + js/highlight + github + + + + + + pdf + generate-resources + + process-asciidoc + + + pdf + + ${project.build.directory}/contents/reference/pdf + coderay + + + + + + + ${project.build.directory}/refdocs/ + index.adoc + book + + ${project.version} + ${project.name} + ${project.version} + true + 4 + true + ${project.basedir} + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + ant-contrib + ant-contrib + 1.0b3 + + + ant + ant + + + + + org.apache.ant + ant-nodeps + 1.8.1 + + + org.tigris.antelope + antelopetasks + 3.2.10 + + + + + package-and-attach-docs-zip + package + + run + + + + + + + + + + + + + setup-maven-properties + validate + + run + + + true + + + + + + + + + + + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.9.1 + + + attach-zip + + attach-artifact + + + + + ${project.build.directory}/${project.artifactId}-${project.version}.zip + zip;zip.type=docs;zip.deployed=false + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/.gitignore b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/.gitignore new file mode 100644 index 00000000..bbc34111 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/.gitignore @@ -0,0 +1,2 @@ +*.html +*.css diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/Guardfile b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/Guardfile new file mode 100644 index 00000000..bdd4d729 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/Guardfile @@ -0,0 +1,20 @@ +require 'asciidoctor' +require 'erb' + +guard 'shell' do + watch(/.*\.adoc$/) {|m| + Asciidoctor.render_file('index.adoc', \ + :in_place => true, \ + :safe => Asciidoctor::SafeMode::UNSAFE, \ + :attributes=> { \ + 'source-highlighter' => 'prettify', \ + 'icons' => 'font', \ + 'linkcss'=> 'true', \ + 'copycss' => 'true', \ + 'doctype' => 'book'}) + } +end + +guard 'livereload' do + watch(%r{^.+\.(css|js|html)$}) +end diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/appendix.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/appendix.adoc new file mode 100644 index 00000000..3027ed6c --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/appendix.adoc @@ -0,0 +1,2 @@ +[[appendix]] += Appendices diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/contributing.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/contributing.adoc new file mode 100644 index 00000000..bcdf6c00 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/contributing.adoc @@ -0,0 +1,42 @@ +[[contributing]] +== Contributing + +Spring Cloud is released under the non-restrictive Apache 2.0 license, +and follows a very standard Github development process, using Github +tracker for issues and merging pull requests into master. If you want +to contribute even something trivial please do not hesitate, but +follow the guidelines below. + +=== Sign the Contributor License Agreement +Before we accept a non-trivial patch or pull request we will need you to sign the +https://support.springsource.com/spring_committer_signup[contributor's agreement]. +Signing the contributor's agreement does not grant anyone commit rights to the main +repository, but it does mean that we can accept your contributions, and you will get an +author credit if we do. Active contributors might be asked to join the core team, and +given the ability to merge pull requests. + +=== Code Conventions and Housekeeping +None of these is essential for a pull request, but they will all help. They can also be +added after the original pull request but before a merge. + +* Use the Spring Framework code format conventions. If you use Eclipse + you can import formatter settings using the + `eclipse-code-formatter.xml` file from the + https://github.com/spring-cloud/build/tree/master/eclipse-coding-conventions.xml[Spring + Cloud Build] project. If using IntelliJ, you can use the + https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter + Plugin] to import the same file. +* Make sure all new `.java` files to have a simple Javadoc class comment with at least an + `@author` tag identifying you, and preferably at least a paragraph on what the class is + for. +* Add the ASF license header comment to all new `.java` files (copy from existing files + in the project) +* Add yourself as an `@author` to the .java files that you modify substantially (more + than cosmetic changes). +* Add some Javadocs and, if you change the namespace, some XSD doc elements. +* A few unit tests would help a lot as well -- someone has to do it. +* If no-one else is using your branch, please rebase it against the current master (or + other target branch in the main project). +* When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], + if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit + message (where XXXX is the issue number). diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/app-starter-naming-conventions.png b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/app-starter-naming-conventions.png new file mode 100644 index 00000000..ce630b78 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/app-starter-naming-conventions.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/logo.png b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/logo.png new file mode 100644 index 00000000..bb1f4f9f Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/logo.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/starters-pom-dependencies.png b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/starters-pom-dependencies.png new file mode 100644 index 00000000..b456ab53 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/images/starters-pom-dependencies.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index-docinfo.xml b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index-docinfo.xml new file mode 100644 index 00000000..011dc8df --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index-docinfo.xml @@ -0,0 +1,14 @@ +Spring Cloud Data Flow +{project-version} + + 2013-2020 + Pivotal Software, Inc. + + + + Copies of this document may be made for your own use and for distribution to + others, provided that you do not charge any fee for such copies and further + provided that each copy contains this Copyright Notice, whether distributed in + print or electronically. + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index.adoc new file mode 100644 index 00000000..0dd752a2 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/index.adoc @@ -0,0 +1,50 @@ += Spring Cloud Stream Applications Reference Guide +Sabby Anandan; Artem Bilan; Marius Bogoevici; Eric Bottard; Mark Fisher; Ilayaperumal Gopinathan; Gunnar Hillert; Mark Pollack; Patrick Peralta; Glenn Renfro; Gary Russell; Thomas Risberg; David Turanski; Janne Valkealahti; Soby Chacko; Christian Tzolov; +:doctype: book +:toc: left +:toclevels: 4 +:source-highlighter: prettify +:numbered: +:icons: font +:hide-uri-scheme: +:docinfo: shared + +:stream-apps-root: https://raw.githubusercontent.com/spring-cloud-stream-app-starters + +:branch: master + +:stream-apps-asciidoc: https://raw.githubusercontent.com/spring-cloud-stream-app-starters/app-starters-release/master/spring-cloud-stream-app-starters-docs/src/main/asciidoc + +:scst-core-version: 3.0.3.RELEASE + +ifdef::backend-html5[] + +Version {project-version} + +(C) 2012-2020 Pivotal Software, Inc. + +_Copies of this document may be made for your own use and for distribution to +others, provided that you do not charge any fee for such copies and further +provided that each copy contains this Copyright Notice, whether distributed in +print or electronically._ + +endif::backend-html5[] + +// ====================================================================================== + += Reference Guide +include::overview.adoc[] + +[[starters]] += Starters +include::sources.adoc[] + +include::processors.adoc[] + +include::sinks.adoc[] + += Appendices +[appendix] +include::contributing.adoc[] + +// ====================================================================================== diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/overview.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/overview.adoc new file mode 100644 index 00000000..c1d12ae8 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/overview.adoc @@ -0,0 +1,102 @@ +[[overview]] + +This section provides you with a detailed overview of the out-of-the-box Spring Cloud Stream Applications. +It assumes familiarity with general Spring Cloud Stream concepts, which you can find in the Spring Cloud Stream https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/{scst-core-version}/[reference documentation]. + +These Spring Cloud Stream Applications provide you with out-of-the-box Spring Cloud Stream utility applications that you can run independently or with Spring Cloud Data Flow. They include: + +* Connectors (sources, processors, and sinks) for a variety of middleware technologies, including message brokers, storage (relational, non-relational, filesystem). +* Adapters for various network protocols. +* Generic processors that you can customize with https://docs.spring.io/spring/docs/4.2.x/spring-framework-reference/html/expressions.html[Spring Expression Language (SpEL)] or by scripting. + +You can find a detailed listing of all the applications and their options in the <> section of this guide. + +== Pre-built Applications + +Out-of-the-box applications are Spring Boot applications that include a Binder implementation on top of the basic logic of the app (a function for example) -- a fully functional uber-jar. +These https://docs.spring.io/spring-boot/docs/current-SNAPSHOT/reference/html/getting-started.html#getting-started-first-application-executable-jar[uber-jars] include the minimal code required for standalone execution. +For each function application, the project provides a prebuilt version for Apache Kafka and Rabbit MQ Binders. + +[NOTE] +Prebuilt applications are generated according to the https://github.com/spring-cloud/spring-cloud-app-starters-maven-plugins/tree/master/spring-cloud-stream-app-maven-plugin[stream apps generator Maven plugin]. + +[[classification]] +== Classification + +Based on their target application type, they can be either: + +* A _source_ that connects to an external resource to poll and receive data that is published to the default "`output`" channel; +* A _processor_ that receives data from an "`input`" channel and processes it, sending the result on the default "`output`" channel; +* A _sink_ that connects to an external resource to send the received data to the default "`input`" channel. + +The prebuilt applications follow a naming convention: `--`. For example, `rabbit-sink-kafka` is a _Rabbit sink_ that uses the Kafka binder that is running with Kafka as the middleware. + +=== Maven and Docker Access + +The core functionality of the applications is available as functions. +See the https://github.com/pivotal/java-functions[Java Functions] repository for more details. +Prebuilt applications are available as Maven artifacts. +You can download the executable jar artifacts from the Spring Maven repositories. +The root directory of the Maven repository that hosts release versions is https://repo.spring.io/release/org/springframework/cloud/stream/app/. +From there, you can navigate to the latest released version of a specific app -- for example, link:https://repo.spring.io/release/org/springframework/cloud/stream/app/log-sink-rabbit/2.0.2.RELEASE/log-sink-rabbit-1.1.1.RELEASE.jar[log-sink-rabbit-2.0.2.RELEASE.jar]. +You need to use the link:https://repo.spring.io/milestone/org/springframework/cloud/stream/app[Milestone] and link:https://repo.spring.io/snapshot/org/springframework/cloud/stream/app[Snapshot] repository locations for Milestone and Snapshot executable jar artifacts. + +The Docker versions of the applications are available in Docker Hub, at `https://hub.docker.com/r/springcloudstream/`. +Naming and versioning follows the same general conventions as Maven -- for example: + +==== +[source,bash] +---- +docker pull springcloudstream/cassandra-sink-kafka +---- +==== + +The preceding command pulls the latest Docker image of the _Cassandra sink_ with the Kafka binder. + +=== Building the Artifacts + +You can build the project and generate the artifacts (including the prebuilt applications) on your own. +This is useful if you want to deploy the artifacts locally or add additional features. +If you are at the root of the repository, https://github.com/spring-cloud-stream-app-starters/stream-applications[steam-applications], doing a maven build generates the entire binder based apps. +If you do not want to do that and instead only are interested in a certain application, then `cd` into the right module and invoke the build from there. +Then run the following Maven command: + +==== +[source,bash] +---- +mvn clean package +---- +==== + +This command generates the applications. By default, the generated projects are placed under a directory called `apps`. +There, you can find the binder based applications, which you can then build and run. + +== Patching Pre-built Applications + +If you are looking to patch the pre-built applications to accommodate the addition of new dependencies, you can use the following example as the reference. +To add `mysql` driver to `jdbc-sink` application: + +. Clone the GitHub repository at https://github.com/spring-cloud-stream-app-starters/stream-applications + +. Open it in an IDE and make the necessary changes in the right generator project. The repository is organized as `source-apps-generator`, `sink-apps-generator`, and `processor-apps-generator`. ++ +Find the module that you want to patch and make the changes. For example, you can add the following to the generator plugin's configuration in the pom.xml: ++ +==== +[source,xml] +---- + + + mysql + mysql-connector-java + 5.1.37 + + + org.springframework.cloud + spring-cloud-stream-binder-rabbit + + +---- +==== + +. Generate the binder based apps as specified above and build the apps. diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/pom-dependencies.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/pom-dependencies.adoc new file mode 100644 index 00000000..20b30195 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/pom-dependencies.adoc @@ -0,0 +1,23 @@ +[[pom-dependencies]] +== Stream Apps POM Dependencies + +Following diagram highlights some of the important `Stream Apps` POM dependencies. + +image::{stream-apps-asciidoc}/images/starters-pom-dependencies.png[PomDependencies, scaledwidth="100%"] + +The dependencies are grouped in three categories: + +* *Core Spring libraries* - represent the core framework libraries such as `Spring Boot`, `Spring Integration`, +`Spring Cloud`. The "Bill Of Materials" (BOM) patterns is used throughout the stack to decouple the dependency +management from the lifecycle configurations. +The `app-starters-build` parent POM and the `app-starters-core-dependencies` BOM use inherit by all app starters. + +* *App Starters* - libraries that contain the complete configuration of a Spring Cloud Stream application with a specific role +Starters are not executable applications, and are intended to be included in the `Pre-build Apps`, along with a Binder +implementation. +The App Starter root pom (`[my-app-name]-app-starters-build`) inherit all compile-tme configuration for its parent +the core `app-starters-build`. Starer's BOM `[my-app-name]-app-dependencies` is used to manage starter's own dependencies. + +* *Pre-build App* - pre-build Spring Boot applications that include the app starters and a Binder implementation. + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/processors.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/processors.adoc new file mode 100644 index 00000000..a11c52da --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/processors.adoc @@ -0,0 +1,15 @@ +[[spring-cloud-stream-modules-processors]] +== Processors + +:leveloffset: +2 + +[[spring-cloud-stream-modules-filter]] +include::{stream-apps-root}/stream-applications/{branch}/processor/filter-processor/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-splitter]] +include::{stream-apps-root}/stream-applications/{branch}/processor/splitter-processor/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-transform]] +include::{stream-apps-root}/stream-applications/{branch}/processor/transform-processor/README.adoc[tags=ref-doc] + +:leveloffset: -2 + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sinks.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sinks.adoc new file mode 100644 index 00000000..17572625 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sinks.adoc @@ -0,0 +1,19 @@ +[[spring-cloud-stream-modules-sinks]] +== Sinks + +:leveloffset: +2 + +[[spring-cloud-stream-modules-cassandra-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/cassandra-sink/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-counter-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/counter-sink/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-jdbc-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/jdbc-sink/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-log-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/log-sink/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-mongodb-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/mongodb-sink/README.adoc[tags=ref-doc] +[[spring-cloud-stream-modules-rabbit-sink]] +include::{stream-apps-root}/stream-applications/{branch}/sink/rabbit-sink/README.adoc[tags=ref-doc] + +:leveloffset: -2 diff --git a/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sources.adoc b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sources.adoc new file mode 100644 index 00000000..8743618c --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/asciidoc/sources.adoc @@ -0,0 +1,20 @@ +[[sources]] +== Sources + +:leveloffset: +2 + + +[[spring-cloud-stream-modules-http-source]] +include::{stream-apps-root}/stream-applications/{branch}/source/http-source/README.adoc[tags=ref-doc] + +[[spring-cloud-stream-modules-jdbc-source]] +include::{stream-apps-root}/stream-applications/{branch}/source/jdbc-source/README.adoc[tags=ref-doc] + +[[spring-cloud-stream-modules-mongodb-source]] +include::{stream-apps-root}/stream-applications/{branch}/source/mongodb-source/README.adoc[tags=ref-doc] + +[[spring-cloud-stream-modules-time-source]] +include::{stream-apps-root}/stream-applications/{branch}/source/time-source/README.adoc[tags=ref-doc] + +:leveloffset: -2 + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/highlight.css b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/highlight.css new file mode 100644 index 00000000..ffefef72 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/highlight.css @@ -0,0 +1,35 @@ +/* + code highlight CSS resemblign the Eclipse IDE default color schema + @author Costin Leau +*/ + +.hl-keyword { + color: #7F0055; + font-weight: bold; +} + +.hl-comment { + color: #3F5F5F; + font-style: italic; +} + +.hl-multiline-comment { + color: #3F5FBF; + font-style: italic; +} + +.hl-tag { + color: #3F7F7F; +} + +.hl-attribute { + color: #7F007F; +} + +.hl-value { + color: #2A00FF; +} + +.hl-string { + color: #2A00FF; +} \ No newline at end of file diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-multipage.css b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-multipage.css new file mode 100644 index 00000000..0c484531 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-multipage.css @@ -0,0 +1,9 @@ +@IMPORT url("manual.css"); + +body.firstpage { + background: url("../images/background.png") no-repeat center top; +} + +div.part h1 { + border-top: none; +} diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-singlepage.css b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-singlepage.css new file mode 100644 index 00000000..4a7fd140 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual-singlepage.css @@ -0,0 +1,6 @@ +@IMPORT url("manual.css"); + +body { + background: url("../images/background.png") no-repeat center top; +} + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual.css b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual.css new file mode 100644 index 00000000..0ecbe2e8 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/css/manual.css @@ -0,0 +1,344 @@ +@IMPORT url("highlight.css"); + +html { + padding: 0pt; + margin: 0pt; +} + +body { + color: #333333; + margin: 15px 30px; + font-family: Helvetica, Arial, Freesans, Clean, Sans-serif; + line-height: 1.6; + -webkit-font-smoothing: antialiased; +} + +code { + font-size: 16px; + font-family: Consolas, "Liberation Mono", Courier, monospace; +} + +:not(a)>code { + color: #6D180B; +} + +:not(pre)>code { + background-color: #F2F2F2; + border: 1px solid #CCCCCC; + border-radius: 4px; + padding: 1px 3px 0; + text-shadow: none; + white-space: nowrap; +} + +body>*:first-child { + margin-top: 0 !important; +} + +div { + margin: 0pt; +} + +hr { + border: 1px solid #CCCCCC; + background: #CCCCCC; +} + +h1,h2,h3,h4,h5,h6 { + color: #000000; + cursor: text; + font-weight: bold; + margin: 30px 0 10px; + padding: 0; +} + +h1,h2,h3 { + margin: 40px 0 10px; +} + +h1 { + margin: 70px 0 30px; + padding-top: 20px; +} + +div.part h1 { + border-top: 1px dotted #CCCCCC; +} + +h1,h1 code { + font-size: 32px; +} + +h2,h2 code { + font-size: 24px; +} + +h3,h3 code { + font-size: 20px; +} + +h4,h1 code,h5,h5 code,h6,h6 code { + font-size: 18px; +} + +div.book,div.chapter,div.appendix,div.part,div.preface { + min-width: 300px; + max-width: 1200px; + margin: 0 auto; +} + +p.releaseinfo { + font-weight: bold; + margin-bottom: 40px; + margin-top: 40px; +} + +div.authorgroup { + line-height: 1; +} + +p.copyright { + line-height: 1; + margin-bottom: -5px; +} + +.legalnotice p { + font-style: italic; + font-size: 14px; + line-height: 1; +} + +div.titlepage+p,div.titlepage+p { + margin-top: 0; +} + +pre { + line-height: 1.0; + color: black; +} + +a { + color: #4183C4; + text-decoration: none; +} + +p { + margin: 15px 0; + text-align: left; +} + +ul,ol { + padding-left: 30px; +} + +li p { + margin: 0; +} + +div.table { + margin: 1em; + padding: 0.5em; + text-align: center; +} + +div.table table,div.informaltable table { + display: table; + width: 100%; +} + +div.table td { + padding-left: 7px; + padding-right: 7px; +} + +.sidebar { + line-height: 1.4; + padding: 0 20px; + background-color: #F8F8F8; + border: 1px solid #CCCCCC; + border-radius: 3px 3px 3px 3px; +} + +.sidebar p.title { + color: #6D180B; +} + +pre.programlisting,pre.screen { + font-size: 15px; + padding: 6px 10px; + background-color: #F8F8F8; + border: 1px solid #CCCCCC; + border-radius: 3px 3px 3px 3px; + clear: both; + overflow: auto; + line-height: 1.4; + font-family: Consolas, "Liberation Mono", Courier, monospace; +} + +table { + border-collapse: collapse; + border-spacing: 0; + border: 1px solid #DDDDDD !important; + border-radius: 4px !important; + border-collapse: separate !important; + line-height: 1.6; +} + +table thead { + background: #F5F5F5; +} + +table tr { + border: none; + border-bottom: none; +} + +table th { + font-weight: bold; +} + +table th,table td { + border: none !important; + padding: 6px 13px; +} + +table tr:nth-child(2n) { + background-color: #F8F8F8; +} + +td p { + margin: 0 0 15px 0; +} + +div.table-contents td p { + margin: 0; +} + +div.important *,div.note *,div.tip *,div.warning *,div.navheader *,div.navfooter *,div.calloutlist * + { + border: none !important; + background: none !important; + margin: 0; +} + +div.important p,div.note p,div.tip p,div.warning p { + color: #6F6F6F; + line-height: 1.6; +} + +div.important code,div.note code,div.tip code,div.warning code { + background-color: #F2F2F2 !important; + border: 1px solid #CCCCCC !important; + border-radius: 4px !important; + padding: 1px 3px 0 !important; + text-shadow: none !important; + white-space: nowrap !important; +} + +.note th,.tip th,.warning th { + display: none; +} + +.note tr:first-child td,.tip tr:first-child td,.warning tr:first-child td + { + border-right: 1px solid #CCCCCC !important; + padding-top: 10px; +} + +div.calloutlist p,div.calloutlist td { + padding: 0; + margin: 0; +} + +div.calloutlist>table>tbody>tr>td:first-child { + padding-left: 10px; + width: 30px !important; +} + +div.important,div.note,div.tip,div.warning { + margin-left: 0px !important; + margin-right: 20px !important; + margin-top: 20px; + margin-bottom: 20px; + padding-top: 10px; + padding-bottom: 10px; +} + +div.toc { + line-height: 1.2; +} + +dl,dt { + margin-top: 1px; + margin-bottom: 0; +} + +div.toc>dl>dt { + font-size: 32px; + font-weight: bold; + margin: 30px 0 10px 0; + display: block; +} + +div.toc>dl>dd>dl>dt { + font-size: 24px; + font-weight: bold; + margin: 20px 0 10px 0; + display: block; +} + +div.toc>dl>dd>dl>dd>dl>dt { + font-weight: bold; + font-size: 20px; + margin: 10px 0 0 0; +} + +tbody.footnotes * { + border: none !important; +} + +div.footnote p { + margin: 0; + line-height: 1; +} + +div.footnote p sup { + margin-right: 6px; + vertical-align: middle; +} + +div.navheader { + border-bottom: 1px solid #CCCCCC; +} + +div.navfooter { + border-top: 1px solid #CCCCCC; +} + +.title { + margin-left: -1em; + padding-left: 1em; +} + +.title>a { + position: absolute; + visibility: hidden; + display: block; + font-size: 0.85em; + margin-top: 0.05em; + margin-left: -1em; + vertical-align: text-top; + color: black; +} + +.title>a:before { + content: "\00A7"; +} + +.title:hover>a,.title>a:hover,.title:hover>a:hover { + visibility: visible; +} + +.title:focus>a,.title>a:focus,.title:focus>a:focus { + outline: 0; +} diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/background.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/background.png new file mode 100644 index 00000000..d4195e5b Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/background.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/1.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/1.png new file mode 100644 index 00000000..7d473430 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/1.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/10.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/10.png new file mode 100644 index 00000000..997bbc82 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/10.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/11.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/11.png new file mode 100644 index 00000000..ce47dac3 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/11.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/12.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/12.png new file mode 100644 index 00000000..31daf4e2 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/12.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/13.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/13.png new file mode 100644 index 00000000..14021a89 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/13.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/14.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/14.png new file mode 100644 index 00000000..64014b75 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/14.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/15.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/15.png new file mode 100644 index 00000000..0d65765f Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/15.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/2.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/2.png new file mode 100644 index 00000000..5d09341b Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/2.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/3.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/3.png new file mode 100644 index 00000000..ef7b7004 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/3.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/4.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/4.png new file mode 100644 index 00000000..adb8364e Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/4.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/5.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/5.png new file mode 100644 index 00000000..4d7eb460 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/5.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/6.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/6.png new file mode 100644 index 00000000..0ba694af Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/6.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/7.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/7.png new file mode 100644 index 00000000..472e96f8 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/7.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/8.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/8.png new file mode 100644 index 00000000..5e60973c Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/8.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/9.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/9.png new file mode 100644 index 00000000..a0676d26 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/callouts/9.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/caution.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/caution.png new file mode 100644 index 00000000..8a5e4fca Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/caution.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/cover.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/cover.png new file mode 100644 index 00000000..b0db312f Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/cover.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/important.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/important.png new file mode 100644 index 00000000..ec54df65 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/important.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.png new file mode 100644 index 00000000..bb1f4f9f Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.svg b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.svg new file mode 100644 index 00000000..e4624f71 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/logo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/note.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/note.png new file mode 100644 index 00000000..88d997b1 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/note.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/tip.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/tip.png new file mode 100644 index 00000000..6530abb4 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/tip.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/warning.png b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/warning.png new file mode 100644 index 00000000..0d5b5244 Binary files /dev/null and b/applications/apps-metadata/stream-apps-docs/src/main/docbook/images/warning.png differ diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/common.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/common.xsl new file mode 100644 index 00000000..5198d622 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/common.xsl @@ -0,0 +1,45 @@ + + + + + + + + 1 + 0 + 1 + + + + images/ + .png + + + book toc,title + 3 + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/epub.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/epub.xsl new file mode 100644 index 00000000..5484bb99 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/epub.xsl @@ -0,0 +1,31 @@ + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-multipage.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-multipage.xsl new file mode 100644 index 00000000..f9f5b272 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-multipage.xsl @@ -0,0 +1,73 @@ + + + + + + + + + + css/manual-multipage.css + + '5' + '1' + + + + + + + + + + + + + + + + + + + + firstpage + + + + + + + + + + + + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-singlepage.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-singlepage.xsl new file mode 100644 index 00000000..c7c0381b --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html-singlepage.xsl @@ -0,0 +1,30 @@ + + + + + + + + + + css/manual-singlepage.css + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html.xsl new file mode 100644 index 00000000..9f036995 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/html.xsl @@ -0,0 +1,141 @@ + + + + + + + + + + + 1 + + + 1 + + + + 120 + images/callouts/ + .png + + + text/css + + text-align: left + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + +

+

Authors

+ +
+ + + + + + + + + + + + + + + + + + # + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/pdf.xsl b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/pdf.xsl new file mode 100644 index 00000000..8278faa3 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/pdf.xsl @@ -0,0 +1,591 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + auto + + + + + underline + #204060 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + Copyright © + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -5em + -5em + 8pt + + + + + + + + + + + + + + + please define title in your docbook file! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 8pt + + + + + + + + + + + + + + + + + + + + + + + + + please define title in your docbook file! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + 0 + 0 + + + + false + + + Helvetica + 10 + 8 + Helvetica + + + 1.4 + + + + left + bold + + + pt + + + + + + + + + + + + + + + 0.6em + 0.6em + 0.6em + + + pt + + 0.1em + 0.1em + 0.1em + + + + 0.4em + 0.4em + 0.4em + + + pt + + 0.1em + 0.1em + 0.1em + + + + 0.4em + 0.4em + 0.4em + + + pt + + 0.1em + 0.1em + 0.1em + + + + 0.3em + 0.3em + 0.3em + + + pt + + 0.1em + 0.1em + 0.1em + + + + + + + + 4pt + 4pt + 4pt + 4pt + + + + 0.1pt + 0.1pt + + + + + + + + + + + + + + + + 7pt + wrap + 1 + + + + 1em + 1em + 1em + 0.1em + 0.1em + 0.1em + + #444444 + solid + 0.1pt + 0.5em + 0.5em + 0.5em + 0.5em + 0.5em + 0.5em + + + + 1 + + #F0F0F0 + + + + 0.1em + 0.1em + 0.1em + 0.1em + 0.1em + 0.1em + + + + 0.5em + 0.5em + 0.5em + 0.1em + 0.1em + 0.1em + + + + #444444 + solid + 0.1pt + #F0F0F0 + + + + + + + normal + italic + + + pt + + false + 0.1em + 0.1em + 0.1em + + + + + + 0 + 1 + + + 90 + + + + + + figure after + example after + equation before + table before + procedure before + + + + 1 + 0pt + + + + + + + + + + + + + + + + + + + + 18pt + + + + 0.1em + 2em + .75pt + solid + #5c5c4f + 0.5em + 1.5em + 1.5em + 1.5em + 1.5em + 1.5em + 1.5em + + + + 10pt + bold + false + always + 0 + + + + 0em + 0em + 0em + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl-config.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl-config.xml new file mode 100644 index 00000000..e354da15 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl-config.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/asciidoc-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/asciidoc-hl.xml new file mode 100644 index 00000000..5478b1d6 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/asciidoc-hl.xml @@ -0,0 +1,41 @@ + + + + + //// + //// + + + // + + + + ^(={1,6} .+)$ + + MULTILINE + + + ^(\.[^\.\s].+)$ + + MULTILINE + + + ^(:!?\w.*?:) + + MULTILINE + + + ^(-|\*{1,5}|\d*\.{1,5})(?= .+$) + + MULTILINE + + + ^(\[.+\])$ + + MULTILINE + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/bourne-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/bourne-hl.xml new file mode 100644 index 00000000..90408acc --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/bourne-hl.xml @@ -0,0 +1,95 @@ + + + + # + + << + ' + " + - + + + + + " + \ + + + ' + \ + + + + 0x + + + + . + + + + + + if + then + else + elif + fi + case + esac + for + while + until + do + done + + exec + shift + exit + times + break + export + trap + continue + readonly + wait + eval + return + + cd + echo + hash + pwd + read + set + test + type + ulimit + umask + unset + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/c-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/c-hl.xml new file mode 100644 index 00000000..7363dba9 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/c-hl.xml @@ -0,0 +1,117 @@ + + + + + /** + */ + + + + + + + + /* + */ + + // + + + # + \ + + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + l + + + + auto + _Bool + break + case + char + _Complex + const + continue + default + do + double + else + enum + extern + float + for + goto + if + _Imaginary + inline + int + long + register + restrict + return + short + signed + sizeof + static + struct + switch + typedef + union + unsigned + void + volatile + while + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/cpp-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/cpp-hl.xml new file mode 100644 index 00000000..b31f7362 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/cpp-hl.xml @@ -0,0 +1,151 @@ + + + + + /** + */ + + + + + + + + /* + */ + + // + + + # + \ + + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + l + + + + + auto + _Bool + break + case + char + _Complex + const + continue + default + do + double + else + enum + extern + float + for + goto + if + _Imaginary + inline + int + long + register + restrict + return + short + signed + sizeof + static + struct + switch + typedef + union + unsigned + void + volatile + while + + asm + dynamic_cast + namespace + reinterpret_cast + try + bool + explicit + new + static_cast + typeid + catch + false + operator + template + typename + class + friend + private + this + using + const_cast + inline + public + throw + virtual + delete + mutable + protected + true + wchar_t + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/csharp-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/csharp-hl.xml new file mode 100644 index 00000000..09373348 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/csharp-hl.xml @@ -0,0 +1,194 @@ + + + + + /** + */ + + + + /// + + + + /* + */ + + // + + + [ + ] + ( + ) + + + + # + \ + + + + + + @" + " + \ + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + d + m + l + + + + abstract + as + base + bool + break + byte + case + catch + char + checked + class + const + continue + decimal + default + delegate + do + double + else + enum + event + explicit + extern + false + finally + fixed + float + for + foreach + goto + if + implicit + in + int + interface + internal + is + lock + long + namespace + new + null + object + operator + out + override + params + private + protected + public + readonly + ref + return + sbyte + sealed + short + sizeof + stackalloc + static + string + struct + switch + this + throw + true + try + typeof + uint + ulong + unchecked + unsafe + ushort + using + virtual + void + volatile + while + + + + add + alias + from + get + global + group + into + join + orderby + partial + remove + select + set + value + where + yield + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/css-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/css-hl.xml new file mode 100644 index 00000000..21439ae6 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/css-hl.xml @@ -0,0 +1,176 @@ + + + + + /* + */ + + + " + \ + + + + ' + \ + + + + . + + + + @charset + @import + @media + @page + + + + - + azimuth + background-attachment + background-color + background-image + background-position + background-repeat + background + border-collapse + border-color + border-spacing + border-style + border-top + border-right + border-bottom + border-left + border-top-color + border-right-color + border-bottom-color + border-left-color + border-top-style + border-right-style + border-bottom-style + border-left-style + border-top-width + border-right-width + border-bottom-width + border-left-width + border-width + border + bottom + caption-side + clear + clip + color + content + counter-increment + counter-reset + cue-after + cue-before + cue + cursor + direction + display + elevation + empty-cells + float + font-family + font-size + font-style + font-variant + font-weight + font + height + left + letter-spacing + line-height + list-style-image + list-style-position + list-style-type + list-style + margin-right + margin-left + margin-top + margin-bottom + margin + max-height + max-width + min-height + min-width + orphans + outline-color + outline-style + outline-width + outline + overflow + padding-top + padding-right + padding-bottom + padding-left + padding + page-break-after + page-break-before + page-break-inside + pause-after + pause-before + pause + pitch-range + pitch + play-during + position + quotes + richness + right + speak-header + speak-numeral + speak-punctuation + speak + speech-rate + stress + table-layout + text-align + text-decoration + text-indent + text-transform + top + unicode-bidi + vertical-align + visibility + voice-family + volume + white-space + widows + width + word-spacing + z-index + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/html-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/html-hl.xml new file mode 100644 index 00000000..366cd46f --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/html-hl.xml @@ -0,0 +1,122 @@ + + + + + + + a + abbr + address + area + article + aside + audio + b + base + bdi + blockquote + body + br + button + caption + canvas + cite + code + command + col + colgroup + dd + del + dialog + div + dl + dt + em + embed + fieldset + figcaption + figure + font + form + footer + h1 + h2 + h3 + h4 + h5 + h6 + head + header + hr + html + i + iframe + img + input + ins + kbd + label + legend + li + link + map + mark + menu + menu + meta + nav + noscript + object + ol + optgroup + option + p + param + pre + q + samp + script + section + select + small + source + span + strong + style + sub + summary + sup + table + tbody + td + textarea + tfoot + th + thead + time + title + tr + track + u + ul + var + video + wbr + xmp + + + + + xsl: + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ini-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ini-hl.xml new file mode 100644 index 00000000..6bd60b3e --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ini-hl.xml @@ -0,0 +1,45 @@ + + + + ; + + + ^(\[.+\]\s*)$ + + MULTILINE + + + + ^(.+)(?==) + + MULTILINE + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/java-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/java-hl.xml new file mode 100644 index 00000000..4fe3c1fe --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/java-hl.xml @@ -0,0 +1,117 @@ + + + + + /** + */ + + + + /* + */ + + // + + " + \ + + + ' + \ + + + @ + ( + ) + + + 0x + + + + . + e + f + d + l + + + + abstract + boolean + break + byte + case + catch + char + class + const + continue + default + do + double + else + extends + final + finally + float + for + goto + if + implements + import + instanceof + int + interface + long + native + new + package + private + protected + public + return + short + static + strictfp + super + switch + synchronized + this + throw + throws + transient + try + void + volatile + while + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/javascript-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/javascript-hl.xml new file mode 100644 index 00000000..91620401 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/javascript-hl.xml @@ -0,0 +1,147 @@ + + + + + /* + */ + + // + + " + \ + + + ' + \ + + + 0x + + + + . + e + + + + break + case + catch + continue + default + delete + do + else + finally + for + function + if + in + instanceof + new + return + switch + this + throw + try + typeof + var + void + while + with + + abstract + boolean + byte + char + class + const + debugger + double + enum + export + extends + final + float + goto + implements + import + int + interface + long + native + package + private + protected + public + short + static + super + synchronized + throws + transient + volatile + + + prototype + + Array + Boolean + Date + Error + EvalError + Function + Math + Number + Object + RangeError + ReferenceError + RegExp + String + SyntaxError + TypeError + URIError + + decodeURI + decodeURIComponent + encodeURI + encodeURIComponent + eval + isFinite + isNaN + parseFloat + parseInt + + Infinity + NaN + undefined + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/json-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/json-hl.xml new file mode 100644 index 00000000..59b9c481 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/json-hl.xml @@ -0,0 +1,37 @@ + + + # + + " + \ + + + ' + \ + + + @ + ( + ) + + + . + e + f + d + l + + + + true + false + + + { + } + , + [ + ] + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/perl-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/perl-hl.xml new file mode 100644 index 00000000..c8a4d590 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/perl-hl.xml @@ -0,0 +1,120 @@ + + + + # + + << + ' + " + + + + " + \ + + + ' + \ + + + + 0x + + + + . + + + + + if + unless + while + until + foreach + else + elsif + for + when + default + given + + caller + continue + die + do + dump + eval + exit + goto + last + next + redo + return + sub + wantarray + + caller + import + local + my + package + use + + do + import + no + package + require + use + + bless + dbmclose + dbmopen + package + ref + tie + tied + untie + use + + and + or + not + eq + ne + lt + gt + le + ge + cmp + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/php-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/php-hl.xml new file mode 100644 index 00000000..3ea15787 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/php-hl.xml @@ -0,0 +1,154 @@ + + + + + /** + */ + + + + + + + + /* + */ + + // + # + + " + \ + + + + ' + \ + + + + <<< + + + 0x + + + + . + e + + + + and + or + xor + __FILE__ + exception + __LINE__ + array + as + break + case + class + const + continue + declare + default + die + do + echo + else + elseif + empty + enddeclare + endfor + endforeach + endif + endswitch + endwhile + eval + exit + extends + for + foreach + function + global + if + include + include_once + isset + list + new + print + require + require_once + return + static + switch + unset + use + var + while + __FUNCTION__ + __CLASS__ + __METHOD__ + final + php_user_filter + interface + implements + extends + public + private + protected + abstract + clone + try + catch + throw + cfunction + old_function + true + false + + namespace + __NAMESPACE__ + goto + __DIR__ + + + + + ?> + <?php + <?= + + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/properties-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/properties-hl.xml new file mode 100644 index 00000000..8205aac3 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/properties-hl.xml @@ -0,0 +1,38 @@ + + + + # + + ^(.+?)(?==|:) + + MULTILINE + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/python-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/python-hl.xml new file mode 100644 index 00000000..72bf0dbb --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/python-hl.xml @@ -0,0 +1,100 @@ + + + + + + @ + ( + ) + + # + + """ + + + + ''' + + + + " + \ + + + ' + \ + + + 0x + l + + + + . + + e + l + + + + and + del + from + not + while + as + elif + global + or + with + assert + else + if + pass + yield + break + except + import + print + class + exec + in + raise + continue + finally + is + return + def + for + lambda + try + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ruby-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ruby-hl.xml new file mode 100644 index 00000000..a2cee724 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/ruby-hl.xml @@ -0,0 +1,109 @@ + + + + # + + << + + + + " + \ + + + %Q{ + } + \ + + + %/ + / + \ + + + ' + \ + + + %q{ + } + \ + + + 0x + + + + . + e + + + + alias + and + BEGIN + begin + break + case + class + def + defined + do + else + elsif + END + end + ensure + false + for + if + in + module + next + nil + not + or + redo + rescue + retry + return + self + super + then + true + undef + unless + until + when + while + yield + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/sql2003-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/sql2003-hl.xml new file mode 100644 index 00000000..65c08a36 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/sql2003-hl.xml @@ -0,0 +1,565 @@ + + + + -- + + /* + */ + + + ' + + + + U' + ' + + + + B' + ' + + + + N' + ' + + + + X' + ' + + + + . + + e + + + + + + A + ABS + ABSOLUTE + ACTION + ADA + ADMIN + AFTER + ALWAYS + ASC + ASSERTION + ASSIGNMENT + ATTRIBUTE + ATTRIBUTES + AVG + BEFORE + BERNOULLI + BREADTH + C + CARDINALITY + CASCADE + CATALOG_NAME + CATALOG + CEIL + CEILING + CHAIN + CHAR_LENGTH + CHARACTER_LENGTH + CHARACTER_SET_CATALOG + CHARACTER_SET_NAME + CHARACTER_SET_SCHEMA + CHARACTERISTICS + CHARACTERS + CHECKED + CLASS_ORIGIN + COALESCE + COBOL + CODE_UNITS + COLLATION_CATALOG + COLLATION_NAME + COLLATION_SCHEMA + COLLATION + COLLECT + COLUMN_NAME + COMMAND_FUNCTION_CODE + COMMAND_FUNCTION + COMMITTED + CONDITION_NUMBER + CONDITION + CONNECTION_NAME + CONSTRAINT_CATALOG + CONSTRAINT_NAME + CONSTRAINT_SCHEMA + CONSTRAINTS + CONSTRUCTORS + CONTAINS + CONVERT + CORR + COUNT + COVAR_POP + COVAR_SAMP + CUME_DIST + CURRENT_COLLATION + CURSOR_NAME + DATA + DATETIME_INTERVAL_CODE + DATETIME_INTERVAL_PRECISION + DEFAULTS + DEFERRABLE + DEFERRED + DEFINED + DEFINER + DEGREE + DENSE_RANK + DEPTH + DERIVED + DESC + DESCRIPTOR + DIAGNOSTICS + DISPATCH + DOMAIN + DYNAMIC_FUNCTION_CODE + DYNAMIC_FUNCTION + EQUALS + EVERY + EXCEPTION + EXCLUDE + EXCLUDING + EXP + EXTRACT + FINAL + FIRST + FLOOR + FOLLOWING + FORTRAN + FOUND + FUSION + G + GENERAL + GO + GOTO + GRANTED + HIERARCHY + IMPLEMENTATION + INCLUDING + INCREMENT + INITIALLY + INSTANCE + INSTANTIABLE + INTERSECTION + INVOKER + ISOLATION + K + KEY_MEMBER + KEY_TYPE + KEY + LAST + LENGTH + LEVEL + LN + LOCATOR + LOWER + M + MAP + MATCHED + MAX + MAXVALUE + MESSAGE_LENGTH + MESSAGE_OCTET_LENGTH + MESSAGE_TEXT + MIN + MINVALUE + MOD + MORE + MUMPS + NAME + NAMES + NESTING + NEXT + NORMALIZE + NORMALIZED + NULLABLE + NULLIF + NULLS + NUMBER + OBJECT + OCTET_LENGTH + OCTETS + OPTION + OPTIONS + ORDERING + ORDINALITY + OTHERS + OVERLAY + OVERRIDING + PAD + PARAMETER_MODE + PARAMETER_NAME + PARAMETER_ORDINAL_POSITION + PARAMETER_SPECIFIC_CATALOG + PARAMETER_SPECIFIC_NAME + PARAMETER_SPECIFIC_SCHEMA + PARTIAL + PASCAL + PATH + PERCENT_RANK + PERCENTILE_CONT + PERCENTILE_DISC + PLACING + PLI + POSITION + POWER + PRECEDING + PRESERVE + PRIOR + PRIVILEGES + PUBLIC + RANK + READ + RELATIVE + REPEATABLE + RESTART + RETURNED_CARDINALITY + RETURNED_LENGTH + RETURNED_OCTET_LENGTH + RETURNED_SQLSTATE + ROLE + ROUTINE_CATALOG + ROUTINE_NAME + ROUTINE_SCHEMA + ROUTINE + ROW_COUNT + ROW_NUMBER + SCALE + SCHEMA_NAME + SCHEMA + SCOPE_CATALOG + SCOPE_NAME + SCOPE_SCHEMA + SECTION + SECURITY + SELF + SEQUENCE + SERIALIZABLE + SERVER_NAME + SESSION + SETS + SIMPLE + SIZE + SOURCE + SPACE + SPECIFIC_NAME + SQRT + STATE + STATEMENT + STDDEV_POP + STDDEV_SAMP + STRUCTURE + STYLE + SUBCLASS_ORIGIN + SUBSTRING + SUM + TABLE_NAME + TABLESAMPLE + TEMPORARY + TIES + TOP_LEVEL_COUNT + TRANSACTION_ACTIVE + TRANSACTION + TRANSACTIONS_COMMITTED + TRANSACTIONS_ROLLED_BACK + TRANSFORM + TRANSFORMS + TRANSLATE + TRIGGER_CATALOG + TRIGGER_NAME + TRIGGER_SCHEMA + TRIM + TYPE + UNBOUNDED + UNCOMMITTED + UNDER + UNNAMED + USAGE + USER_DEFINED_TYPE_CATALOG + USER_DEFINED_TYPE_CODE + USER_DEFINED_TYPE_NAME + USER_DEFINED_TYPE_SCHEMA + VIEW + WORK + WRITE + ZONE + + ADD + ALL + ALLOCATE + ALTER + AND + ANY + ARE + ARRAY + AS + ASENSITIVE + ASYMMETRIC + AT + ATOMIC + AUTHORIZATION + BEGIN + BETWEEN + BIGINT + BINARY + BLOB + BOOLEAN + BOTH + BY + CALL + CALLED + CASCADED + CASE + CAST + CHAR + CHARACTER + CHECK + CLOB + CLOSE + COLLATE + COLUMN + COMMIT + CONNECT + CONSTRAINT + CONTINUE + CORRESPONDING + CREATE + CROSS + CUBE + CURRENT_DATE + CURRENT_DEFAULT_TRANSFORM_GROUP + CURRENT_PATH + CURRENT_ROLE + CURRENT_TIME + CURRENT_TIMESTAMP + CURRENT_TRANSFORM_GROUP_FOR_TYPE + CURRENT_USER + CURRENT + CURSOR + CYCLE + DATE + DAY + DEALLOCATE + DEC + DECIMAL + DECLARE + DEFAULT + DELETE + DEREF + DESCRIBE + DETERMINISTIC + DISCONNECT + DISTINCT + DOUBLE + DROP + DYNAMIC + EACH + ELEMENT + ELSE + END + END-EXEC + ESCAPE + EXCEPT + EXEC + EXECUTE + EXISTS + EXTERNAL + FALSE + FETCH + FILTER + FLOAT + FOR + FOREIGN + FREE + FROM + FULL + FUNCTION + GET + GLOBAL + GRANT + GROUP + GROUPING + HAVING + HOLD + HOUR + IDENTITY + IMMEDIATE + IN + INDICATOR + INNER + INOUT + INPUT + INSENSITIVE + INSERT + INT + INTEGER + INTERSECT + INTERVAL + INTO + IS + ISOLATION + JOIN + LANGUAGE + LARGE + LATERAL + LEADING + LEFT + LIKE + LOCAL + LOCALTIME + LOCALTIMESTAMP + MATCH + MEMBER + MERGE + METHOD + MINUTE + MODIFIES + MODULE + MONTH + MULTISET + NATIONAL + NATURAL + NCHAR + NCLOB + NEW + NO + NONE + NOT + NULL + NUMERIC + OF + OLD + ON + ONLY + OPEN + OR + ORDER + OUT + OUTER + OUTPUT + OVER + OVERLAPS + PARAMETER + PARTITION + PRECISION + PREPARE + PRIMARY + PROCEDURE + RANGE + READS + REAL + RECURSIVE + REF + REFERENCES + REFERENCING + REGR_AVGX + REGR_AVGY + REGR_COUNT + REGR_INTERCEPT + REGR_R2 + REGR_SLOPE + REGR_SXX + REGR_SXY + REGR_SYY + RELEASE + RESULT + RETURN + RETURNS + REVOKE + RIGHT + ROLLBACK + ROLLUP + ROW + ROWS + SAVEPOINT + SCROLL + SEARCH + SECOND + SELECT + SENSITIVE + SESSION_USER + SET + SIMILAR + SMALLINT + SOME + SPECIFIC + SPECIFICTYPE + SQL + SQLEXCEPTION + SQLSTATE + SQLWARNING + START + STATIC + SUBMULTISET + SYMMETRIC + SYSTEM_USER + SYSTEM + TABLE + THEN + TIME + TIMESTAMP + TIMEZONE_HOUR + TIMEZONE_MINUTE + TO + TRAILING + TRANSLATION + TREAT + TRIGGER + TRUE + UESCAPE + UNION + UNIQUE + UNKNOWN + UNNEST + UPDATE + UPPER + USER + USING + VALUE + VALUES + VAR_POP + VAR_SAMP + VARCHAR + VARYING + WHEN + WHENEVER + WHERE + WIDTH_BUCKET + WINDOW + WITH + WITHIN + WITHOUT + YEAR + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/yaml-hl.xml b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/yaml-hl.xml new file mode 100644 index 00000000..a28008ec --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/docbook/xsl/xslthl/yaml-hl.xml @@ -0,0 +1,47 @@ + + + # + + " + \ + + + ' + \ + + + @ + ( + ) + + + . + e + f + d + l + + + + true + false + + + { + } + , + [ + ] + + + + ^(---)$ + + MULTILINE + + + ^(.+?)(?==|:) + + MULTILINE + + diff --git a/applications/apps-metadata/stream-apps-docs/src/main/javadoc/spring-javadoc.css b/applications/apps-metadata/stream-apps-docs/src/main/javadoc/spring-javadoc.css new file mode 100644 index 00000000..06ad4227 --- /dev/null +++ b/applications/apps-metadata/stream-apps-docs/src/main/javadoc/spring-javadoc.css @@ -0,0 +1,599 @@ +/* Javadoc style sheet */ +/* +Overall document style +*/ + +@import url('resources/fonts/dejavu.css'); + +body { + background-color:#ffffff; + color:#353833; + font-family:'DejaVu Sans', Arial, Helvetica, sans-serif; + font-size:14px; + margin:0; +} +a:link, a:visited { + text-decoration:none; + color:#4A6782; +} +a:hover, a:focus { + text-decoration:none; + color:#bb7a2a; +} +a:active { + text-decoration:none; + color:#4A6782; +} +a[name] { + color:#353833; +} +a[name]:hover { + text-decoration:none; + color:#353833; +} +pre { + font-family:'DejaVu Sans Mono', monospace; + font-size:14px; +} +h1 { + font-size:20px; +} +h2 { + font-size:18px; +} +h3 { + font-size:16px; + font-style:italic; +} +h4 { + font-size:13px; +} +h5 { + font-size:12px; +} +h6 { + font-size:11px; +} +ul { + list-style-type:disc; +} +code, tt { + font-family:'DejaVu Sans Mono', monospace; + font-size:14px; + padding-top:4px; + margin-top:8px; + line-height:1.4em; +} +dt code { + font-family:'DejaVu Sans Mono', monospace; + font-size:14px; + padding-top:4px; +} +table tr td dt code { + font-family:'DejaVu Sans Mono', monospace; + font-size:14px; + vertical-align:top; + padding-top:4px; +} +sup { + font-size:8px; +} +/* +Document title and Copyright styles +*/ +.clear { + clear:both; + height:0px; + overflow:hidden; +} +.aboutLanguage { + float:right; + padding:0px 21px; + font-size:11px; + z-index:200; + margin-top:-9px; +} +.legalCopy { + margin-left:.5em; +} +.bar a, .bar a:link, .bar a:visited, .bar a:active { + color:#FFFFFF; + text-decoration:none; +} +.bar a:hover, .bar a:focus { + color:#bb7a2a; +} +.tab { + background-color:#0066FF; + color:#ffffff; + padding:8px; + width:5em; + font-weight:bold; +} +/* +Navigation bar styles +*/ +.bar { + background-color:#4D7A97; + color:#FFFFFF; + padding:.8em .5em .4em .8em; + height:auto;/*height:1.8em;*/ + font-size:11px; + margin:0; +} +.topNav { + background-color:#4D7A97; + color:#FFFFFF; + float:left; + padding:0; + width:100%; + clear:right; + height:2.8em; + padding-top:10px; + overflow:hidden; + font-size:12px; +} +.bottomNav { + margin-top:10px; + background-color:#4D7A97; + color:#FFFFFF; + float:left; + padding:0; + width:100%; + clear:right; + height:2.8em; + padding-top:10px; + overflow:hidden; + font-size:12px; +} +.subNav { + background-color:#dee3e9; + float:left; + width:100%; + overflow:hidden; + font-size:12px; +} +.subNav div { + clear:left; + float:left; + padding:0 0 5px 6px; + text-transform:uppercase; +} +ul.navList, ul.subNavList { + float:left; + margin:0 25px 0 0; + padding:0; +} +ul.navList li{ + list-style:none; + float:left; + padding: 5px 6px; + text-transform:uppercase; +} +ul.subNavList li{ + list-style:none; + float:left; +} +.topNav a:link, .topNav a:active, .topNav a:visited, .bottomNav a:link, .bottomNav a:active, .bottomNav a:visited { + color:#FFFFFF; + text-decoration:none; + text-transform:uppercase; +} +.topNav a:hover, .bottomNav a:hover { + text-decoration:none; + color:#bb7a2a; + text-transform:uppercase; +} +.navBarCell1Rev { + background-color:#F8981D; + color:#253441; + margin: auto 5px; +} +.skipNav { + position:absolute; + top:auto; + left:-9999px; + overflow:hidden; +} +/* +Page header and footer styles +*/ +.header, .footer { + clear:both; + margin:0 20px; + padding:5px 0 0 0; +} +.indexHeader { + margin:10px; + position:relative; +} +.indexHeader span{ + margin-right:15px; +} +.indexHeader h1 { + font-size:13px; +} +.title { + color:#2c4557; + margin:10px 0; +} +.subTitle { + margin:5px 0 0 0; +} +.header ul { + margin:0 0 15px 0; + padding:0; +} +.footer ul { + margin:20px 0 5px 0; +} +.header ul li, .footer ul li { + list-style:none; + font-size:13px; +} +/* +Heading styles +*/ +div.details ul.blockList ul.blockList ul.blockList li.blockList h4, div.details ul.blockList ul.blockList ul.blockListLast li.blockList h4 { + background-color:#dee3e9; + border:1px solid #d0d9e0; + margin:0 0 6px -8px; + padding:7px 5px; +} +ul.blockList ul.blockList ul.blockList li.blockList h3 { + background-color:#dee3e9; + border:1px solid #d0d9e0; + margin:0 0 6px -8px; + padding:7px 5px; +} +ul.blockList ul.blockList li.blockList h3 { + padding:0; + margin:15px 0; +} +ul.blockList li.blockList h2 { + padding:0px 0 20px 0; +} +/* +Page layout container styles +*/ +.contentContainer, .sourceContainer, .classUseContainer, .serializedFormContainer, .constantValuesContainer { + clear:both; + padding:10px 20px; + position:relative; +} +.indexContainer { + margin:10px; + position:relative; + font-size:12px; +} +.indexContainer h2 { + font-size:13px; + padding:0 0 3px 0; +} +.indexContainer ul { + margin:0; + padding:0; +} +.indexContainer ul li { + list-style:none; + padding-top:2px; +} +.contentContainer .description dl dt, .contentContainer .details dl dt, .serializedFormContainer dl dt { + font-size:12px; + font-weight:bold; + margin:10px 0 0 0; + color:#4E4E4E; +} +.contentContainer .description dl dd, .contentContainer .details dl dd, .serializedFormContainer dl dd { + margin:5px 0 10px 0px; + font-size:14px; + font-family:'DejaVu Sans Mono',monospace; +} +.serializedFormContainer dl.nameValue dt { + margin-left:1px; + font-size:1.1em; + display:inline; + font-weight:bold; +} +.serializedFormContainer dl.nameValue dd { + margin:0 0 0 1px; + font-size:1.1em; + display:inline; +} +/* +List styles +*/ +ul.horizontal li { + display:inline; + font-size:0.9em; +} +ul.inheritance { + margin:0; + padding:0; +} +ul.inheritance li { + display:inline; + list-style:none; +} +ul.inheritance li ul.inheritance { + margin-left:15px; + padding-left:15px; + padding-top:1px; +} +ul.blockList, ul.blockListLast { + margin:10px 0 10px 0; + padding:0; +} +ul.blockList li.blockList, ul.blockListLast li.blockList { + list-style:none; + margin-bottom:15px; + line-height:1.4; +} +ul.blockList ul.blockList li.blockList, ul.blockList ul.blockListLast li.blockList { + padding:0px 20px 5px 10px; + border:1px solid #ededed; + background-color:#f8f8f8; +} +ul.blockList ul.blockList ul.blockList li.blockList, ul.blockList ul.blockList ul.blockListLast li.blockList { + padding:0 0 5px 8px; + background-color:#ffffff; + border:none; +} +ul.blockList ul.blockList ul.blockList ul.blockList li.blockList { + margin-left:0; + padding-left:0; + padding-bottom:15px; + border:none; +} +ul.blockList ul.blockList ul.blockList ul.blockList li.blockListLast { + list-style:none; + border-bottom:none; + padding-bottom:0; +} +table tr td dl, table tr td dl dt, table tr td dl dd { + margin-top:0; + margin-bottom:1px; +} +/* +Table styles +*/ +.overviewSummary, .memberSummary, .typeSummary, .useSummary, .constantsSummary, .deprecatedSummary { + width:100%; + border-left:1px solid #EEE; + border-right:1px solid #EEE; + border-bottom:1px solid #EEE; +} +.overviewSummary, .memberSummary { + padding:0px; +} +.overviewSummary caption, .memberSummary caption, .typeSummary caption, +.useSummary caption, .constantsSummary caption, .deprecatedSummary caption { + position:relative; + text-align:left; + background-repeat:no-repeat; + color:#253441; + font-weight:bold; + clear:none; + overflow:hidden; + padding:0px; + padding-top:10px; + padding-left:1px; + margin:0px; + white-space:pre; +} +.overviewSummary caption a:link, .memberSummary caption a:link, .typeSummary caption a:link, +.useSummary caption a:link, .constantsSummary caption a:link, .deprecatedSummary caption a:link, +.overviewSummary caption a:hover, .memberSummary caption a:hover, .typeSummary caption a:hover, +.useSummary caption a:hover, .constantsSummary caption a:hover, .deprecatedSummary caption a:hover, +.overviewSummary caption a:active, .memberSummary caption a:active, .typeSummary caption a:active, +.useSummary caption a:active, .constantsSummary caption a:active, .deprecatedSummary caption a:active, +.overviewSummary caption a:visited, .memberSummary caption a:visited, .typeSummary caption a:visited, +.useSummary caption a:visited, .constantsSummary caption a:visited, .deprecatedSummary caption a:visited { + color:#FFFFFF; +} +.overviewSummary caption span, .memberSummary caption span, .typeSummary caption span, +.useSummary caption span, .constantsSummary caption span, .deprecatedSummary caption span { + white-space:nowrap; + padding-top:5px; + padding-left:12px; + padding-right:12px; + padding-bottom:7px; + display:inline-block; + float:left; + background-color:#F8981D; + border: none; + height:16px; +} +.memberSummary caption span.activeTableTab span { + white-space:nowrap; + padding-top:5px; + padding-left:12px; + padding-right:12px; + margin-right:3px; + display:inline-block; + float:left; + background-color:#F8981D; + height:16px; +} +.memberSummary caption span.tableTab span { + white-space:nowrap; + padding-top:5px; + padding-left:12px; + padding-right:12px; + margin-right:3px; + display:inline-block; + float:left; + background-color:#4D7A97; + height:16px; +} +.memberSummary caption span.tableTab, .memberSummary caption span.activeTableTab { + padding-top:0px; + padding-left:0px; + padding-right:0px; + background-image:none; + float:none; + display:inline; +} +.overviewSummary .tabEnd, .memberSummary .tabEnd, .typeSummary .tabEnd, +.useSummary .tabEnd, .constantsSummary .tabEnd, .deprecatedSummary .tabEnd { + display:none; + width:5px; + position:relative; + float:left; + background-color:#F8981D; +} +.memberSummary .activeTableTab .tabEnd { + display:none; + width:5px; + margin-right:3px; + position:relative; + float:left; + background-color:#F8981D; +} +.memberSummary .tableTab .tabEnd { + display:none; + width:5px; + margin-right:3px; + position:relative; + background-color:#4D7A97; + float:left; + +} +.overviewSummary td, .memberSummary td, .typeSummary td, +.useSummary td, .constantsSummary td, .deprecatedSummary td { + text-align:left; + padding:0px 0px 12px 10px; + width:100%; +} +th.colOne, th.colFirst, th.colLast, .useSummary th, .constantsSummary th, +td.colOne, td.colFirst, td.colLast, .useSummary td, .constantsSummary td{ + vertical-align:top; + padding-right:0px; + padding-top:8px; + padding-bottom:3px; +} +th.colFirst, th.colLast, th.colOne, .constantsSummary th { + background:#dee3e9; + text-align:left; + padding:8px 3px 3px 7px; +} +td.colFirst, th.colFirst { + white-space:nowrap; + font-size:13px; +} +td.colLast, th.colLast { + font-size:13px; +} +td.colOne, th.colOne { + font-size:13px; +} +.overviewSummary td.colFirst, .overviewSummary th.colFirst, +.overviewSummary td.colOne, .overviewSummary th.colOne, +.memberSummary td.colFirst, .memberSummary th.colFirst, +.memberSummary td.colOne, .memberSummary th.colOne, +.typeSummary td.colFirst{ + width:25%; + vertical-align:top; +} +td.colOne a:link, td.colOne a:active, td.colOne a:visited, td.colOne a:hover, td.colFirst a:link, td.colFirst a:active, td.colFirst a:visited, td.colFirst a:hover, td.colLast a:link, td.colLast a:active, td.colLast a:visited, td.colLast a:hover, .constantValuesContainer td a:link, .constantValuesContainer td a:active, .constantValuesContainer td a:visited, .constantValuesContainer td a:hover { + font-weight:bold; +} +.tableSubHeadingColor { + background-color:#EEEEFF; +} +.altColor { + background-color:#FFFFFF; +} +.rowColor { + background-color:#EEEEEF; +} +/* +Content styles +*/ +.description pre { + margin-top:0; +} +.deprecatedContent { + margin:0; + padding:10px 0; +} +.docSummary { + padding:0; +} + +ul.blockList ul.blockList ul.blockList li.blockList h3 { + font-style:normal; +} + +div.block { + font-size:14px; + font-family:'DejaVu Serif', Georgia, "Times New Roman", Times, serif; +} + +td.colLast div { + padding-top:0px; +} + + +td.colLast a { + padding-bottom:3px; +} +/* +Formatting effect styles +*/ +.sourceLineNo { + color:green; + padding:0 30px 0 0; +} +h1.hidden { + visibility:hidden; + overflow:hidden; + font-size:10px; +} +.block { + display:block; + margin:3px 10px 2px 0px; + color:#474747; +} +.deprecatedLabel, .descfrmTypeLabel, .memberNameLabel, .memberNameLink, +.overrideSpecifyLabel, .packageHierarchyLabel, .paramLabel, .returnLabel, +.seeLabel, .simpleTagLabel, .throwsLabel, .typeNameLabel, .typeNameLink { + font-weight:bold; +} +.deprecationComment, .emphasizedPhrase, .interfaceName { + font-style:italic; +} + +div.block div.block span.deprecationComment, div.block div.block span.emphasizedPhrase, +div.block div.block span.interfaceName { + font-style:normal; +} + +div.contentContainer ul.blockList li.blockList h2{ + padding-bottom:0px; +} + + + +/* +Spring +*/ + +pre.code { + background-color: #F8F8F8; + border: 1px solid #CCCCCC; + border-radius: 3px 3px 3px 3px; + overflow: auto; + padding: 10px; + margin: 4px 20px 2px 0px; +} + +pre.code code, pre.code code * { + font-size: 1em; +} + +pre.code code, pre.code code * { + padding: 0 !important; + margin: 0 !important; +} + diff --git a/applications/pom.xml b/applications/pom.xml new file mode 100644 index 00000000..458eb2d5 --- /dev/null +++ b/applications/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + org.springframework.cloud.stream.app + stream-apps + 3.0.0.BUILD-SNAPSHOT + stream-apps + Infrastructure for stream applications + pom + + + apps-core + apps-metadata + source + sink + processor + + + diff --git a/applications/processor/bridge-processor/README.adoc b/applications/processor/bridge-processor/README.adoc new file mode 100644 index 00000000..82ce758b --- /dev/null +++ b/applications/processor/bridge-processor/README.adoc @@ -0,0 +1,10 @@ +//tag::ref-doc[] += Bridge Processor + +A processor that bridges the input and ouput by simply passing the incoming payload to the outbound. + +=== Payload + +Any + +//end::ref-doc[] diff --git a/applications/processor/bridge-processor/pom.xml b/applications/processor/bridge-processor/pom.xml new file mode 100644 index 00000000..e3b612a0 --- /dev/null +++ b/applications/processor/bridge-processor/pom.xml @@ -0,0 +1,80 @@ + + + 4.0.0 + bridge-processor + bridge-processor + bridge processor apps + 3.0.0.BUILD-SNAPSHOT + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + bridge + processor + ${project.version} + org.springframework.cloud.stream.app.BridgeProcessorConfiguration.class + bridgeFunction + + + + + org.springframework.cloud.stream.app + bridge-processor + ${project.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + diff --git a/applications/processor/bridge-processor/src/main/java/org/springframework/cloud/stream/app/BridgeProcessorConfiguration.java b/applications/processor/bridge-processor/src/main/java/org/springframework/cloud/stream/app/BridgeProcessorConfiguration.java new file mode 100644 index 00000000..22bf47ef --- /dev/null +++ b/applications/processor/bridge-processor/src/main/java/org/springframework/cloud/stream/app/BridgeProcessorConfiguration.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import java.util.function.Function; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + */ +@Configuration +public class BridgeProcessorConfiguration { + + @Bean + public Function bridgeFunction() { + return Function.identity(); + } +} diff --git a/applications/processor/bridge-processor/src/test/java/org/springframework/cloud/stream/app/BridgeProcessorTests.java b/applications/processor/bridge-processor/src/test/java/org/springframework/cloud/stream/app/BridgeProcessorTests.java new file mode 100644 index 00000000..fa011636 --- /dev/null +++ b/applications/processor/bridge-processor/src/test/java/org/springframework/cloud/stream/app/BridgeProcessorTests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class BridgeProcessorTests { + + @Test + public void testFilterProcessor() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(BridgeTestAppConfiguration.class)) + .web(WebApplicationType.NONE) + .run()) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + String inMessage = "hello world"; + processorInput.send(new GenericMessage<>(inMessage.getBytes(StandardCharsets.UTF_8))); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo(inMessage); + } + } + + @EnableAutoConfiguration + @Import({ BridgeProcessorConfiguration.class }) + public static class BridgeTestAppConfiguration { } +} diff --git a/applications/processor/filter-processor/README.adoc b/applications/processor/filter-processor/README.adoc new file mode 100644 index 00000000..ddbe62cd --- /dev/null +++ b/applications/processor/filter-processor/README.adoc @@ -0,0 +1,22 @@ +//tag::ref-doc[] += Filter Processor + +Filter processor enables an application to examine the incoming payload and then applies a predicate against it which decides if the record needs to be continued. +For example, if the incoming payload is of type `String` and you want to filter out anything that has less than five characters, you can run the filter processor as below. + +`java -jar filter-processor-kafka-.jar --spel.function.expression=payload.length() > 4` + +Change kafka to rabbit if you want to run it against RabbitMQ. + +=== Payload + +You can pass any type as payload and then apply SpEL expressions against it to filter. +If the incoming type is `byte[]` and the content type is set to `text/plain` or `application/json`, then the application converts the `byte[]` into `String`. + +== Options + +//tag::configuration-properties[] +$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/processor/filter-processor/pom.xml b/applications/processor/filter-processor/pom.xml new file mode 100644 index 00000000..5331aff5 --- /dev/null +++ b/applications/processor/filter-processor/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + filter-processor + filter-processor + filter processor apps + 3.0.0.BUILD-SNAPSHOT + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.cloud.function + filter-function + ${java-functions.version} + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-starter-json + test + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + filter + processor + ${project.version} + org.springframework.cloud.fn.filter.FilterFunctionConfiguration.class + byteArrayTextToString|filterFunction + + + + + org.springframework.cloud.fn + filter-function + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + diff --git a/applications/processor/filter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/processor/filter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..ff1bc322 --- /dev/null +++ b/applications/processor/filter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=SpelFunctionProperties \ No newline at end of file diff --git a/applications/processor/filter-processor/src/test/java/org/springframework/cloud/stream/app/filter/processor/FilterProcessorTests.java b/applications/processor/filter-processor/src/test/java/org/springframework/cloud/stream/app/filter/processor/FilterProcessorTests.java new file mode 100644 index 00000000..51e8f5d6 --- /dev/null +++ b/applications/processor/filter-processor/src/test/java/org/springframework/cloud/stream/app/filter/processor/FilterProcessorTests.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.filter.processor; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.filter.FilterFunctionConfiguration; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +public class FilterProcessorTests { + + @Test + public void testFilterProcessor() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(FilterProcessorConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=byteArrayTextToString|filterFunction", + "--spel.function.expression=payload.length() > 5")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + String longMessage = "hello world message"; + processorInput.send(new GenericMessage<>(longMessage.getBytes(StandardCharsets.UTF_8))); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo(longMessage); + + String shortMessage = "foo"; + processorInput.send(new GenericMessage<>(shortMessage.getBytes(StandardCharsets.UTF_8))); + Message sourceMessage2 = processorOutput.receive(5000); + assertThat(sourceMessage2).isNull(); + } + } + + @EnableAutoConfiguration + @Import({ FilterFunctionConfiguration.class }) + public static class FilterProcessorConfiguration {} + +} diff --git a/applications/processor/pom.xml b/applications/processor/pom.xml new file mode 100644 index 00000000..6390a810 --- /dev/null +++ b/applications/processor/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + org.springframework.cloud.stream.app + processor + 3.0.0.BUILD-SNAPSHOT + pom + + + bridge-processor + splitter-processor + filter-processor + transform-processor + + + diff --git a/applications/processor/splitter-processor/README.adoc b/applications/processor/splitter-processor/README.adoc new file mode 100644 index 00000000..08834790 --- /dev/null +++ b/applications/processor/splitter-processor/README.adoc @@ -0,0 +1,28 @@ +//tag::ref-doc[] += Splitter Processor + +The splitter app builds upon the concept of the same name in Spring Integration and allows the splitting of a single message into several distinct messages. +The processor uses a function that takes a `Message` as input and then produces a `List` as output based on various properties (see below). +You can use a SpEL expression or a delimiter to specify how you want to split the incoming message. + +=== Payload + +* Incoming payload - `Message + +If the incoming type is `byte[]` and the content type is set to `text/plain` or `application/json`, then the application converts the `byte[]` into `String`. + +* Outgoing payload - `List` + + +== Options + +//tag::configuration-properties[] +$$splitter.apply-sequence$$:: $$Add correlation/sequence information in headers to facilitate later aggregation.$$ *($$Boolean$$, default: `$$true$$`)* +$$splitter.charset$$:: $$The charset to use when converting bytes in text-based files to String.$$ *($$String$$, default: `$$$$`)* +$$splitter.delimiters$$:: $$When expression is null, delimiters to use when tokenizing {@link String} payloads.$$ *($$String$$, default: `$$$$`)* +$$splitter.expression$$:: $$A SpEL expression for splitting payloads.$$ *($$String$$, default: `$$$$`)* +$$splitter.file-markers$$:: $$Set to true or false to use a {@code FileSplitter} (to split text-based files by line) that includes (or not) beginning/end of file markers.$$ *($$Boolean$$, default: `$$$$`)* +$$splitter.markers-json$$:: $$When 'fileMarkers == true', specify if they should be produced as FileSplitter.FileMarker objects or JSON.$$ *($$Boolean$$, default: `$$true$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/processor/splitter-processor/pom.xml b/applications/processor/splitter-processor/pom.xml new file mode 100644 index 00000000..2f6f8aaf --- /dev/null +++ b/applications/processor/splitter-processor/pom.xml @@ -0,0 +1,99 @@ + + + 4.0.0 + splitter-processor + 3.0.0.BUILD-SNAPSHOT + splitter-processor + splitter processor apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.cloud.function + splitter-function + ${java-functions.version} + + + io.pivotal.java.function + payload-converter-function + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + splitter + processor + ${project.version} + org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration.class + byteArrayTextToString|splitterFunction + + + + + org.springframework.cloud.fn + payload-converter-function + ${java-functions.version} + + + org.springframework.cloud.fn + splitter-function + ${java-functions.version} + + + + + true + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/processor/splitter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/processor/splitter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..70bc4605 --- /dev/null +++ b/applications/processor/splitter-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.splitter.SplitterFunctionProperties \ No newline at end of file diff --git a/applications/processor/splitter-processor/src/test/java/org/springframework/cloud/stream/app/splitter/processor/SplitterProcessorTests.java b/applications/processor/splitter-processor/src/test/java/org/springframework/cloud/stream/app/splitter/processor/SplitterProcessorTests.java new file mode 100644 index 00000000..e36d4a45 --- /dev/null +++ b/applications/processor/splitter-processor/src/test/java/org/springframework/cloud/stream/app/splitter/processor/SplitterProcessorTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.splitter.processor; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SplitterProcessorTests { + + @Test + public void testSplitterProcessor() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(SplitterProcessorConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=byteArrayTextToString|splitterFunction", + "--splitter.expression=payload.split(',')")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + String payload = "hello,world,message"; + processorInput.send(new GenericMessage<>(payload.getBytes(StandardCharsets.UTF_8))); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload(), StandardCharsets.UTF_8)).isEqualTo("hello"); + sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload(), StandardCharsets.UTF_8)).isEqualTo("world"); + sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload(), StandardCharsets.UTF_8)).isEqualTo("message"); + } + } + + @EnableAutoConfiguration + @Import({ SplitterFunctionConfiguration.class }) + public static class SplitterProcessorConfiguration {} +} diff --git a/applications/processor/splitter-processor/src/test/resources/META-INF/spring.binders b/applications/processor/splitter-processor/src/test/resources/META-INF/spring.binders new file mode 100644 index 00000000..737ae2f4 --- /dev/null +++ b/applications/processor/splitter-processor/src/test/resources/META-INF/spring.binders @@ -0,0 +1,2 @@ +integration:\ +org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration \ No newline at end of file diff --git a/applications/processor/transform-processor/README.adoc b/applications/processor/transform-processor/README.adoc new file mode 100644 index 00000000..2441adf2 --- /dev/null +++ b/applications/processor/transform-processor/README.adoc @@ -0,0 +1,23 @@ +//tag::ref-doc[] += Transform Processor + +Transformer processor allows you to convert the message payload structure based on a SpEL expression. + +Here is an example of how you can run this application. + +`java -jar filter-processor-kafka-.jar --spel.function.expression=toUpperCase()` + +Change kafka to rabbit if you want to run it against RabbitMQ. + + +=== Payload + +Incoming message can contain any type of payload. + +== Options + +//tag::configuration-properties[] +$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/processor/transform-processor/pom.xml b/applications/processor/transform-processor/pom.xml new file mode 100644 index 00000000..233e2616 --- /dev/null +++ b/applications/processor/transform-processor/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + transform-processor + 3.0.0.BUILD-SNAPSHOT + transform-processor + transform processor apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + org.springframework.cloud.function + spel-function + ${java-functions.version} + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + transform + processor + ${project.version} + org.springfamework.cloud.fn.spel.SpelFunctionConfiguration.class + byteArrayTextToString|spelFunction + + + + + org.springframework.cloud.fn + spel-function + ${java-functions.version} + + + + + true + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/processor/transform-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/processor/transform-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..aa9720e2 --- /dev/null +++ b/applications/processor/transform-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.spel.SpelFunctionProperties \ No newline at end of file diff --git a/applications/processor/transform-processor/src/test/java/org/springframework/cloud/stream/app/transform/processor/TransformProcessorTests.java b/applications/processor/transform-processor/src/test/java/org/springframework/cloud/stream/app/transform/processor/TransformProcessorTests.java new file mode 100644 index 00000000..b2198dd8 --- /dev/null +++ b/applications/processor/transform-processor/src/test/java/org/springframework/cloud/stream/app/transform/processor/TransformProcessorTests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.transform.processor; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.spel.SpelFunctionConfiguration; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TransformProcessorTests { + + @Test + public void testTransformProcessor() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(TransformProcessorConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=byteArrayTextToString|spelFunction", + "--spel.function.expression=payload.toUpperCase()")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + String payload = "hello world"; + processorInput.send(new GenericMessage<>(payload.getBytes(StandardCharsets.UTF_8))); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo(payload.toUpperCase()); + } + } + + @EnableAutoConfiguration + @Import({ SpelFunctionConfiguration.class }) + public static class TransformProcessorConfiguration {} + +} diff --git a/applications/sink/cassandra-sink/README.adoc b/applications/sink/cassandra-sink/README.adoc new file mode 100644 index 00000000..add39433 --- /dev/null +++ b/applications/sink/cassandra-sink/README.adoc @@ -0,0 +1,44 @@ +//tag::ref-doc[] += Cassandra Sink + +This sink application writes the content of each message it receives into Cassandra. + +It expects a payload of JSON String and uses it’s properties to map to table columns. + +=== Payload +A JSON String or byte array representing the entity (or a list of entities) to be persisted. + +== Options + +The **$$cassandra$$** $$sink$$ has the following options: + + +//tag::configuration-properties[] +$$cassandra.cluster.create-keyspace$$:: $$Flag to create (or not) keyspace on application startup.$$ *($$Boolean$$, default: `$$false$$`)* +$$cassandra.cluster.entity-base-packages$$:: $$Base packages to scan for entities annotated with Table annotations.$$ *($$String[]$$, default: `$$[]$$`)* +$$cassandra.cluster.init-script$$:: $$Resource with CQL scripts (delimited by ';') to initialize keyspace schema.$$ *($$Resource$$, default: `$$$$`)* +$$cassandra.cluster.skip-ssl-validation$$:: $$Flag to validate the Servers' SSL certs$$ *($$Boolean$$, default: `$$false$$`)* +$$cassandra.consistency-level$$:: $$The consistency level for write operation.$$ *($$ConsistencyLevel$$, default: `$$$$`)* +$$cassandra.ingest-query$$:: $$Ingest Cassandra query.$$ *($$String$$, default: `$$$$`)* +$$cassandra.query-type$$:: $$QueryType for Cassandra Sink.$$ *($$Type$$, default: `$$$$`, possible values: `INSERT`,`UPDATE`,`DELETE`,`STATEMENT`)* +$$cassandra.statement-expression$$:: $$Expression in Cassandra query DSL style.$$ *($$Expression$$, default: `$$$$`)* +$$cassandra.ttl$$:: $$Time-to-live option of WriteOptions.$$ *($$Integer$$, default: `$$0$$`)* +$$spring.data.cassandra.cluster-name$$:: $$$$ *($$String$$, default: `$$$$`)* +$$spring.data.cassandra.compression$$:: $$Compression supported by the Cassandra binary protocol.$$ *($$Compression$$, default: `$$none$$`, possible values: `LZ4`,`SNAPPY`,`NONE`)* +$$spring.data.cassandra.connect-timeout$$:: $$Socket option: connection time out.$$ *($$Duration$$, default: `$$$$`)* +$$spring.data.cassandra.consistency-level$$:: $$Queries consistency level.$$ *($$DefaultConsistencyLevel$$, default: `$$$$`, possible values: `ANY`,`ONE`,`TWO`,`THREE`,`QUORUM`,`ALL`,`LOCAL_ONE`,`LOCAL_QUORUM`,`EACH_QUORUM`,`SERIAL`,`LOCAL_SERIAL`)* +$$spring.data.cassandra.contact-points$$:: $$Cluster node addresses in the form 'host:port'.$$ *($$List$$, default: `$$[127.0.0.1:9042]$$`)* +$$spring.data.cassandra.fetch-size$$:: $$$$ *($$Integer$$, default: `$$$$`)* +$$spring.data.cassandra.keyspace-name$$:: $$Keyspace name to use.$$ *($$String$$, default: `$$$$`)* +$$spring.data.cassandra.local-datacenter$$:: $$Datacenter that is considered "local". Contact points should be from this datacenter.$$ *($$String$$, default: `$$$$`)* +$$spring.data.cassandra.page-size$$:: $$Queries default page size.$$ *($$Integer$$, default: `$$5000$$`)* +$$spring.data.cassandra.password$$:: $$Login password of the server.$$ *($$String$$, default: `$$$$`)* +$$spring.data.cassandra.read-timeout$$:: $$Socket option: read time out.$$ *($$Duration$$, default: `$$$$`)* +$$spring.data.cassandra.schema-action$$:: $$Schema action to take at startup.$$ *($$String$$, default: `$$none$$`)* +$$spring.data.cassandra.serial-consistency-level$$:: $$Queries serial consistency level.$$ *($$DefaultConsistencyLevel$$, default: `$$$$`, possible values: `ANY`,`ONE`,`TWO`,`THREE`,`QUORUM`,`ALL`,`LOCAL_ONE`,`LOCAL_QUORUM`,`EACH_QUORUM`,`SERIAL`,`LOCAL_SERIAL`)* +$$spring.data.cassandra.session-name$$:: $$Name of the Cassandra session.$$ *($$String$$, default: `$$$$`)* +$$spring.data.cassandra.ssl$$:: $$Enable SSL support.$$ *($$Boolean$$, default: `$$false$$`)* +$$spring.data.cassandra.username$$:: $$Login user of the server.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/cassandra-sink/pom.xml b/applications/sink/cassandra-sink/pom.xml new file mode 100644 index 00000000..01cfde05 --- /dev/null +++ b/applications/sink/cassandra-sink/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + cassandra-sink + 3.0.0.BUILD-SNAPSHOT + cassandra-sink + cassandra sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + cassandra-consumer + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + cassandra + sink + ${project.version} + org.springframework.cloud.fn.consumer.cassandra.CassandraConsumerConfiguration.class + + + + + org.springframework.cloud.fn + cassandra-consumer + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/cassandra-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/cassandra-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..38811295 --- /dev/null +++ b/applications/sink/cassandra-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,3 @@ +configuration-properties.classes=CassandraConsumerProperties, \ + CassandraClusterProperties, \ + org.springframework.boot.autoconfigure.cassandra.CassandraProperties \ No newline at end of file diff --git a/applications/sink/counter-sink/README.adoc b/applications/sink/counter-sink/README.adoc new file mode 100644 index 00000000..1d68e25d --- /dev/null +++ b/applications/sink/counter-sink/README.adoc @@ -0,0 +1,60 @@ +//tag::ref-doc[] +:images-asciidoc: https://github.com/spring-cloud-stream-app-starters/stream-applications/raw/master/sink/counter-sink/src/main/resources += Counter Sink + +Counter that compute multiple counters from the received messages. It leverages the Micrometer library and can use various popular TSDB to persist the counter values. + +By default the Counter Sink increments the `message`.`name` counter on every received message. The `message-counter-enabled` allows you to disable this counter when required. + +If tag expressions are provided (via the `counter.tag.expression.= property) then the `name` counter is incremented. Note that each SpEL expression can evaluate into multiple values resulting into multiple counter increments (one fore every value resolved). + +If fixed tags are provided they are include in all message and expression counter increment measurements. + +Counter's implementation is based on the https://micrometer.io/[Micrometer library] which is a Vendor-neutral application metrics facade that supports the most popular monitoring systems. +See the https://micrometer.io/docs[Micrometer documentation] for the list of supported monitoring systems. Starting with Spring Boot 2.0, Micrometer is the instrumentation library powering the delivery of application metrics from Spring Boot. + +All Spring Cloud Stream App Starters are configured to support two of the most popular monitoring systems, Prometheus and InfluxDB. You can declaratively select which monitoring system to use. +If you are not using Prometheus or InfluxDB, you can customise the App starters to use a different monitoring system as well as include your preferred micrometer monitoring system library in your own custom applications. + +https://grafana.com/[Grafana] is a popular platform for building visualization dashboards. + +To enable Micrometer’s Prometheus meter registry for Spring Cloud Stream application starters, set the following properties. + +``` +management.metrics.export.prometheus.enabled=true +management.endpoints.web.exposure.include=prometheus +``` + +and disable the application’s security which allows for a simple Prometheus configuration to scrape counter information by setting the following property. + +``` +spring.cloud.streamapp.security.enabled=false +``` + +To enable Micrometer’s Influx meter registry for Spring Cloud Stream application starters, set the following property. + +``` +management.metrics.export.influx.enabled=true +management.metrics.export.influx.uri={influxdb-server-url} +``` + +NOTE: if the https://docs.spring.io/spring-cloud-dataflow/docs/2.0.0.BUILD-SNAPSHOT/reference/htmlsingle/#streams-monitoring[Data Flow Server metrics] is enabled then the `Counter` will reuse the exiting configurations. + +Following diagram illustrates Counter's information collection and processing flow. + +image::{images-asciidoc}/MicrometerCounterAppStarter.png[Counter Architecture, scaledwidth="70%"] + +=== Payload + +== Options + +//tag::configuration-properties[] +$$counter.amount-expression$$:: $$A SpEL expression (against the incoming Message) to derive the amount to add to the counter. If not set the counter is incremented by 1.0$$ *($$Expression$$, default: `$$$$`)* +$$counter.message-counter-enabled$$:: $$Enables counting the number of messages processed. Uses the 'message.' counter name prefix to distinct it form the expression based counter. The message counter includes the fixed tags when provided.$$ *($$Boolean$$, default: `$$true$$`)* +$$counter.name$$:: $$The name of the counter to increment. The 'name' and 'nameExpression' are mutually exclusive. Only one can be set.$$ *($$String$$, default: `$$$$`)* +$$counter.name-expression$$:: $$A SpEL expression (against the incoming Message) to derive the name of the counter to increment. The 'name' and 'nameExpression' are mutually exclusive. Only one can be set.$$ *($$Expression$$, default: `$$$$`)* +$$counter.tag.expression$$:: $$Computes tags from SpEL expression. Single SpEL expression can produce an array of values, which in turn means distinct name/value tags. Every name/value tag will produce a separate counter increment. Tag expression format is: counter.tag.expression.[tag-name]=[SpEL expression]$$ *($$Map$$, default: `$$$$`)* +$$counter.tag.fixed$$:: $$Custom tags assigned to every counter increment measurements. This is a map so the property convention fixed tags is: counter.tag.fixed.[tag-name]=[tag-value]$$ *($$Map$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/counter-sink/pom.xml b/applications/sink/counter-sink/pom.xml new file mode 100644 index 00000000..d6a6ac9c --- /dev/null +++ b/applications/sink/counter-sink/pom.xml @@ -0,0 +1,98 @@ + + + 4.0.0 + counter-sink + 3.0.0.BUILD-SNAPSHOT + counter-sink + counter sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.cloud.fn + counter-consumer + ${java-functions.version} + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.awaitility + awaitility + test + + + junit + junit + + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + counter + sink + ${project.version} + org.springframework.cloud.fn.consumer.counter.CounterConsumerConfiguration.class + byteArrayTextToString|counterConsumer + + + + org.springframework.cloud.fn + counter-consumer + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/counter-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/counter-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..66583ecc --- /dev/null +++ b/applications/sink/counter-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,2 @@ +configuration-properties.classes=CounterConsumerProperties, \ + CounterConsumerProperties$MetricsTag diff --git a/applications/sink/counter-sink/src/main/resources/MicrometerCounterAppStarter.png b/applications/sink/counter-sink/src/main/resources/MicrometerCounterAppStarter.png new file mode 100644 index 00000000..6dcb5cc3 Binary files /dev/null and b/applications/sink/counter-sink/src/main/resources/MicrometerCounterAppStarter.png differ diff --git a/applications/sink/counter-sink/src/test/java/org/springframework/cloud/stream/app/counter/sink/CounterSinkTests.java b/applications/sink/counter-sink/src/test/java/org/springframework/cloud/stream/app/counter/sink/CounterSinkTests.java new file mode 100644 index 00000000..320cb591 --- /dev/null +++ b/applications/sink/counter-sink/src/test/java/org/springframework/cloud/stream/app/counter/sink/CounterSinkTests.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.counter.sink; + +import java.nio.charset.StandardCharsets; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.cloud.fn.consumer.counter.CounterConsumerConfiguration; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@ExtendWith(OutputCaptureExtension.class) +public class CounterSinkTests { + + @Test + public void testCounterSink(CapturedOutput output) { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(CounterSinkConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=byteArrayTextToString|counterConsumer", + "--counter.name=counter666", + //"--counter.amount-expression=new String(payload).length()", + "--counter.amount-expression=payload.length()", + "--counter.tag.expression.foo='bar'")) { + + SimpleMeterRegistry meterRegistry = context.getBean(SimpleMeterRegistry.class); + + String message = "hello world message"; + InputDestination source = context.getBean(InputDestination.class); + source.send(new GenericMessage<>(message.getBytes(StandardCharsets.UTF_8))); + + Counter counter = meterRegistry.find("counter666").counter(); + assertThat(counter.count()).isEqualTo(message.length()); + assertThat(counter.getId().getTag("foo")).isEqualTo("bar"); + } + } + + @EnableAutoConfiguration + //@Import({CounterConsumerConfiguration.class, PayloadConverterConfiguration.class}) + @Import({CounterConsumerConfiguration.class}) + public static class CounterSinkConfiguration {} + +} diff --git a/applications/sink/file-sink/README.adoc b/applications/sink/file-sink/README.adoc new file mode 100644 index 00000000..d09c637a --- /dev/null +++ b/applications/sink/file-sink/README.adoc @@ -0,0 +1,28 @@ +//tag::ref-doc[] += File Sink + +The file sink app writes each message it receives to a file. + +=== Payload + +* `java.io.File` +* `java.io.InputStream` +* `byte[]` +* `String` + +== Options + +The `file-sink` has the following options: + +//tag::configuration-properties[] +$$file.consumer.binary$$:: $$A flag to indicate whether adding a newline after the write should be suppressed.$$ *($$Boolean$$, default: `$$false$$`)* +$$file.consumer.charset$$:: $$The charset to use when writing text content.$$ *($$String$$, default: `$$UTF-8$$`)* +$$file.consumer.directory$$:: $$The parent directory of the target file.$$ *($$File$$, default: `$$$$`)* +$$file.consumer.directory-expression$$:: $$The expression to evaluate for the parent directory of the target file.$$ *($$String$$, default: `$$$$`)* +$$file.consumer.mode$$:: $$The FileExistsMode to use if the target file already exists.$$ *($$FileExistsMode$$, default: `$$$$`, possible values: `APPEND`,`APPEND_NO_FLUSH`,`FAIL`,`IGNORE`,`REPLACE`,`REPLACE_IF_MODIFIED`)* +$$file.consumer.name$$:: $$The name of the target file.$$ *($$String$$, default: `$$file-consumer$$`)* +$$file.consumer.name-expression$$:: $$The expression to evaluate for the name of the target file.$$ *($$String$$, default: `$$$$`)* +$$file.consumer.suffix$$:: $$The suffix to append to file name.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/file-sink/pom.xml b/applications/sink/file-sink/pom.xml new file mode 100644 index 00000000..b1a8003c --- /dev/null +++ b/applications/sink/file-sink/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + file-sink + 3.0.0.BUILD-SNAPSHOT + file-sink + file sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + file-consumer + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + file + sink + ${project.version} + org.springframework.cloud.fn.consumer.file.FileConsumerConfiguration.class + + + + org.springframework.cloud.fn + file-consumer + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/file-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/file-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..3a01d9ee --- /dev/null +++ b/applications/sink/file-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.consumer.file.FileConsumerProperties \ No newline at end of file diff --git a/applications/sink/file-sink/src/test/java/org/springframework/cloud/stream/app/FileSinkTests.java b/applications/sink/file-sink/src/test/java/org/springframework/cloud/stream/app/FileSinkTests.java new file mode 100644 index 00000000..189d18cb --- /dev/null +++ b/applications/sink/file-sink/src/test/java/org/springframework/cloud/stream/app/FileSinkTests.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import java.io.File; +import java.io.FileReader; +import java.nio.file.Path; + +import org.springframework.cloud.fn.consumer.file.FileConsumerConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class FileSinkTests { + + @TempDir + static Path tempDir; + + @Test + public void testFileSink() throws Exception { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration + .getCompleteConfiguration(FileSinkConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=fileConsumer", + "--file.consumer.name=test", + "--file.consumer.suffix=txt", + "--file.consumer.directory=" + tempDir.toAbsolutePath().toString())) { + + final Message message = MessageBuilder.withPayload("hello").build(); + InputDestination source = context.getBean(InputDestination.class); + source.send(message); + File file = new File(tempDir.toFile(), "test.txt"); + assertThat(file.exists()).isTrue(); + assertThat("hello" + System.lineSeparator()) + .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); + } + } + + @EnableAutoConfiguration + @Import(FileConsumerConfiguration.class) + public static class FileSinkConfiguration {} +} diff --git a/applications/sink/jdbc-sink/README.adoc b/applications/sink/jdbc-sink/README.adoc new file mode 100644 index 00000000..53432af7 --- /dev/null +++ b/applications/sink/jdbc-sink/README.adoc @@ -0,0 +1,58 @@ +//tag::ref-doc[] += JDBC Sink + +JDBC sink allows you to persist incoming payload into an RDBMS database. + +The `jdbc.consumer.columns` property represents pairs of `COLUMN_NAME[:EXPRESSION_FOR_VALUE]` where `EXPRESSION_FOR_VALUE` (together with the colon) is optional. +In this case the value is evaluated via generated expression like `payload.COLUMN_NAME`, so this way we have a direct mapping from object properties to the table column. +For example we have a JSON payload like: +``` +{ + "name": "My Name" + "address": { + "city": "Big City", + "street": "Narrow Alley" + } +} +``` +So, we can insert it into the table with `name`, `city` and `street` structure using the configuration: +``` +--jdbc.consumer.columns=name,city:address.city,street:address.street +``` + +This sink supports batch inserts, as far as supported by the underlying JDBC driver. +Batch inserts are configured via the `batch-size` and `idle-timeout` properties: +Incoming messages are aggregated until `batch-size` messages are present, then inserted as a batch. +If `idle-timeout` milliseconds pass with no new messages, the aggregated batch is inserted even if it is smaller than `batch-size`, capping maximum latency. + +NOTE: The module also uses Spring Boot's https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-configure-datasource[DataSource support] for configuring the database connection, so properties like `spring.datasource.url` _etc._ apply. + +== Examples + +``` +java -jar jdbc-sink.jar --jdbc.consumer.tableName=names --jdbc.consumer.columns=name --spring.datasource.driver-class-name=org.mariadb.jdbc.Driver \ +--spring.datasource.url='jdbc:mysql://localhost:3306/test +``` + +=== Payload + +== Options + +The **$$jdbc$$** $$sink$$ has the following options: + +//tag::configuration-properties[] +$$jdbc.consumer.batch-size$$:: $$Threshold in number of messages when data will be flushed to database table.$$ *($$Integer$$, default: `$$1$$`)* +$$jdbc.consumer.columns$$:: $$The comma separated colon-based pairs of column names and SpEL expressions for values to insert/update. Names are used at initialization time to issue the DDL.$$ *($$String$$, default: `$$payload:payload.toString()$$`)* +$$jdbc.consumer.idle-timeout$$:: $$Idle timeout in milliseconds when data is automatically flushed to database table.$$ *($$Long$$, default: `$$-1$$`)* +$$jdbc.consumer.initialize$$:: $$'true', 'false' or the location of a custom initialization script for the table.$$ *($$String$$, default: `$$false$$`)* +$$jdbc.consumer.table-name$$:: $$The name of the table to write into.$$ *($$String$$, default: `$$messages$$`)* +$$spring.datasource.data$$:: $$Data (DML) script resource references.$$ *($$List$$, default: `$$$$`)* +$$spring.datasource.driver-class-name$$:: $$Fully qualified name of the JDBC driver. Auto-detected based on the URL by default.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.initialization-mode$$:: $$Initialize the datasource with available DDL and DML scripts.$$ *($$DataSourceInitializationMode$$, default: `$$embedded$$`, possible values: `ALWAYS`,`EMBEDDED`,`NEVER`)* +$$spring.datasource.password$$:: $$Login password of the database.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.schema$$:: $$Schema (DDL) script resource references.$$ *($$List$$, default: `$$$$`)* +$$spring.datasource.url$$:: $$JDBC URL of the database.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.username$$:: $$Login username of the database.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/jdbc-sink/pom.xml b/applications/sink/jdbc-sink/pom.xml new file mode 100644 index 00000000..a36c808d --- /dev/null +++ b/applications/sink/jdbc-sink/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + jdbc-sink + 3.0.0.BUILD-SNAPSHOT + jdbc-sink + jdbc sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.cloud.fn + jdbc-consumer + ${java-functions.version} + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + com.h2database + h2 + test + + + org.springframework.boot + spring-boot-starter-jdbc + test + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + jdbc + sink + ${project.version} + org.springframework.cloud.fn.consumer.jdbc.JdbcConsumerConfiguration.class + + + + org.springframework.cloud.fn + jdbc-consumer + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/jdbc-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/jdbc-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..764ba642 --- /dev/null +++ b/applications/sink/jdbc-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,9 @@ +configuration-properties.classes=org.springframework.cloud.fn.consumer.jdbc.JdbcConsumerProperties +configuration-properties.names=\ +spring.datasource.url,\ +spring.datasource.driver-class-name,\ +spring.datasource.username,\ +spring.datasource.password,\ +spring.datasource.schema,\ +spring.datasource.data,\ +spring.datasource.initialization-mode \ No newline at end of file diff --git a/applications/sink/jdbc-sink/src/test/java/org/springframework/cloud/stream/app/jdbc/sink/JdbcSinkTests.java b/applications/sink/jdbc-sink/src/test/java/org/springframework/cloud/stream/app/jdbc/sink/JdbcSinkTests.java new file mode 100644 index 00000000..f75156ea --- /dev/null +++ b/applications/sink/jdbc-sink/src/test/java/org/springframework/cloud/stream/app/jdbc/sink/JdbcSinkTests.java @@ -0,0 +1,94 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.sink; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.consumer.jdbc.JdbcConsumerConfiguration; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.messaging.Message; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JdbcSinkTests { + + @Test + public void testSimpleInserts() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(JdbcSinkConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=jdbcConsumer")) { + + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + InputDestination source = context.getBean(InputDestination.class); + source.send(message); + + final JdbcOperations jdbcOperations = context.getBean(JdbcOperations.class); + String result = jdbcOperations.queryForObject("select payload from messages", String.class); + assertThat(result).isEqualTo(("hello42")); + } + } + + @EnableAutoConfiguration + @Import(JdbcConsumerConfiguration.class) + public static class JdbcSinkConfiguration {} + + static class Payload { + + private String a; + + private Integer b; + + public Payload() { + } + + public Payload(String a, Integer b) { + this.a = a; + this.b = b; + } + + public String getA() { + return a; + } + + public Integer getB() { + return b; + } + + public void setA(String a) { + this.a = a; + } + + public void setB(Integer b) { + this.b = b; + } + + @Override + public String toString() { + return a + b; + } + + } +} diff --git a/applications/sink/jdbc-sink/src/test/resources/schema.sql b/applications/sink/jdbc-sink/src/test/resources/schema.sql new file mode 100644 index 00000000..65e85283 --- /dev/null +++ b/applications/sink/jdbc-sink/src/test/resources/schema.sql @@ -0,0 +1,7 @@ +-- Run by default by Boot infrastructure + +create table messages( + a varchar(2000), + b VARCHAR (2000), + payload VARCHAR (2000) +); diff --git a/applications/sink/log-sink/README.adoc b/applications/sink/log-sink/README.adoc new file mode 100644 index 00000000..44b7671e --- /dev/null +++ b/applications/sink/log-sink/README.adoc @@ -0,0 +1,21 @@ +//tag::ref-doc[] += Log Sink + +The `log` sink uses the application logger to output the data for inspection. + +Please understand that `log` sink uses type-less handler, which affects how the actual logging will be performed. +This means that if the content-type is textual, then raw payload bytes will be converted to String, otherwise raw bytes will be logged. +Please see more info in the https://docs.spring.io/spring-cloud-stream/docs/Elmhurst.RELEASE/reference/htmlsingle/#_content_type_versus_argument_type[user-guide]. + +== Options + +The **$$log$$** $$sink$$ has the following options: + + +//tag::configuration-properties[] +$$log.expression$$:: $$A SpEL expression (against the incoming message) to evaluate as the logged message.$$ *($$String$$, default: `$$payload$$`)* +$$log.level$$:: $$The level at which to log messages.$$ *($$Level$$, default: `$$$$`, possible values: `FATAL`,`ERROR`,`WARN`,`INFO`,`DEBUG`,`TRACE`)* +$$log.name$$:: $$The name of the logger to use.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/log-sink/pom.xml b/applications/sink/log-sink/pom.xml new file mode 100644 index 00000000..fb24a566 --- /dev/null +++ b/applications/sink/log-sink/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + log-sink + 3.0.0.BUILD-SNAPSHOT + log-sink + log sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + log-consumer + ${java-functions.version} + + + org.awaitility + awaitility + test + + + junit + junit + + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + log + sink + ${project.version} + org.springframework.cloud.fn.consumer.log.LogConsumerConfiguration.class + byteArrayTextToString|logConsumer + + + + org.springframework.cloud.fn + log-consumer + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/log-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/log-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..89d8345b --- /dev/null +++ b/applications/sink/log-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.consumer.log.LogConsumerProperties \ No newline at end of file diff --git a/applications/sink/log-sink/src/test/java/org/springframework/cloud/stream/app/LogSinkTests.java b/applications/sink/log-sink/src/test/java/org/springframework/cloud/stream/app/LogSinkTests.java new file mode 100644 index 00000000..9539e5c4 --- /dev/null +++ b/applications/sink/log-sink/src/test/java/org/springframework/cloud/stream/app/LogSinkTests.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import java.nio.charset.StandardCharsets; + +import org.springframework.cloud.fn.consumer.log.LogConsumerConfiguration; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.support.GenericMessage; + +/** + * @author Soby Chacko + */ +@ExtendWith(OutputCaptureExtension.class) +public class LogSinkTests { + + @Test + public void testSourceFromSupplier(CapturedOutput output) { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(LogSinkConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=byteArrayTextToString|logConsumer")) { + + InputDestination source = context.getBean(InputDestination.class); + source.send(new GenericMessage("hello".getBytes(StandardCharsets.UTF_8))); + Awaitility.await().until(output::getOut, value -> value.contains("hello")); + } + } + + @EnableAutoConfiguration + @Import(LogConsumerConfiguration.class) + public static class LogSinkConfiguration {} +} diff --git a/applications/sink/mongodb-sink/README.adoc b/applications/sink/mongodb-sink/README.adoc new file mode 100644 index 00000000..1f78d259 --- /dev/null +++ b/applications/sink/mongodb-sink/README.adoc @@ -0,0 +1,25 @@ +//tag::ref-doc[] += MongoDB Sink + +This sink application ingest incoming data into MongoDB. +This application is fully based on the `MongoDataAutoConfiguration`, so refer to the https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-nosql.html#boot-features-mongodb[Spring Boot MongoDB Support] for more information. + +== Input + +=== Payload + +* Any POJO +* `String` +* `byte[]` + +== Options + +The **$$mongodb$$** $$sink$$ has the following options: + + +//tag::configuration-properties[] +$$mongodb.consumer.collection$$:: $$The MongoDB collection to store data$$ *($$String$$, default: `$$$$`)* +$$mongodb.consumer.collection-expression$$:: $$The SpEL expression to evaluate MongoDB collection$$ *($$Expression$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/mongodb-sink/pom.xml b/applications/sink/mongodb-sink/pom.xml new file mode 100644 index 00000000..9af3540d --- /dev/null +++ b/applications/sink/mongodb-sink/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + mongodb-sink + 3.0.0.BUILD-SNAPSHOT + mongodb-sink + mongodb sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + mongodb-consumer + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + mongodb + sink + ${project.version} + org.springframework.cloud.fn.consumer.mongo.MongoDbConsumerConfiguration.class + + + + org.springframework.cloud.fn + mongodb-consumer + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/mongodb-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/mongodb-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..0570c896 --- /dev/null +++ b/applications/sink/mongodb-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.consumer.mongo.MongoDbConsumerProperties \ No newline at end of file diff --git a/applications/sink/pom.xml b/applications/sink/pom.xml new file mode 100644 index 00000000..d02982b9 --- /dev/null +++ b/applications/sink/pom.xml @@ -0,0 +1,21 @@ + + + 4.0.0 + + org.springframework.cloud.stream.app + sink + 3.0.0.BUILD-SNAPSHOT + pom + + + file-sink + rabbit-sink + log-sink + cassandra-sink + mongodb-sink + counter-sink + jdbc-sink + + diff --git a/applications/sink/rabbit-sink/README.adoc b/applications/sink/rabbit-sink/README.adoc new file mode 100644 index 00000000..d2d06bf0 --- /dev/null +++ b/applications/sink/rabbit-sink/README.adoc @@ -0,0 +1,34 @@ +//tag::ref-doc[] += RabbitMQ Sink + +This module sends messages to RabbitMQ. + +== Options + +The **$$rabbit$$** $$sink$$ has the following options: + +(See the Spring Boot documentation for RabbitMQ connection properties) + +//tag::configuration-properties[] +$$rabbit.converter-bean-name$$:: $$The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you.$$ *($$String$$, default: `$$$$`)* +$$rabbit.exchange$$:: $$Exchange name - overridden by exchangeNameExpression, if supplied.$$ *($$String$$, default: `$$$$`)* +$$rabbit.exchange-expression$$:: $$A SpEL expression that evaluates to an exchange name.$$ *($$Expression$$, default: `$$$$`)* +$$rabbit.mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$[*]$$`)* +$$rabbit.own-connection$$:: $$When true, use a separate connection based on the boot properties.$$ *($$Boolean$$, default: `$$false$$`)* +$$rabbit.persistent-delivery-mode$$:: $$Default delivery mode when 'amqp_deliveryMode' header is not present, true for PERSISTENT.$$ *($$Boolean$$, default: `$$false$$`)* +$$rabbit.routing-key$$:: $$Routing key - overridden by routingKeyExpression, if supplied.$$ *($$String$$, default: `$$$$`)* +$$rabbit.routing-key-expression$$:: $$A SpEL expression that evaluates to a routing key.$$ *($$Expression$$, default: `$$$$`)* +$$spring.rabbitmq.addresses$$:: $$Comma-separated list of addresses to which the client should connect.$$ *($$String$$, default: `$$$$`)* +$$spring.rabbitmq.connection-timeout$$:: $$Connection timeout. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$$$`)* +$$spring.rabbitmq.host$$:: $$RabbitMQ host.$$ *($$String$$, default: `$$localhost$$`)* +$$spring.rabbitmq.password$$:: $$Login to authenticate against the broker.$$ *($$String$$, default: `$$guest$$`)* +$$spring.rabbitmq.port$$:: $$RabbitMQ port.$$ *($$Integer$$, default: `$$5672$$`)* +$$spring.rabbitmq.publisher-confirm-type$$:: $$Type of publisher confirms to use.$$ *($$ConfirmType$$, default: `$$$$`, possible values: `SIMPLE`,`CORRELATED`,`NONE`)* +$$spring.rabbitmq.publisher-returns$$:: $$Whether to enable publisher returns.$$ *($$Boolean$$, default: `$$false$$`)* +$$spring.rabbitmq.requested-channel-max$$:: $$Number of channels per connection requested by the client. Use 0 for unlimited.$$ *($$Integer$$, default: `$$2047$$`)* +$$spring.rabbitmq.requested-heartbeat$$:: $$Requested heartbeat timeout; zero for none. If a duration suffix is not specified, seconds will be used.$$ *($$Duration$$, default: `$$$$`)* +$$spring.rabbitmq.username$$:: $$Login user to authenticate to the broker.$$ *($$String$$, default: `$$guest$$`)* +$$spring.rabbitmq.virtual-host$$:: $$Virtual host to use when connecting to the broker.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/sink/rabbit-sink/pom.xml b/applications/sink/rabbit-sink/pom.xml new file mode 100644 index 00000000..c3146046 --- /dev/null +++ b/applications/sink/rabbit-sink/pom.xml @@ -0,0 +1,112 @@ + + + 4.0.0 + rabbit-sink + 3.0.0.BUILD-SNAPSHOT + rabbit-sink + rabbit sink apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-starter-amqp + test + + + org.springframework.cloud + spring-cloud-stream-binder-rabbit-test-support + test + + + org.springframework.cloud.fn + rabbit-consumer + ${java-functions.version} + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + org.testcontainers + testcontainers + 1.9.1 + test + + + org.testcontainers + rabbitmq + 1.12.5 + test + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + rabbit + sink + ${project.version} + org.springframework.cloud.fn.consumer.rabbit.RabbitConsumerConfiguration.class + + + + + org.springframework.cloud.fn + rabbit-consumer + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/sink/rabbit-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/rabbit-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..c052f6e5 --- /dev/null +++ b/applications/sink/rabbit-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,2 @@ +configuration-properties.classes=RabbitConsumerProperties, \ + org.springframework.boot.autoconfigure.amqp.RabbitProperties \ No newline at end of file diff --git a/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/OwnConnectionTest.java b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/OwnConnectionTest.java new file mode 100644 index 00000000..e81d2b8e --- /dev/null +++ b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/OwnConnectionTest.java @@ -0,0 +1,61 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.rabbit.sink; + +import java.util.function.Consumer; + +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.PortBinding; +import com.github.dockerjava.api.model.Ports; +import org.junit.ClassRule; +import org.junit.jupiter.api.Test; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.Queue; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.containers.GenericContainer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; + +@TestPropertySource(properties = {"rabbit.routingKey=scsapp-testOwn", + "rabbit.own-connection=true"}) +public class OwnConnectionTest extends RabbitSinkIntegrationTests { + + /** + * RabbitMQ + */ +// @ClassRule +// public static GenericContainer rabbitMq = new GenericContainer("rabbitmq:3.5.3") +// .withExposedPorts(5672); + + + + @Test + public void test() { + this.rabbitAdmin.declareQueue( + new Queue("scsapp-testOwn", false, false, true)); + this.bootFactory.resetConnection(); + this.channels.send(MessageBuilder.withPayload("foo".getBytes()) + .build()); + this.rabbitTemplate.setReceiveTimeout(10000); + Message received = this.rabbitTemplate.receive("scsapp-testOwn"); + assertEquals("foo", new String(received.getBody())); + assertThat(this.bootFactory.getCacheProperties().getProperty("localPort")).isEqualTo("0"); + } +} diff --git a/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkIntegrationTests.java b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkIntegrationTests.java new file mode 100644 index 00000000..9077e8ca --- /dev/null +++ b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkIntegrationTests.java @@ -0,0 +1,102 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.rabbit.sink; + +import java.util.function.Consumer; + +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.PortBinding; +import com.github.dockerjava.api.model.Ports; +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.fn.consumer.rabbit.RabbitConsumerConfiguration; +import org.springframework.cloud.fn.consumer.rabbit.RabbitConsumerProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.test.annotation.DirtiesContext; +import org.testcontainers.containers.GenericContainer; + +@SpringBootTest( + properties = {"spring.cloud.stream.function.definition=rabbitConsumer"}, + webEnvironment = SpringBootTest.WebEnvironment.NONE) +@DirtiesContext +@Import(RabbitSinkIntegrationTests.FooConfiguration.class) +abstract class RabbitSinkIntegrationTests { + + @Qualifier("rabbitConsumer-in-0") + @Autowired + protected SubscribableChannel channels; + + @Autowired + protected RabbitConsumerProperties rabbitSinkProperties; + + @Autowired + protected RabbitTemplate rabbitTemplate; + + @Autowired + protected RabbitAdmin rabbitAdmin; + + @Autowired(required = false) + protected MessageConverter myConverter; + + @Autowired + protected CachingConnectionFactory bootFactory; + + static { + Consumer cmd = e -> e.withPortBindings(new PortBinding(Ports.Binding.bindPort(5672), new ExposedPort(5672))); + + GenericContainer rabbitMq = new GenericContainer("rabbitmq:3.5.3") + .withExposedPorts(5672) + .withCreateContainerCmdModifier(cmd); + rabbitMq.start(); + } + + static class FooConfiguration { + + @Bean + public Queue queue() { + return new Queue("scsapp-testq", false, false, true); + } + + @Bean + public DirectExchange exchange() { + return new DirectExchange("scsapp-testex", false, true); + } + + @Bean + public Binding binding() { + return BindingBuilder.bind(queue()).to(exchange()).with("scsapp-testrk"); + } + + } + + @SpringBootApplication + @Import({RabbitConsumerConfiguration.class}) + public static class RabbitSinkConfiguration {} +} diff --git a/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkInvalidConfigTests.java b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkInvalidConfigTests.java new file mode 100644 index 00000000..f02324ad --- /dev/null +++ b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/RabbitSinkInvalidConfigTests.java @@ -0,0 +1,84 @@ +/* + * Copyright 2016-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.rabbit.sink; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.fail; + +import io.pivotal.java.function.rabbit.consumer.RabbitConsumerProperties; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.BeanCreationException; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.bind.validation.BindValidationException; +import org.springframework.boot.context.properties.bind.validation.ValidationErrors; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.FieldError; + +/** + * Tests for RabbitSource with invalid config. + * + * @author Gary Russell + * @author Chris Schaefer + */ +public class RabbitSinkInvalidConfigTests { + + @Test + public void testNoRoutingKey() { + try { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.register(Config.class); + context.refresh(); + context.close(); + fail("BeanCreationException expected"); + } + catch (Exception e) { + assertThat(e, instanceOf(BeanCreationException.class)); + + BindValidationException bindValidationException = (BindValidationException) e.getCause().getCause(); + ValidationErrors validationErrors = bindValidationException.getValidationErrors(); + FieldError fieldError = (FieldError) validationErrors.getAllErrors().get(0); + + assertThat(fieldError.getDefaultMessage(), containsString("routingKey or routingKeyExpression is required")); + } + } + + @Configuration + @EnableConfigurationProperties(RabbitConsumerProperties.class) + static class Config { + + } + +} \ No newline at end of file diff --git a/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/SimpleRoutingKeyAndCustomHeaderTests.java b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/SimpleRoutingKeyAndCustomHeaderTests.java new file mode 100644 index 00000000..872b9ea9 --- /dev/null +++ b/applications/sink/rabbit-sink/src/test/java/org/springframework/cloud/stream/app/rabbit/sink/SimpleRoutingKeyAndCustomHeaderTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.rabbit.sink; + +import org.junit.jupiter.api.Test; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +@TestPropertySource(properties = {"rabbit.routingKey=scsapp-testq", + "rabbit.persistentDeliveryMode=true", + "rabbit.mappedRequestHeaders=STANDARD_REQUEST_HEADERS,bar"}) +public class SimpleRoutingKeyAndCustomHeaderTests extends RabbitSinkIntegrationTests { + + @Test + public void test() { + this.channels.send(MessageBuilder.withPayload("foo".getBytes()) + .setHeader("bar", "baz") + .setHeader("qux", "fiz") + .build()); + this.rabbitTemplate.setReceiveTimeout(10000); + Message received = this.rabbitTemplate.receive("scsapp-testq"); + assertEquals("foo", new String(received.getBody())); + assertEquals("baz", received.getMessageProperties().getHeaders().get("bar")); + assertNull(received.getMessageProperties().getHeaders().get("qux")); + assertEquals(MessageDeliveryMode.PERSISTENT, received.getMessageProperties().getReceivedDeliveryMode()); + } +} diff --git a/applications/source/file-source/README.adoc b/applications/source/file-source/README.adoc new file mode 100644 index 00000000..66209c20 --- /dev/null +++ b/applications/source/file-source/README.adoc @@ -0,0 +1,33 @@ +//tag::ref-doc[] += File Source + +This application polls a directory and sends new files or their contents to the output channel. +The file source provides the contents of a File as a byte array by default. +However, this can be customized using the --file.supplier.mode option: + +* ref Provides a java.io.File reference + +* lines Will split files line-by-line and emit a new message for each line + +* contents The default. Provides the contents of a file as a byte array + +When using `--file.supplier.mode=lines`, you can also provide the additional option `--file.supplier.withMarkers=true`. +If set to true, the underlying FileSplitter will emit additional start-of-file and end-of-file marker messages before and after the actual data. +The payload of these 2 additional marker messages is of type `FileSplitter.FileMarker`. The option withMarkers defaults to false if not explicitly set. + +== Options + +The **$$file$$** $$source$$ has the following options: + +//tag::configuration-properties[] +$$file.consumer.markers-json$$:: $$When 'fileMarkers == true', specify if they should be produced as FileSplitter.FileMarker objects or JSON.$$ *($$Boolean$$, default: `$$true$$`)* +$$file.consumer.mode$$:: $$The FileReadingMode to use for file reading sources. Values are 'ref' - The File object, 'lines' - a message per line, or 'contents' - the contents as bytes.$$ *($$FileReadingMode$$, default: `$$$$`, possible values: `ref`,`lines`,`contents`)* +$$file.consumer.with-markers$$:: $$Set to true to emit start of file/end of file marker messages before/after the data. Only valid with FileReadingMode 'lines'.$$ *($$Boolean$$, default: `$$$$`)* +$$file.supplier.delay-when-empty$$:: $$Duration of delay when no new files are detected.$$ *($$Duration$$, default: `$$1s$$`)* +$$file.supplier.directory$$:: $$The directory to poll for new files.$$ *($$File$$, default: `$$$$`)* +$$file.supplier.filename-pattern$$:: $$A simple ant pattern to match files.$$ *($$String$$, default: `$$$$`)* +$$file.supplier.filename-regex$$:: $$A regex pattern to match files.$$ *($$Pattern$$, default: `$$$$`)* +$$file.supplier.prevent-duplicates$$:: $$Set to true to include an AcceptOnceFileListFilter which prevents duplicates.$$ *($$Boolean$$, default: `$$true$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/source/file-source/pom.xml b/applications/source/file-source/pom.xml new file mode 100644 index 00000000..7c238071 --- /dev/null +++ b/applications/source/file-source/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + file-source + 3.0.0.BUILD-SNAPSHOT + file-source + file source apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + file-supplier + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + file + source + ${project.version} + org.springframework.cloud.fn.supplier.file.FileSupplierConfiguration.class + + + + org.springframework.cloud.fn + file-supplier + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/source/file-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/file-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..f750801d --- /dev/null +++ b/applications/source/file-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,2 @@ +configuration-properties.classes=FileSupplierProperties,\ + FileConsumerProperties \ No newline at end of file diff --git a/applications/source/file-source/src/test/java/org.springframework.cloud.stream.app/FileSourceTests.java b/applications/source/file-source/src/test/java/org.springframework.cloud.stream.app/FileSourceTests.java new file mode 100644 index 00000000..a8554648 --- /dev/null +++ b/applications/source/file-source/src/test/java/org.springframework.cloud.stream.app/FileSourceTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.supplier.file.FileSupplierConfiguration; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +public class FileSourceTests { + + @TempDir + static Path tempDir; + + @Test + public void testFileSource() throws Exception { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration + .getCompleteConfiguration(FileSourceConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=fileSupplier", "--file.supplier.directory=" + tempDir.toAbsolutePath().toString())) { + + Path firstFile = tempDir.resolve("test.file"); + Files.write(firstFile, "testing".getBytes()); + + OutputDestination target = context.getBean(OutputDestination.class); + Message sourceMessage = target.receive(10000); + final String actual = new String(sourceMessage.getPayload()); + assertThat(actual).isEqualTo("testing"); + } + } + + @EnableAutoConfiguration + @Import(FileSupplierConfiguration.class) + public static class FileSourceConfiguration {} +} diff --git a/applications/source/http-source/README.adoc b/applications/source/http-source/README.adoc new file mode 100644 index 00000000..143d8d45 --- /dev/null +++ b/applications/source/http-source/README.adoc @@ -0,0 +1,31 @@ +//tag::ref-doc[] += Http Source + +A source application that listens for HTTP requests and emits the body as a message payload. +If the Content-Type matches `text/*` or `application/json`, the payload will be a String, +otherwise the payload will be a byte array. + +==== Payload: + +If content type matches `text/*` or `application/json` + +* `String` + +If content type does not match `text/*` or `application/json` + +* `byte array` + +== Options + +The **$$http$$** $$source$$ supports the following configuration properties: + +//tag::configuration-properties[] +$$http.cors.allow-credentials$$:: $$Whether the browser should include any cookies associated with the domain of the request being annotated.$$ *($$Boolean$$, default: `$$$$`)* +$$http.cors.allowed-headers$$:: $$List of request headers that can be used during the actual request.$$ *($$String[]$$, default: `$$$$`)* +$$http.cors.allowed-origins$$:: $$List of allowed origins, e.g. "http://domain1.com".$$ *($$String[]$$, default: `$$$$`)* +$$http.mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$$$`)* +$$http.path-pattern$$:: $$HTTP endpoint path mapping.$$ *($$String$$, default: `$$/$$`)* +$$server.port$$:: $$Server HTTP port.$$ *($$Integer$$, default: `$$8080$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/source/http-source/pom.xml b/applications/source/http-source/pom.xml new file mode 100644 index 00000000..b5f0674e --- /dev/null +++ b/applications/source/http-source/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + http-source + 3.0.0.BUILD-SNAPSHOT + http-source + http source apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + http-supplier + ${java-functions.version} + + + org.springframework.boot + spring-boot-starter-webflux + test + + + io.projectreactor + reactor-test + test + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + http + source + ${project.version} + org.springframework.cloud.fn.supplier.http.HttpSupplierConfiguration.class + + + + + org.springframework.cloud.fn + http-supplier + ${java-functions.version} + + + + + spring.main.web-application-type=reactive + spring.cloud.streamapp.security.enabled=false + spring.cloud.streamapp.security.csrf-enabled=false + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/source/http-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/http-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..81cfa8ee --- /dev/null +++ b/applications/source/http-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,3 @@ +configuration-properties.classes=HttpSourceProperties,\ + HttpSourceProperties$Cors +configuration-properties.names=server.port \ No newline at end of file diff --git a/applications/source/http-source/src/test/java/org/springframework/cloud/stream/app/HttpSourceTests.java b/applications/source/http-source/src/test/java/org/springframework/cloud/stream/app/HttpSourceTests.java new file mode 100644 index 00000000..c357a759 --- /dev/null +++ b/applications/source/http-source/src/test/java/org/springframework/cloud/stream/app/HttpSourceTests.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.cloud.fn.supplier.http.HttpSupplierConfiguration; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.config.BindingServiceConfiguration; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = {"spring.cloud.function.definition=httpSupplier","debug=true"}) +public class HttpSourceTests { + + @Autowired + private TestRestTemplate testRestTemplate; + + @Autowired + OutputDestination outputDestination; + + @Test + public void testSourceFromSupplier() { + testRestTemplate.postForObject("/", "test1", Object.class); + Message sourceMessage = outputDestination.receive(10000); + final String actual = new String(sourceMessage.getPayload()); + assertThat(actual).isEqualTo("test1"); + } + + @SpringBootApplication + @Import({HttpSupplierConfiguration.class, TestChannelBinderConfiguration.class, BindingServiceConfiguration.class}) + public static class HttpSourceConfiguration {} +} diff --git a/applications/source/jdbc-source/README.adoc b/applications/source/jdbc-source/README.adoc new file mode 100644 index 00000000..78dd7544 --- /dev/null +++ b/applications/source/jdbc-source/README.adoc @@ -0,0 +1,36 @@ +//tag::ref-doc[] += JDBC Source + +This source polls data from an RDBMS. +This source is fully based on the `DataSourceAutoConfiguration`, so refer to the https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html[Spring Boot JDBC Support] for more information. + +=== Payload + +* `Map` when `jdbc.split == true` (default) and `List>` otherwise + +== Options + +The **$$jdbc$$** $$source$$ has the following options: + +//tag::configuration-properties[] +$$jdbc.supplier.max-rows$$:: $$Max numbers of rows to process for query.$$ *($$Integer$$, default: `$$0$$`)* +$$jdbc.supplier.query$$:: $$The query to use to select data.$$ *($$String$$, default: `$$$$`)* +$$jdbc.supplier.split$$:: $$Whether to split the SQL result as individual messages.$$ *($$Boolean$$, default: `$$true$$`)* +$$jdbc.supplier.update$$:: $$An SQL update statement to execute for marking polled messages as 'seen'.$$ *($$String$$, default: `$$$$`)* +$$spring.cloud.stream.poller.cron$$:: $$Cron expression value for the Cron Trigger.$$ *($$String$$, default: `$$$$`)* +$$spring.cloud.stream.poller.fixed-delay$$:: $$Fixed delay for default poller.$$ *($$Long$$, default: `$$1000$$`)* +$$spring.cloud.stream.poller.initial-delay$$:: $$Initial delay for periodic triggers.$$ *($$Integer$$, default: `$$0$$`)* +$$spring.cloud.stream.poller.max-messages-per-poll$$:: $$Maximum messages per poll for the default poller.$$ *($$Long$$, default: `$$1$$`)* +$$spring.datasource.data$$:: $$Data (DML) script resource references.$$ *($$List$$, default: `$$$$`)* +$$spring.datasource.driver-class-name$$:: $$Fully qualified name of the JDBC driver. Auto-detected based on the URL by default.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.initialization-mode$$:: $$Initialize the datasource with available DDL and DML scripts.$$ *($$DataSourceInitializationMode$$, default: `$$embedded$$`, possible values: `ALWAYS`,`EMBEDDED`,`NEVER`)* +$$spring.datasource.password$$:: $$Login password of the database.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.schema$$:: $$Schema (DDL) script resource references.$$ *($$List$$, default: `$$$$`)* +$$spring.datasource.url$$:: $$JDBC URL of the database.$$ *($$String$$, default: `$$$$`)* +$$spring.datasource.username$$:: $$Login username of the database.$$ *($$String$$, default: `$$$$`)* +//end::configuration-properties[] + +Also see the https://docs.spring.io/spring-boot/docs/current/reference/html/common-application-properties.html[Spring Boot Documentation] +for addition `DataSource` properties and `TriggerProperties` and `MaxMessagesProperties` for polling options. + +//end::ref-doc[] diff --git a/applications/source/jdbc-source/pom.xml b/applications/source/jdbc-source/pom.xml new file mode 100644 index 00000000..90ff70c6 --- /dev/null +++ b/applications/source/jdbc-source/pom.xml @@ -0,0 +1,100 @@ + + + 4.0.0 + jdbc-source + 3.0.0.BUILD-SNAPSHOT + jdbc-source + JDBC source apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.cloud.fn + jdbc-supplier + ${java-functions.version} + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + com.h2database + h2 + test + + + org.springframework.boot + spring-boot-starter-json + test + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + jdbc + source + ${project.version} + org.springframework.cloud.fn.supplier.jdbc.JdbcSupplierConfiguration.class + + + + + org.springframework.cloud.fn + jdbc-supplier + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/source/jdbc-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/jdbc-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..5444c6cd --- /dev/null +++ b/applications/source/jdbc-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,10 @@ +configuration-properties.classes=JdbcSupplierProperties,\ + org.springframework.cloud.stream.config.DefaultPollerProperties +configuration-properties.names=\ +spring.datasource.url,\ +spring.datasource.driver-class-name,\ +spring.datasource.username,\ +spring.datasource.password,\ +spring.datasource.schema,\ +spring.datasource.data,\ +spring.datasource.initialization-mode \ No newline at end of file diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/DefaultBehaviorTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/DefaultBehaviorTests.java new file mode 100644 index 00000000..9324da79 --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/DefaultBehaviorTests.java @@ -0,0 +1,58 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import static org.assertj.core.api.Assertions.assertThat; + +@TestPropertySource(properties = "jdbc.supplier.query=select id, name from test order by id") +class DefaultBehaviorTests extends JdbcSourceIntegrationTests { + + @Test + void testExtraction() throws Exception { + Message received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + Map payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(1, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(2, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(3, payload.get("ID")); + } +} diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/JdbcSourceIntegrationTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/JdbcSourceIntegrationTests.java new file mode 100644 index 00000000..a1871098 --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/JdbcSourceIntegrationTests.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.fn.supplier.jdbc.JdbcSupplierConfiguration; +import org.springframework.cloud.stream.test.binder.MessageCollector; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.messaging.MessageChannel; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest(properties = "spring.cloud.stream.function.definition=jdbcSupplier") +@DirtiesContext +public class JdbcSourceIntegrationTests { + + @Autowired + protected ObjectMapper objectMapper; + + @Qualifier("jdbcSupplier-out-0") + @Autowired + protected MessageChannel output; + + @Autowired + protected JdbcOperations jdbcOperations; + + @Autowired + protected MessageCollector messageCollector; + + @SpringBootApplication + @Import(JdbcSupplierConfiguration.class) + public static class JdbcSourceConfiguration { } + +} diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/Select2PerPollNoSplitWithUpdateTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/Select2PerPollNoSplitWithUpdateTests.java new file mode 100644 index 00000000..2ad4d592 --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/Select2PerPollNoSplitWithUpdateTests.java @@ -0,0 +1,65 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.databind.type.CollectionLikeType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@TestPropertySource(properties = { + "jdbc.supplier.query=select id, name, tag from test where tag is NULL order by id", + "jdbc.supplier.split=false", + "jdbc.supplier.maxRows=2", + "jdbc.supplier.update=update test set tag='1' where id in (:id)" }) +public class Select2PerPollNoSplitWithUpdateTests extends JdbcSourceIntegrationTests { + + @Test + public void testExtraction() throws Exception { + Message received = this.messageCollector.forChannel(this.output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + CollectionLikeType valueType = TypeFactory.defaultInstance() + .constructCollectionLikeType(List.class, Map.class); + + List> payload = this.objectMapper.readValue((String) received.getPayload(), valueType); + + assertEquals(2, payload.size()); + assertEquals(1, payload.get(0).get("ID")); + assertEquals(2, payload.get(1).get("ID")); + received = this.messageCollector.forChannel(this.output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + payload = this.objectMapper.readValue((String) received.getPayload(), valueType); + assertEquals(1, payload.size()); + assertEquals(3, payload.get(0).get("ID")); + } + +} diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllNoSplitTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllNoSplitTests.java new file mode 100644 index 00000000..e04dde44 --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllNoSplitTests.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.databind.type.CollectionLikeType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@TestPropertySource(properties = { + "jdbc.supplier.query=select id, name, tag from test where tag is NULL order by id", + "jdbc.supplier.split=false" +}) +public class SelectAllNoSplitTests extends JdbcSourceIntegrationTests { + + @Test + public void testExtraction() throws Exception { + Message received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + CollectionLikeType valueType = TypeFactory.defaultInstance() + .constructCollectionLikeType(List.class, Map.class); + + List> payload = this.objectMapper.readValue((String) received.getPayload(), valueType); + + assertEquals(3, payload.size()); + assertEquals(1, payload.get(0).get("ID")); + assertEquals("John", payload.get(2).get("NAME")); + } + +} diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithDelayTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithDelayTests.java new file mode 100644 index 00000000..700ae9fc --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithDelayTests.java @@ -0,0 +1,59 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@TestPropertySource(properties = { "jdbc.supplier.query=select id, name from test order by id", "spring.cloud.stream.poller.fixedDelay=60000" }) +public class SelectAllWithDelayTests extends JdbcSourceIntegrationTests { + + @Test + public void testExtraction() throws Exception { + Message received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + Map payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(1, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + assertEquals(2, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + assertEquals(3, payload.get("ID")); + // should not wrap around to the beginning since delay is 60 + received = messageCollector.forChannel(output).poll(1, TimeUnit.SECONDS); + assertNull(received); + } +} diff --git a/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithMinDelayTests.java b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithMinDelayTests.java new file mode 100644 index 00000000..27bbda7a --- /dev/null +++ b/applications/source/jdbc-source/src/test/java/org/springframework/cloud/stream/app/jdbc/source/SelectAllWithMinDelayTests.java @@ -0,0 +1,65 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.jdbc.source; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@TestPropertySource(properties = { "jdbc.supplier.query=select id, name from test order by id", "spring.cloud.stream.poller.fixedDelay=1" }) +public class SelectAllWithMinDelayTests extends JdbcSourceIntegrationTests { + + @Test + public void testExtraction() throws Exception { + Message received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + Map payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(1, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(2, payload.get("ID")); + received = messageCollector.forChannel(output).poll(10, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(3, payload.get("ID")); + // should wrap around to the beginning + received = messageCollector.forChannel(output).poll(2, TimeUnit.SECONDS); + assertNotNull(received); + assertThat(received.getPayload().getClass()).isEqualTo(String.class); + + payload = this.objectMapper.readValue((String) received.getPayload(), Map.class); + + assertEquals(1, payload.get("ID")); + } + +} diff --git a/applications/source/jdbc-source/src/test/resources/schema.sql b/applications/source/jdbc-source/src/test/resources/schema.sql new file mode 100644 index 00000000..bb5835cb --- /dev/null +++ b/applications/source/jdbc-source/src/test/resources/schema.sql @@ -0,0 +1,10 @@ +-- Run by default by Boot infrastructure + +create table test( + id bigint, + name varchar (2000), + tag char(1) +); +insert into test values (1, 'Bob', NULL); +insert into test values (2, 'Jane', NULL); +insert into test values (3, 'John', NULL); \ No newline at end of file diff --git a/applications/source/mongodb-source/README.adoc b/applications/source/mongodb-source/README.adoc new file mode 100644 index 00000000..ea65829c --- /dev/null +++ b/applications/source/mongodb-source/README.adoc @@ -0,0 +1,25 @@ +//tag::ref-doc[] += MongoDB Source + +This source polls data from MongoDB. +This source is fully based on the `MongoDataAutoConfiguration`, so refer to the +https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-nosql.html#boot-features-mongodb[Spring Boot MongoDB Support] +for more information. + + +== Options + +The **$$mongodb$$** $$source$$ has the following options: + + +//tag::configuration-properties[] +$$mongodb.supplier.collection$$:: $$The MongoDB collection to query$$ *($$String$$, default: `$$$$`)* +$$mongodb.supplier.query$$:: $$The MongoDB query$$ *($$String$$, default: `$${ }$$`)* +$$mongodb.supplier.query-expression$$:: $$The SpEL expression in MongoDB query DSL style$$ *($$Expression$$, default: `$$$$`)* +$$mongodb.supplier.split$$:: $$Whether to split the query result as individual messages.$$ *($$Boolean$$, default: `$$true$$`)* +//end::configuration-properties[] + +Also see the https://docs.spring.io/spring-boot/docs/current/reference/html/common-application-properties.html[Spring Boot Documentation] for additional `MongoProperties` properties. +See and `TriggerProperties` for polling options. + +//end::ref-doc[] diff --git a/applications/source/mongodb-source/pom.xml b/applications/source/mongodb-source/pom.xml new file mode 100644 index 00000000..8d6c7e1a --- /dev/null +++ b/applications/source/mongodb-source/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + mongodb-source + 3.0.0.BUILD-SNAPSHOT + mongodb-source + mongodb source apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + mongodb-supplier + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + mongodb + source + ${project.version} + org.springframework.cloud.fn.supplier.mongo.MongodbSupplierConfiguration.class + + + + + org.springframework.cloud.fn + mongodb-supplier + ${java-functions.version} + + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/source/mongodb-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/mongodb-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..db0d96af --- /dev/null +++ b/applications/source/mongodb-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1 @@ +configuration-properties.classes=org.springframework.cloud.fn.supplier.mongo.MongodbSupplierProperties \ No newline at end of file diff --git a/applications/source/pom.xml b/applications/source/pom.xml new file mode 100644 index 00000000..0cb00b77 --- /dev/null +++ b/applications/source/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + org.springframework.cloud.stream.app + source + 3.0.0.BUILD-SNAPSHOT + pom + + + file-source + jdbc-source + http-source + time-source + mongodb-source + + diff --git a/applications/source/time-source/README.adoc b/applications/source/time-source/README.adoc new file mode 100644 index 00000000..9358c5ce --- /dev/null +++ b/applications/source/time-source/README.adoc @@ -0,0 +1,18 @@ +//tag::ref-doc[] += Time Source + +The time source will simply emit a String with the current time every so often. + +== Options + +The **$$time$$** $$source$$ has the following options: + +//tag::configuration-properties[] +$$spring.cloud.stream.poller.cron$$:: $$Cron expression value for the Cron Trigger.$$ *($$String$$, default: `$$$$`)* +$$spring.cloud.stream.poller.fixed-delay$$:: $$Fixed delay for default poller.$$ *($$Long$$, default: `$$1000$$`)* +$$spring.cloud.stream.poller.initial-delay$$:: $$Initial delay for periodic triggers.$$ *($$Integer$$, default: `$$0$$`)* +$$spring.cloud.stream.poller.max-messages-per-poll$$:: $$Maximum messages per poll for the default poller.$$ *($$Long$$, default: `$$1$$`)* +$$time.date-format$$:: $$Format for the date value.$$ *($$String$$, default: `$$MM/dd/yy HH:mm:ss$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/source/time-source/pom.xml b/applications/source/time-source/pom.xml new file mode 100644 index 00000000..1de53685 --- /dev/null +++ b/applications/source/time-source/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + time-source + 3.0.0.BUILD-SNAPSHOT + time-source + time source apps + jar + + + org.springframework.cloud.stream.app + stream-apps-parent + 3.0.0.BUILD-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud.fn + time-supplier + ${java-functions.version} + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + time + source + ${project.version} + org.springframework.cloud.fn.supplier.time.TimeSupplierConfiguration.class + + + + org.springframework.cloud.fn + time-supplier + ${java-functions.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + diff --git a/applications/source/time-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/time-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..a405e9c7 --- /dev/null +++ b/applications/source/time-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,2 @@ +configuration-properties.classes=TimeProperties,\ + org.springframework.cloud.stream.config.DefaultPollerProperties \ No newline at end of file diff --git a/applications/source/time-source/src/test/java/org/springframework/cloud/stream/app/TimeSourceTests.java b/applications/source/time-source/src/test/java/org/springframework/cloud/stream/app/TimeSourceTests.java new file mode 100644 index 00000000..9565462c --- /dev/null +++ b/applications/source/time-source/src/test/java/org/springframework/cloud/stream/app/TimeSourceTests.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.fn.supplier.time.TimeProperties; +import org.springframework.cloud.fn.supplier.time.TimeSupplierConfiguration; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** + * @author Soby Chacko + */ +public class TimeSourceTests { + + @Test + public void testSourceFromSupplier() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration + .getCompleteConfiguration(TimeSourceConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=timeSupplier")) { + + OutputDestination target = context.getBean(OutputDestination.class); + Message sourceMessage = target.receive(10000); + final String actual = new String(sourceMessage.getPayload()); + + TimeProperties timeProperties = context.getBean(TimeProperties.class); + SimpleDateFormat dateFormat = new SimpleDateFormat(timeProperties.getDateFormat()); + assertThatCode(() -> { + Date date = dateFormat.parse(actual); + assertThat(date).isNotNull(); + }).doesNotThrowAnyException(); + } + } + + @EnableAutoConfiguration + @Import(TimeSupplierConfiguration.class) + public static class TimeSourceConfiguration {} +} diff --git a/functions/consumer/cassandra-consumer/.gitignore b/functions/consumer/cassandra-consumer/.gitignore new file mode 100644 index 00000000..3a568a50 --- /dev/null +++ b/functions/consumer/cassandra-consumer/.gitignore @@ -0,0 +1,30 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/functions/consumer/cassandra-consumer/.toDelete b/functions/consumer/cassandra-consumer/.toDelete new file mode 100644 index 00000000..e69de29b diff --git a/functions/consumer/cassandra-consumer/pom.xml b/functions/consumer/cassandra-consumer/pom.xml new file mode 100644 index 00000000..117d5875 --- /dev/null +++ b/functions/consumer/cassandra-consumer/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + cassandra-consumer + 1.0.0.BUILD-SNAPSHOT + cassandra-consumer + Cassandra Consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + 0.8.0.BUILD-SNAPSHOT + 4.3.1.0 + + + + + org.springframework.boot + spring-boot-starter-data-cassandra-reactive + + + org.springframework.boot + spring-boot-starter-json + + + org.springframework.integration + spring-integration-cassandra + ${springIntegrationCassandara.version} + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.cassandraunit + cassandra-unit-spring + ${cassandra-unit-spring.version} + test + + + com.addthis.metrics + reporter-config3 + + + + + io.projectreactor + reactor-test + test + + + org.awaitility + awaitility + test + + + + diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java new file mode 100644 index 00000000..f458f4fb --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java @@ -0,0 +1,209 @@ +/* + * Copyright 2015-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.fn.consumer.cassandra.query.InsertQueryColumnNameExtractor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.cassandra.core.InsertOptions; +import org.springframework.data.cassandra.core.ReactiveCassandraOperations; +import org.springframework.data.cassandra.core.UpdateOptions; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.data.cassandra.core.cql.WriteOptions; +import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.integration.transformer.AbstractPayloadTransformer; +import org.springframework.messaging.MessageHandler; +import org.springframework.util.StringUtils; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.util.StdDateFormat; +import org.springframework.cloud.fn.consumer.cassandra.query.ColumnNameExtractor; +import org.springframework.cloud.fn.consumer.cassandra.query.UpdateQueryColumnNameExtractor; +import reactor.core.publisher.Mono; + +/** + * @author Artem Bilan + * @author Thomas Risberg + * @author Ashu Gairola + * @author Akos Ratku + */ +@Configuration +@EnableConfigurationProperties(CassandraConsumerProperties.class) +public class CassandraConsumerConfiguration { + + @Autowired + private CassandraConsumerProperties cassandraSinkProperties; + + @Bean + public IntegrationFlow cassandraConsumerFlow(MessageHandler cassandraSinkMessageHandler, + ObjectMapper objectMapper) { + IntegrationFlowBuilder integrationFlowBuilder = + IntegrationFlows.from(CassandraConsumerFunction.class); + if (StringUtils.hasText(this.cassandraSinkProperties.getIngestQuery())) { + integrationFlowBuilder.transform( + new PayloadToMatrixTransformer(objectMapper, this.cassandraSinkProperties.getIngestQuery(), + CassandraMessageHandler.Type.UPDATE == this.cassandraSinkProperties.getQueryType() + ? new UpdateQueryColumnNameExtractor() + : new InsertQueryColumnNameExtractor())); + } + return integrationFlowBuilder + .handle(cassandraSinkMessageHandler) + .get(); + } + + @Bean + public MessageHandler cassandraSinkMessageHandler(ReactiveCassandraOperations cassandraOperations) { + CassandraMessageHandler cassandraMessageHandler = + this.cassandraSinkProperties.getQueryType() != null + ? new CassandraMessageHandler(cassandraOperations, this.cassandraSinkProperties.getQueryType()) + : new CassandraMessageHandler(cassandraOperations); + cassandraMessageHandler.setProducesReply(true); + cassandraMessageHandler.setAsync(true); + if (this.cassandraSinkProperties.getConsistencyLevel() != null + || this.cassandraSinkProperties.getTtl() > 0) { + + WriteOptions.WriteOptionsBuilder writeOptionsBuilder = WriteOptions.builder(); + + switch (this.cassandraSinkProperties.getQueryType()) { + + case INSERT: + writeOptionsBuilder = InsertOptions.builder(); + break; + case UPDATE: + writeOptionsBuilder = UpdateOptions.builder(); + break; + } + + if (this.cassandraSinkProperties.getConsistencyLevel() != null) { + writeOptionsBuilder.consistencyLevel(this.cassandraSinkProperties.getConsistencyLevel()); + } + + if (this.cassandraSinkProperties.getTtl() > 0) { + writeOptionsBuilder.ttl(this.cassandraSinkProperties.getTtl()); + } + + cassandraMessageHandler.setWriteOptions(writeOptionsBuilder.build()); + } + if (StringUtils.hasText(this.cassandraSinkProperties.getIngestQuery())) { + cassandraMessageHandler.setIngestQuery(this.cassandraSinkProperties.getIngestQuery()); + } + else if (this.cassandraSinkProperties.getStatementExpression() != null) { + cassandraMessageHandler.setStatementExpression(this.cassandraSinkProperties.getStatementExpression()); + } + return cassandraMessageHandler; + } + + private static boolean isUuid(String uuid) { + if (uuid.length() == 36) { + String[] parts = uuid.split("-"); + if (parts.length == 5) { + return (parts[0].length() == 8) && (parts[1].length() == 4) && + (parts[2].length() == 4) && (parts[3].length() == 4) && + (parts[4].length() == 12); + } + } + return false; + } + + + private static class PayloadToMatrixTransformer extends AbstractPayloadTransformer>> { + + private final Jackson2JsonObjectMapper jsonObjectMapper; + + private final List columns = new LinkedList<>(); + + private final ISO8601StdDateFormat dateFormat = new ISO8601StdDateFormat(); + + PayloadToMatrixTransformer(ObjectMapper objectMapper, String query, ColumnNameExtractor columnNameExtractor) { + this.jsonObjectMapper = new Jackson2JsonObjectMapper(objectMapper); + this.columns.addAll(columnNameExtractor.extract(query)); + this.jsonObjectMapper.getObjectMapper() + .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + } + + @Override + @SuppressWarnings("unchecked") + protected List> transformPayload(Object payload) { + if (payload instanceof List) { + return (List>) payload; + } + else { + try { + List> model = this.jsonObjectMapper.fromJson(payload, List.class); + List> data = new ArrayList<>(model.size()); + for (Map entity : model) { + List row = new ArrayList<>(this.columns.size()); + for (String column : this.columns) { + Object value = entity.get(column); + if (value instanceof String) { + String string = (String) value; + if (this.dateFormat.looksLikeISO8601(string)) { + synchronized (this.dateFormat) { + value = new Date(this.dateFormat.parse(string).getTime()).toLocalDate(); + } + } + if (isUuid(string)) { + value = UUID.fromString(string); + } + } + row.add(value); + } + data.add(row); + } + return data; + } + catch (Exception ex) { + throw new IllegalArgumentException("Cannot parse json into matrix", ex); + } + } + } + + } + + /* + * We need this to provide visibility to the protected method. + */ + @SuppressWarnings("serial") + private static class ISO8601StdDateFormat extends StdDateFormat { + + @Override + protected boolean looksLikeISO8601(String dateStr) { + return super.looksLikeISO8601(dateStr); + } + + } + + interface CassandraConsumerFunction extends Function> { + + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java new file mode 100644 index 00000000..1af4094d --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; + +/** + * @author Artem Bilan + * @author Thomas Risberg + */ +@ConfigurationProperties("cassandra") +public class CassandraConsumerProperties { + + /** + * Time-to-live option of WriteOptions. + */ + private int ttl; + + /** + * QueryType for Cassandra Sink. + */ + private CassandraMessageHandler.Type queryType; + + /** + * Ingest Cassandra query. + */ + private String ingestQuery; + + /** + * Expression in Cassandra query DSL style. + */ + private Expression statementExpression; + + /** + * The consistency level for write operation. + */ + private ConsistencyLevel consistencyLevel; + + public int getTtl() { + return this.ttl; + } + + public void setTtl(int ttl) { + this.ttl = ttl; + } + + public CassandraMessageHandler.Type getQueryType() { + return this.queryType; + } + + public void setQueryType(CassandraMessageHandler.Type queryType) { + this.queryType = queryType; + } + + public String getIngestQuery() { + return this.ingestQuery; + } + + public void setIngestQuery(String ingestQuery) { + this.ingestQuery = ingestQuery; + } + + public Expression getStatementExpression() { + return this.statementExpression; + } + + public void setStatementExpression(Expression statementExpression) { + this.statementExpression = statementExpression; + } + + public ConsistencyLevel getConsistencyLevel() { + return this.consistencyLevel; + } + + public void setConsistencyLevel(ConsistencyLevel consistencyLevel) { + this.consistencyLevel = consistencyLevel; + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java new file mode 100644 index 00000000..06e60ab4 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Scanner; + +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.boot.autoconfigure.cassandra.CassandraProperties; +import org.springframework.boot.autoconfigure.cassandra.CqlSessionBuilderCustomizer; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.domain.EntityScanPackages; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.context.EnvironmentAware; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.data.cassandra.config.CqlSessionFactoryBean; +import org.springframework.data.cassandra.core.ReactiveCassandraTemplate; +import org.springframework.data.cassandra.core.cql.CqlTemplate; +import org.springframework.data.cassandra.core.cql.ReactiveCqlOperations; +import org.springframework.data.cassandra.core.cql.generator.CreateKeyspaceCqlGenerator; +import org.springframework.data.cassandra.core.cql.keyspace.CreateKeyspaceSpecification; +import org.springframework.util.StringUtils; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import reactor.core.publisher.Flux; + +/** + * @author Artem Bilan + * @author Thomas Risberg + * @author Rob Hardt + */ +@Configuration +@EnableConfigurationProperties(CassandraClusterProperties.class) +@Import(CassandraAppClusterConfiguration.CassandraPackageRegistrar.class) +public class CassandraAppClusterConfiguration { + + @Bean + public CqlSessionBuilderCustomizer clusterBuilderCustomizer( + CassandraClusterProperties cassandraClusterProperties) { + + PropertyMapper map = PropertyMapper.get(); + return builder -> + map.from(cassandraClusterProperties::isSkipSslValidation) + .whenTrue() + .toCall(() -> { + try { + builder.withSslContext(TrustAllSSLContextFactory.getSslContext()); + } + catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new BeanInitializationException( + "Unable to configure a Cassandra cluster using SSL.", e); + } + + }); + } + + @Bean + @ConditionalOnProperty("cassandra.cluster.create-keyspace") + public Object keyspaceCreator(CassandraProperties cassandraProperties, CqlSessionBuilder cqlSessionBuilder) { + CreateKeyspaceSpecification createKeyspaceSpecification = + CreateKeyspaceSpecification + .createKeyspace(cassandraProperties.getKeyspaceName()) + .withSimpleReplication() + .ifNotExists(); + + String createKeySpaceQuery = new CreateKeyspaceCqlGenerator(createKeyspaceSpecification).toCql(); + CqlSession systemSession = + cqlSessionBuilder.withKeyspace(CqlSessionFactoryBean.CASSANDRA_SYSTEM_SESSION).build(); + + CqlTemplate template = new CqlTemplate(systemSession); + template.execute(createKeySpaceQuery); + + return null; + } + + @Bean + @Lazy + @DependsOn("keyspaceCreator") + public CqlSession cassandraSession(CqlSessionBuilder cqlSessionBuilder) { + return cqlSessionBuilder.build(); + } + + + @Bean + @ConditionalOnProperty("cassandra.cluster.init-script") + public Object keyspaceInitializer(CassandraClusterProperties cassandraClusterProperties, + ReactiveCassandraTemplate reactiveCassandraTemplate) throws IOException { + + String scripts = + new Scanner(cassandraClusterProperties.getInitScript().getInputStream(), + StandardCharsets.UTF_8.name()) + .useDelimiter("\\A") + .next(); + + ReactiveCqlOperations reactiveCqlOperations = + reactiveCassandraTemplate.getReactiveCqlOperations(); + + Flux.fromArray(StringUtils.delimitedListToStringArray(scripts, ";", "\r\n\f")) + .filter(StringUtils::hasText) // an empty String after the last ';' + .flatMap(script -> reactiveCqlOperations.execute(script + ";")) + .blockLast(); + + return null; + + } + + static class CassandraPackageRegistrar implements ImportBeanDefinitionRegistrar, EnvironmentAware { + + private Environment environment; + + @Override + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + @Override + public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, + BeanDefinitionRegistry registry) { + + Binder.get(this.environment) + .bind("cassandra.cluster.entity-base-packages", String[].class) + .map(Arrays::asList) + .ifBound(packagesToScan -> EntityScanPackages.register(registry, packagesToScan)); + } + + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java new file mode 100644 index 00000000..67e7e549 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java @@ -0,0 +1,85 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.core.io.Resource; + +/** + * Common properties for the cassandra modules. + * + * @author Artem Bilan + * @author Thomas Risberg + * @author Rob Hardt + */ +@ConfigurationProperties("cassandra.cluster") +public class CassandraClusterProperties { + + /** + * Flag to create (or not) keyspace on application startup. + */ + private boolean createKeyspace; + + /** + * Resource with CQL scripts (delimited by ';') to initialize keyspace schema. + */ + private Resource initScript; + + /** + * Flag to validate the Servers' SSL certs + */ + private boolean skipSslValidation; + + /** + * Base packages to scan for entities annotated with Table annotations. + */ + private String[] entityBasePackages = { }; + + + public void setCreateKeyspace(boolean createKeyspace) { + this.createKeyspace = createKeyspace; + } + + public void setInitScript(Resource initScript) { + this.initScript = initScript; + } + + public void setSkipSslValidation(boolean skipSslValidation) { + this.skipSslValidation = skipSslValidation; + } + + public boolean isCreateKeyspace() { + return this.createKeyspace; + } + + public Resource getInitScript() { + return this.initScript; + } + + public boolean isSkipSslValidation() { + return this.skipSslValidation; + } + + public String[] getEntityBasePackages() { + return this.entityBasePackages; + } + + public void setEntityBasePackages(String[] entityBasePackages) { + this.entityBasePackages = entityBasePackages; + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java new file mode 100644 index 00000000..97535139 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.cluster; + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + + +/** + * Helper to provide an SSL Context that does not validate + * certificates presented in the SSL handshake. + * + * The usual caveats apply. + * + * @author Rob Hardt + * @author Artem Bilan + */ +class TrustAllSSLContextFactory { + + static SSLContext getSslContext() throws NoSuchAlgorithmException, KeyManagementException { + + TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager() { + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + + } + }; + + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new SecureRandom()); + return sc; + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java new file mode 100644 index 00000000..19d0d769 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017-2019 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.List; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +@FunctionalInterface +public interface ColumnNameExtractor { + + List extract(String query); + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java new file mode 100644 index 00000000..967fcd0f --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2017 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +public class InsertQueryColumnNameExtractor implements ColumnNameExtractor { + + private static final Pattern PATTERN = Pattern.compile(".+\\((.+)\\).+(?:values\\s*\\((.+)\\))"); + + @Override + public List extract(String query) { + List extractedColumns = new LinkedList<>(); + Matcher matcher = PATTERN.matcher(query); + if (matcher.matches()) { + String[] columns = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); + String[] params = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); + for (int i = 0; i < columns.length; i++) { + String param = params[i]; + if (param.equals("?")) { + extractedColumns.add(columns[i]); + } + else if (param.startsWith(":")) { + extractedColumns.add(param.substring(1)); + } + } + } + else { + throw new IllegalArgumentException("Invalid CQL insert query syntax: " + query); + } + return extractedColumns; + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java new file mode 100644 index 00000000..462478b5 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.query; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * @author Akos Ratku + * @author Artem Bilan + */ +public class UpdateQueryColumnNameExtractor implements ColumnNameExtractor { + + private static final Pattern PATTERN = Pattern.compile("(?i)(?<=set)(.*)(?=where)where(.*)"); + + @Override + public List extract(String query) { + List extractedColumns = new LinkedList<>(); + Matcher matcher = PATTERN.matcher(query); + if (matcher.find()) { + String[] settings = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); + String[] where = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); + readPairs(extractedColumns, settings); + readPairs(extractedColumns, where); + } + else { + throw new IllegalArgumentException("Invalid CQL update query syntax: " + query); + } + return extractedColumns; + } + + protected void readPairs(List extractedColumns, String[] settings) { + for (String setting : settings) { + String[] columnValuePair = StringUtils.delimitedListToStringArray(setting, "=", " "); + if (columnValuePair[1].startsWith(":") || columnValuePair[1].equals("?")) { + extractedColumns.add(columnValuePair[0]); + } + } + } + +} diff --git a/functions/consumer/cassandra-consumer/src/main/resources/application.properties b/functions/consumer/cassandra-consumer/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java new file mode 100644 index 00000000..df0652bd --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.cassandraunit.spring.CassandraUnitDependencyInjectionIntegrationTestExecutionListener; +import org.cassandraunit.spring.EmbeddedCassandra; +import org.cassandraunit.utils.EmbeddedCassandraServerHelper; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import reactor.core.publisher.Mono; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.cassandra.core.CassandraOperations; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestExecutionListeners; + +/** + * @author Artem Bilan + */ +@TestExecutionListeners(mergeMode = TestExecutionListeners.MergeMode.MERGE_WITH_DEFAULTS, + listeners = CassandraUnitDependencyInjectionIntegrationTestExecutionListener.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.data.cassandra.keyspaceName=" + CassandraConsumerApplicationTests.CASSANDRA_KEYSPACE, + "spring.data.cassandra.localDatacenter=datacenter1", + "cassandra.cluster.createKeyspace=true" }) +@EmbeddedCassandra(configuration = EmbeddedCassandraServerHelper.CASSANDRA_RNDPORT_YML_FILE, timeout = 120000) +@DirtiesContext +abstract class CassandraConsumerApplicationTests { + + static final String CASSANDRA_KEYSPACE = "test"; + + @Autowired + protected CassandraOperations cassandraTemplate; + + @Autowired + protected Function> cassandraConsumer; + + @BeforeAll + static void setUp() { + EmbeddedCassandraServerHelper.getSession(); + System.setProperty("spring.data.cassandra.contactPoints", + EmbeddedCassandraServerHelper.getHost() + ':' + EmbeddedCassandraServerHelper.getNativeTransportPort()); + } + + @AfterAll + static void cleanup() { + System.clearProperty("spring.data.cassandra.contactPoints"); + } + + @AfterEach + void tearDown() { + this.cassandraTemplate.truncate(Book.class); + } + + protected static List getBookList(int numBooks) { + + List books = new ArrayList<>(); + + Book b; + for (int i = 0; i < numBooks; i++) { + b = new Book(); + b.setIsbn(UUID.randomUUID()); + b.setTitle("Spring Cloud Data Flow Guide"); + b.setAuthor("SCDF Guru"); + b.setPages(i * 10 + 5); + b.setInStock(true); + b.setSaleDate(LocalDate.now()); + books.add(b); + } + + return books; + } + + @SpringBootApplication + static class TestApplication {} + +} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java new file mode 100644 index 00000000..9ea74e77 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.LocalDate; +import java.util.UUID; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.test.context.TestPropertySource; + +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "spring.data.cassandra.schema-action=RECREATE", + "cassandra.cluster.entity-base-packages=io.pivotal.java.function.cassandra.consumer.domain" }) +class CassandraEntityInsertTests extends CassandraConsumerApplicationTests { + + @Test + @Disabled + void testInsert() { + Book book = new Book(); + book.setIsbn(UUID.randomUUID()); + book.setTitle("Spring Integration Cassandra"); + book.setAuthor("Cassandra Guru"); + book.setPages(521); + book.setSaleDate(LocalDate.now()); + book.setInStock(true); + + Mono result = this.cassandraConsumer.apply(book); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(1)) + .verifyComplete(); + } + +} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java new file mode 100644 index 00000000..dd3cdabd --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "insert into book (isbn, title, author, pages, saleDate, inStock) values (?, ?, ?, ?, ?, ?)" }) +class CassandraIngestInsertTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + Mono result = + this.cassandraConsumer.apply(mapper.toJson(books)); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java new file mode 100644 index 00000000..844a12cc --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.StringUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "insert into book (isbn, title, author, pages, saleDate, inStock) " + + "values (:myIsbn, :myTitle, :myAuthor, ?, ?, ?)" }) +class CassandraIngestNamedParamsTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + String booksJsonWithNamedParams = mapper.toJson(books); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "isbn", "myIsbn"); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "title", "myTitle"); + booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "author", "myAuthor"); + + Mono result = + this.cassandraConsumer.apply(booksJsonWithNamedParams); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java new file mode 100644 index 00000000..2f64956c --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.fn.consumer.cassandra.domain.Book; +import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.test.context.TestPropertySource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +/** + * @author Artem Bilan + */ +@DisabledOnOs(OS.WINDOWS) +@TestPropertySource(properties = { + "cassandra.cluster.init-script=init-db.cql", + "cassandra.ingest-query=" + + "update book set inStock = :inStock, author = :author, pages = :pages, " + + "saleDate = :saleDate, title = :title where isbn = :isbn", + "cassandra.queryType=UPDATE" }) +class CassandraIngestUpdateTests extends CassandraConsumerApplicationTests { + + @Test + void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { + List books = getBookList(5); + + Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); + + Mono result = + this.cassandraConsumer.apply(mapper.toJson(books)); + + StepVerifier.create(result) + .expectNextCount(1) + .then(() -> + assertThat(this.cassandraTemplate.query(Book.class) + .count()) + .isEqualTo(5)) + .verifyComplete(); + } + +} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java new file mode 100644 index 00000000..f1d30388 --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java @@ -0,0 +1,146 @@ +/* + * Copyright 2015-2020 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.cassandra.domain; + +import java.time.LocalDate; +import java.util.UUID; + +import org.springframework.data.cassandra.core.mapping.PrimaryKey; +import org.springframework.data.cassandra.core.mapping.Table; + +/** + * Test POJO + * + * @author David Webb + * @author Artem Bilan + */ +@Table("book") +public class Book { + + @PrimaryKey + private UUID isbn; + + private String title; + + private String author; + + private int pages; + + private LocalDate saleDate; + + private boolean inStock; + + public Book() { + } + + public Book(UUID isbn, String title, String author) { + this.isbn = isbn; + this.title = title; + this.author = author; + } + + /** + * @return Returns the isbn. + */ + public UUID getIsbn() { + return this.isbn; + } + + /** + * @return Returns the saleDate. + */ + public LocalDate getSaleDate() { + return this.saleDate; + } + + /** + * @param saleDate The saleDate to set. + */ + public void setSaleDate(LocalDate saleDate) { + this.saleDate = saleDate; + } + + /** + * @return Returns the inStock. + */ + public boolean isInStock() { + return this.inStock; + } + + /** + * @param inStock The isInStock to set. + */ + public void setInStock(boolean inStock) { + this.inStock = inStock; + } + + /** + * @param isbn The isbn to set. + */ + public void setIsbn(UUID isbn) { + this.isbn = isbn; + } + + /** + * @return Returns the title. + */ + public String getTitle() { + return this.title; + } + + /** + * @param title The title to set. + */ + public void setTitle(String title) { + this.title = title; + } + + /** + * @return Returns the author. + */ + public String getAuthor() { + return this.author; + } + + /** + * @param author The author to set. + */ + public void setAuthor(String author) { + this.author = author; + } + + /** + * @return Returns the pages. + */ + public int getPages() { + return this.pages; + } + + /** + * @param pages The pages to set. + */ + public void setPages(int pages) { + this.pages = pages; + } + + @Override + public String toString() { + return ("isbn -> " + this.isbn) + "\n" + "tile -> " + this.title + "\n" + "author -> " + this.author + + "\n" + "pages -> " + this.pages + "\n"; + } + +} diff --git a/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql b/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql new file mode 100644 index 00000000..6c1397dc --- /dev/null +++ b/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS book; + +CREATE TABLE book ( + isbn uuid PRIMARY KEY, + author text, + instock boolean, + pages int, + saledate date, + title text +); diff --git a/functions/consumer/counter-consumer/.gitignore b/functions/consumer/counter-consumer/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/functions/consumer/counter-consumer/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/functions/consumer/counter-consumer/pom.xml b/functions/consumer/counter-consumer/pom.xml new file mode 100644 index 00000000..1030fddf --- /dev/null +++ b/functions/consumer/counter-consumer/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + counter-consumer + 1.0.0.BUILD-SNAPSHOT + counter-consumer + Spring Native Consumer for computing counters + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.fn + payload-converter-function + ${project.version} + + + org.springframework.boot + spring-boot-starter-integration + + + io.micrometer + micrometer-core + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java new file mode 100644 index 00000000..09bd9f0d --- /dev/null +++ b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerConfiguration.java @@ -0,0 +1,178 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.convert.converter.Converter; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.messaging.Message; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +/** + * + * @author Christian Tzolov + */ +@Configuration +@EnableConfigurationProperties({ CounterConsumerProperties.class }) +public class CounterConsumerConfiguration { + + @Bean + public Function stringToSpelFunction(@Lazy EvaluationContext evaluationContext) { + return new StringToSpelConversionFunction(evaluationContext); + } + + @Bean + @ConfigurationPropertiesBinding + public Converter propertiesSpelConverter(Function stringToSpelFunction) { + return new Converter() { // NOTE Using lambda causes Java Generics issues. + @Override + public Expression convert(String source) { + return stringToSpelFunction.apply(source); + } + }; + } + + @Bean(name = "counterConsumer") + public Consumer> counterConsumer(CounterConsumerProperties properties, MeterRegistry[] meterRegistries, + @Qualifier(IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME) EvaluationContext context) { + + return message -> { + + String counterName = properties.getComputedNameExpression().getValue(context, message, CharSequence.class).toString(); + + // All fixed tags together are passed with every counter increment. + Tags fixedTags = this.toTags(properties.getTag().getFixed()); + + double amount = properties.getComputedAmountExpression().getValue(context, message, double.class); + + Map> allGroupedTags = new HashMap<>(); + // Tag Expressions Counter + if (properties.getTag().getExpression() != null) { + + Map> groupedTags = properties.getTag().getExpression().entrySet().stream() + // maps a pair into [, ... ] Tag array. + .map(namedExpression -> + toList(namedExpression.getValue().getValue(context, message)).stream() + .map(tagValue -> Tag.of(namedExpression.getKey(), tagValue)) + .collect(Collectors.toList())).flatMap(List::stream) + .collect(Collectors.groupingBy(tag -> tag.getKey(), Collectors.toList())); + allGroupedTags.putAll(groupedTags); + } + + this.count(meterRegistries, counterName, fixedTags, allGroupedTags, amount); + }; + } + + /** + * Converts a key/value Map into Tag(key,value) list. Filters out the empty key/value pairs. + * @param keyValueMap key/value map to convert into tags. + * @return Returns Tags list representing every non-empty key/value pair. + */ + protected Tags toTags(Map keyValueMap) { + return CollectionUtils.isEmpty(keyValueMap) ? Tags.empty() : + Tags.of(keyValueMap.entrySet().stream() + .filter(e -> StringUtils.hasText(e.getKey()) && StringUtils.hasText(e.getValue())) + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toList())); + } + + /** + * Converts the input value into an list of values. If the value is not a collection/array type the result + * is a single element list. For collection/array input value the result is the list of stringifie content of + * this collection. + * @param value input value can be array, collection or single value. + * @return Returns value list. + */ + protected List toList(Object value) { + if (value == null) { + return Collections.emptyList(); + } + + if ((value instanceof Collection) || ObjectUtils.isArray(value)) { + Collection valueCollection = (value instanceof Collection) ? (Collection) value + : Arrays.asList(ObjectUtils.toObjectArray(value)); + + return valueCollection.stream() + .filter(v -> v != null) + .map(Object::toString) + .filter(StringUtils::hasText) + .collect(Collectors.toList()); + } + else { + return Arrays.asList(value.toString()); + } + } + + private void count(MeterRegistry[] meterRegistries, String counterName, Tags fixedTags, Map> groupedTags, double amount) { + if (!CollectionUtils.isEmpty(groupedTags)) { + groupedTags.values().stream().map(List::size).max(Integer::compareTo).ifPresent( + max -> { + for (int i = 0; i < max; i++) { + Tags currentTags = Tags.of(fixedTags); + for (Map.Entry> e : groupedTags.entrySet()) { + currentTags = (e.getValue().size() > i) ? + currentTags.and(e.getValue().get(i)) : + currentTags.and(Tags.of(e.getKey(), "")); + } + + // Increment the counterName increment for every configured MaterRegistry. + for (MeterRegistry meterRegistry : meterRegistries) { + meterRegistry.counter(counterName, currentTags).increment(amount); + } + } + } + ); + } + else { + // Increment the counterName increment for every configured MaterRegistry. + for (MeterRegistry meterRegistry : meterRegistries) { + meterRegistry.counter(counterName, fixedTags).increment(amount); + } + } + } + + @Bean + @ConditionalOnMissingBean + public SimpleMeterRegistry simpleMeterRegistry() { + return new SimpleMeterRegistry(); + } +} diff --git a/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java new file mode 100644 index 00000000..a3442fd8 --- /dev/null +++ b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerProperties.java @@ -0,0 +1,172 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Map; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.validation.annotation.Validated; + +/** + * @author Christian Tzolov + */ +@ConfigurationProperties("counter") +@Validated +public class CounterConsumerProperties { + + /** + * The default name of the increment + */ + @Value("${spring.application.name:counts}") + private String defaultName; + + /** + * The name of the counter to increment. The 'name' and 'nameExpression' are mutually exclusive. + * Only one can be set. + */ + private String name; + + /** + * A SpEL expression (against the incoming Message) to derive the name of the counter to increment. + * The 'name' and 'nameExpression' are mutually exclusive. Only one can be set. + */ + private Expression nameExpression; + + /** + * A SpEL expression (against the incoming Message) to derive the amount to add to the counter. + * If not set the counter is incremented by 1.0 + */ + private Expression amountExpression; + + /** + * Enables counting the number of messages processed. Uses the 'message.' counter name prefix to distinct it + * form the expression based counter. The message counter includes the fixed tags when provided. + */ + private boolean messageCounterEnabled = true; + + /** + * Fixed and computed tags to be assignee with the counter increment measurement. + */ + private MetricsTag tag = new MetricsTag(); + + public static class MetricsTag { + + /** + * Custom tags assigned to every counter increment measurements. + * This is a map so the property convention fixed tags is: counter.tag.fixed.[tag-name]=[tag-value] + */ + private Map fixed; + + /** + * Computes tags from SpEL expression. + * Single SpEL expression can produce an array of values, which in turn means distinct name/value tags. + * Every name/value tag will produce a separate counter increment. + * Tag expression format is: counter.tag.expression.[tag-name]=[SpEL expression] + */ + private Map expression; + + public Map getFixed() { + return fixed; + } + + public void setFixed(Map fixed) { + this.fixed = fixed; + } + + public Map getExpression() { + return expression; + } + + public void setExpression(Map expression) { + this.expression = expression; + } + + @Override + public String toString() { + return "MetricsTag{" + + "fixed=" + fixed + + ", expression=" + expression + + '}'; + } + } + + public MetricsTag getTag() { + return tag; + } + + public String getName() { + if (name == null && nameExpression == null) { + return defaultName; + } + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Expression getNameExpression() { + return nameExpression; + } + + public void setNameExpression(Expression nameExpression) { + this.nameExpression = nameExpression; + } + + public Expression getAmountExpression() { + return amountExpression; + } + + public void setAmountExpression(Expression amountExpression) { + this.amountExpression = amountExpression; + } + + public Expression getComputedAmountExpression() { + return (amountExpression != null ? amountExpression : new LiteralExpression("1.0")); + } + + public Expression getComputedNameExpression() { + return (nameExpression != null ? nameExpression : new LiteralExpression(getName())); + } + + public boolean isMessageCounterEnabled() { + return messageCounterEnabled; + } + + public void setMessageCounterEnabled(boolean messageCounterEnabled) { + this.messageCounterEnabled = messageCounterEnabled; + } + + @AssertTrue(message = "exactly one of 'name' and 'nameExpression' must be set") + public boolean isExclusiveOptions() { + return getName() != null ^ getNameExpression() != null; + } + + @Override + public String toString() { + return "CounterFunctionProperties{" + + "defaultName='" + defaultName + '\'' + + ", name=" + name + + ", tag=" + tag + + '}'; + } +} diff --git a/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java new file mode 100644 index 00000000..a8a5e99f --- /dev/null +++ b/functions/consumer/counter-consumer/src/main/java/org/springframework/cloud/fn/consumer/counter/StringToSpelConversionFunction.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Function; + +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.ParseException; +import org.springframework.expression.spel.standard.SpelExpression; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +/** + * Converter from String to Spring Expression. + * + * TODO: This could be a top level project. + */ +public class StringToSpelConversionFunction implements Function { + + private final SpelExpressionParser parser; + + private final EvaluationContext evaluationContext; + + public StringToSpelConversionFunction(EvaluationContext evaluationContext) { + this(new SpelExpressionParser(), evaluationContext); + } + + public StringToSpelConversionFunction(SpelExpressionParser parser, EvaluationContext evaluationContext) { + this.evaluationContext = evaluationContext; + this.parser = parser; + } + + @Override + public Expression apply(String source) { + try { + Expression expression = parser.parseExpression(source); + if (expression instanceof SpelExpression) { + ((SpelExpression) expression).setEvaluationContext(evaluationContext); + } + return expression; + } + catch (ParseException e) { + throw new IllegalArgumentException(String.format( + "Could not convert '%s' into a SpEL expression", source), e); + } + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java new file mode 100644 index 00000000..14f21775 --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ConverterFunctionAdapter.java @@ -0,0 +1,22 @@ +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Function; + +import org.springframework.core.convert.converter.Converter; + +/** + * @author Christian Tzolov + */ +public class ConverterFunctionAdapter implements Converter { + + private Function function; + + public ConverterFunctionAdapter(Function function) { + this.function = function; + } + + @Override + public T convert(S s) { + return this.function.apply(s); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java new file mode 100644 index 00000000..92ef115c --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CountWithAmountTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.expression.foo='bar'", + "counter.amount-expression=payload.length()" +}) +class CountWithAmountTest extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + String message = "hello world message"; + double messageSize = Long.valueOf(message.length()).doubleValue(); + counterConsumer.accept(new GenericMessage(message)); + assertThat(meterRegistry.find("counter666").counter().count()).isEqualTo(messageSize); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java new file mode 100644 index 00000000..78f4790f --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/CounterConsumerParentTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.function.Consumer; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE) +@DirtiesContext +public class CounterConsumerParentTest { + + @Autowired + protected SimpleMeterRegistry meterRegistry; + + @Autowired + protected Consumer> counterConsumer; + + protected Message message(String payload) { + return MessageBuilder.withPayload(payload.getBytes()).build(); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java new file mode 100644 index 00000000..898e67ac --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/EmptyTagsTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Collection; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=", + "counter.tag.expression.tag666=#jsonPath(payload,'$..noField')", + "counter.tag.expression.test=#jsonPath(payload,'$..test')", +}) +class EmptyTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + counterConsumer.accept(message("{\"test\": \"Bar\"}")); + + Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); + assertThat(fixedTagsCounters.size()).isEqualTo(0); + + Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); + assertThat(expressionTagsCounters.size()).isEqualTo(0); + + Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); + assertThat(testExpTagsCounters.size()).isEqualTo(1); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java new file mode 100644 index 00000000..99d5c881 --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/ExpressionCounterNameTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name-expression=payload" +}) +public class ExpressionCounterNameTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + assertThat(meterRegistry.find("hello").counter().count()).isEqualTo(13.0); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java new file mode 100644 index 00000000..a2debb41 --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/FixedTagsTests.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; +import java.util.stream.StreamSupport; + +import io.micrometer.core.instrument.Meter; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=bar", + "counter.tag.fixed.gork=bork" +}) +public class FixedTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + Meter counterMeter = meterRegistry.find("counter666").meter(); + assertThat(StreamSupport.stream(counterMeter.measure().spliterator(), false) + .mapToDouble(m -> m.getValue()).sum()).isEqualTo(13.0); + + assertThat(counterMeter.getId().getTags().size()).isEqualTo(2); + assertThat(counterMeter.getId().getTag("foo")).isEqualTo("bar"); + assertThat(counterMeter.getId().getTag("gork")).isEqualTo("bork"); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java new file mode 100644 index 00000000..63a783e8 --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/LiteralTagExpressionsTests.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.stream.IntStream; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.expression.foo='bar'", + "counter.tag.expression.gork='bork'" +}) +public class LiteralTagExpressionsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + IntStream.range(0, 13).forEach(i -> counterConsumer.accept(new GenericMessage("hello"))); + + Counter fooCounter = meterRegistry.find("counter666").tag("foo", "bar").counter(); + assertThat(fooCounter.count()).isEqualTo(13.0); + + Counter gorkCounter = meterRegistry.find("counter666").tag("gork", "bork").counter(); + assertThat(gorkCounter.count()).isEqualTo(13.0); + + assertThat(fooCounter.getId()).isEqualTo(gorkCounter.getId()); + } +} diff --git a/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java new file mode 100644 index 00000000..79c75708 --- /dev/null +++ b/functions/consumer/counter-consumer/src/test/java/org/springframework/cloud/fn/consumer/counter/NullTagsTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.counter; + +import java.util.Collection; + +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.Test; + +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +@TestPropertySource(properties = { + "counter.name=counter666", + "counter.tag.fixed.foo=", + "counter.tag.expression.tag666=#jsonPath(payload,'$..noField')", + "counter.tag.expression.test=#jsonPath(payload,'$..test')", +}) +public class NullTagsTests extends CounterConsumerParentTest { + + @Test + void testCounterSink() { + + counterConsumer.accept(message("{\"test\": null}")); + + Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); + assertThat(fixedTagsCounters.size()).isEqualTo(0); + + Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); + assertThat(expressionTagsCounters.size()).isEqualTo(0); + + Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); + assertThat(testExpTagsCounters.size()).isEqualTo(0); + } +} diff --git a/functions/consumer/file-consumer/pom.xml b/functions/consumer/file-consumer/pom.xml new file mode 100644 index 00000000..9ab3918d --- /dev/null +++ b/functions/consumer/file-consumer/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + file-consumer + 1.0.0.BUILD-SNAPSHOT + file-consumer + file consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-file + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + + diff --git a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java new file mode 100644 index 00000000..d00f57f5 --- /dev/null +++ b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java @@ -0,0 +1,73 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.util.function.Consumer; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.file.DefaultFileNameGenerator; +import org.springframework.integration.file.FileNameGenerator; +import org.springframework.integration.file.FileWritingMessageHandler; +import org.springframework.messaging.Message; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(FileConsumerProperties.class) +public class FileConsumerConfiguration { + + private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); + + private final FileConsumerProperties properties; + + public FileConsumerConfiguration(FileConsumerProperties properties) { + this.properties = properties; + } + + @Bean + public Consumer> fileConsumer() { + return fileWritingMessageHandler()::handleMessage; + } + + @Bean + public FileWritingMessageHandler fileWritingMessageHandler() { + FileWritingMessageHandler handler = (properties.getDirectoryExpression() != null) + ? new FileWritingMessageHandler(EXPRESSION_PARSER.parseExpression(properties.getDirectoryExpression())) + : new FileWritingMessageHandler(properties.getDirectory()); + handler.setAutoCreateDirectory(true); + handler.setAppendNewLine(!properties.isBinary()); + handler.setCharset(properties.getCharset()); + handler.setExpectReply(false); + handler.setFileExistsMode(properties.getMode()); + handler.setFileNameGenerator(fileNameGenerator()); + return handler; + } + + @Bean + public FileNameGenerator fileNameGenerator() { + DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); + fileNameGenerator.setExpression(properties.getNameExpression()); + return fileNameGenerator; + } +} diff --git a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java new file mode 100644 index 00000000..eb861a61 --- /dev/null +++ b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java @@ -0,0 +1,162 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.file.support.FileExistsMode; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; + +/** + * Properties for the file sink. + * + * @author Mark Fisher + * @author Gary Russell + */ +@ConfigurationProperties("file.consumer") +@Validated +public class FileConsumerProperties { + + static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + "file-consumer"; + + private static final String DEFAULT_NAME = "file-consumer"; + + /** + * A flag to indicate whether adding a newline after the write should be suppressed. + */ + private boolean binary = false; + + /** + * The charset to use when writing text content. + */ + private String charset = "UTF-8"; + + /** + * The parent directory of the target file. + */ + private File directory = new File(DEFAULT_DIR); + + /** + * The expression to evaluate for the parent directory of the target file. + */ + private String directoryExpression; + + /** + * The FileExistsMode to use if the target file already exists. + */ + private FileExistsMode mode = FileExistsMode.APPEND; + + /** + * The name of the target file. + */ + private String name = DEFAULT_NAME; + + /** + * The expression to evaluate for the name of the target file. + */ + private String nameExpression; + + /** + * The suffix to append to file name. + */ + private String suffix = ""; + + public boolean isBinary() { + return binary; + } + + public void setBinary(boolean binary) { + this.binary = binary; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public File getDirectory() { + return directory; + } + + public void setDirectory(File directory) { + this.directory = directory; + } + + public String getDirectoryExpression() { + return directoryExpression; + } + + public void setDirectoryExpression(String directoryExpression) { + this.directoryExpression = directoryExpression; + } + + public FileExistsMode getMode() { + return mode; + } + + public void setMode(FileExistsMode mode) { + this.mode = mode; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getNameExpression() { + return (nameExpression != null) + ? nameExpression + " + '" + getSuffix() + "'" + : "'" + name + getSuffix() + "'"; + } + + public void setNameExpression(String nameExpression) { + this.nameExpression = nameExpression; + } + + public String getSuffix() { + String suffixWithDotIfNecessary = ""; + if (StringUtils.hasText(suffix)) { + suffixWithDotIfNecessary = suffix.startsWith(".") ? suffix : "." + suffix; + } + return suffixWithDotIfNecessary; + } + + public void setSuffix(String suffix) { + this.suffix = suffix; + } + + @AssertTrue(message = "Exactly one of 'name' or 'nameExpression' must be set") + public boolean isMutuallyExclusiveNameAndNameExpression() { + return DEFAULT_NAME.equals(name) || nameExpression == null; + } + + @AssertTrue(message = "Exactly one of 'directory' or 'directoryExpression' must be set") + public boolean isMutuallyExclusiveDirectoryAndDirectoryExpression() { + return new File(DEFAULT_DIR).equals(directory) || directoryExpression == null; + } + +} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java new file mode 100644 index 00000000..e886e5ec --- /dev/null +++ b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.nio.file.Path; +import java.util.function.Consumer; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class AbstractFileConsumerTests { + + @TempDir + static Path tempDir; + + @Autowired + Consumer> fileConsumer; + + @BeforeAll + public static void beforeAll() { + System.setProperty("file.consumer.directory", tempDir.toAbsolutePath().toString()); + } + + @AfterAll + public static void afterAll() { + System.clearProperty("file.consumer.directory"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java new file mode 100644 index 00000000..15b4ac02 --- /dev/null +++ b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.binary = true") +public class BinaryFileTests extends AbstractFileConsumerTests { + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer".getBytes()).build()); + File file = new File(tempDir.toFile(), "file-consumer"); + assertThat(file.exists()).isTrue(); + byte[] results = FileCopyUtils.copyToByteArray(file); + assertThat("hello file-consumer".getBytes()).isEqualTo(results); + } + +} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java new file mode 100644 index 00000000..5eeb4cfd --- /dev/null +++ b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; +import java.io.FileReader; +import java.nio.file.Path; +import java.util.function.Consumer; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + * + * We don't need a separate SpringBootApplication for this test as there is already one available in this package. + * {@link AbstractFileConsumerTests}. + */ +@SpringBootTest(properties = {"file.consumer.nameExpression = payload.substring(0, 4)", + "file.consumer.directoryExpression = '${java.io.tmpdir}'+'/'+headers.dir", + "file.consumer.suffix=out"}) +@DirtiesContext +public class ExpressionTests { + + @TempDir + static Path tempDir; + + @Autowired + Consumer> fileConsumer; + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("this is something").setHeader("dir", "expression").build()); + File file = new File(System.getProperty("java.io.tmpdir") + File.separator + "expression", "this.out"); + file.deleteOnExit(); + assertThat(file.exists()).isTrue(); + assertThat("this is something" + System.lineSeparator()) + .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); + } +} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java new file mode 100644 index 00000000..38aa77a0 --- /dev/null +++ b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.file; + +import java.io.File; +import java.io.FileReader; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.FileCopyUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Mark Fisher + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.name = test", "file.consumer.suffix=txt"}) +public class TextFileTests extends AbstractFileConsumerTests { + + @Test + public void test() throws Exception { + fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer").build()); + File file = new File(tempDir.toFile(), "test.txt"); + assertThat(file.exists()).isTrue(); + assertThat("hello file-consumer" + System.lineSeparator()) + .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); + } + +} diff --git a/functions/consumer/jdbc-consumer/pom.xml b/functions/consumer/jdbc-consumer/pom.xml new file mode 100644 index 00000000..fbd5fd1c --- /dev/null +++ b/functions/consumer/jdbc-consumer/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + jdbc-consumer + 1.0.0.BUILD-SNAPSHOT + jdbc-consumer + jdbc consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-jdbc + + + org.springframework.boot + spring-boot-starter-json + + + org.springframework.boot + spring-boot-starter-jdbc + + + com.h2database + h2 + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.awaitility + awaitility + test + + + junit + junit + + + + + + diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java new file mode 100644 index 00000000..90d671c4 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.core.io.ByteArrayResource; + +/** + * An in-memory script crafted for dropping-creating the table we're working with. + * All columns are created as VARCHAR(2000). + * + * @author Eric Bottard + * @author Thomas Risberg + */ +public class DefaultInitializationScriptResource extends ByteArrayResource { + + private static final Log logger = LogFactory.getLog(DefaultInitializationScriptResource.class); + + public DefaultInitializationScriptResource(String tableName, Collection columns) { + super(scriptFor(tableName, columns).getBytes(StandardCharsets.UTF_8)); + } + + private static String scriptFor(String tableName, Collection columns) { + StringBuilder result = new StringBuilder("DROP TABLE "); + result.append(tableName).append(";\n\n"); + + result.append("CREATE TABLE ").append(tableName).append('('); + int i = 0; + for (String column : columns) { + if (i++ > 0) { + result.append(", "); + } + result.append(column).append(" VARCHAR(2000)"); + } + result.append(");\n"); + logger.debug(String.format("Generated the following initializing script for table %s:\n%s", tableName, + result.toString())); + return result.toString(); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java new file mode 100644 index 00000000..ebe29278 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java @@ -0,0 +1,314 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import javax.sql.DataSource; + +import com.fasterxml.jackson.databind.JsonNode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ResourceLoader; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.EvaluationException; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.SpelParseException; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.integration.aggregator.DefaultAggregatingMessageGroupProcessor; +import org.springframework.integration.aggregator.MessageCountReleaseStrategy; +import org.springframework.integration.config.AggregatorFactoryBean; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.integration.expression.ValueExpression; +import org.springframework.integration.jdbc.JdbcMessageHandler; +import org.springframework.integration.jdbc.SqlParameterSourceFactory; +import org.springframework.integration.json.JsonPropertyAccessor; +import org.springframework.integration.store.MessageGroupStore; +import org.springframework.integration.store.SimpleMessageStore; +import org.springframework.integration.support.MutableMessage; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; +import org.springframework.jdbc.datasource.init.DataSourceInitializer; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MimeTypeUtils; +import org.springframework.util.MultiValueMap; + +/** + * + * @author Eric Bottard + * @author Thomas Risberg + * @author Robert St. John + * @author Oliver Flasch + * @author Artem Bilan + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@Configuration +@EnableConfigurationProperties(JdbcConsumerProperties.class) +public class JdbcConsumerConfiguration { + + private static final Log logger = LogFactory.getLog(JdbcConsumerConfiguration.class); + + private static final Object NOT_SET = new Object(); + + private final JdbcConsumerProperties properties; + + private SpelExpressionParser spelExpressionParser = new SpelExpressionParser(); + + private EvaluationContext evaluationContext; + + public JdbcConsumerConfiguration(JdbcConsumerProperties properties, BeanFactory beanFactory) { + this.properties = properties; + this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(beanFactory); + StandardEvaluationContext standardEvaluationContext = (StandardEvaluationContext) this.evaluationContext; + standardEvaluationContext.addPropertyAccessor(new JsonPropertyAccessor()); + } + + @Bean + IntegrationFlow jdbcConsumerFlow(@Qualifier("aggregator") MessageHandler aggregator, + JdbcMessageHandler jdbcMessageHandler) { + + final IntegrationFlowBuilder builder = + IntegrationFlows.from(Consumer.class, gateway -> gateway.beanName("jdbcConsumer")); + if (properties.getBatchSize() > 1 || properties.getIdleTimeout() > 0) { + builder.handle(aggregator); + } + return builder.handle(jdbcMessageHandler).get(); + } + + @Bean + FactoryBean aggregator(MessageGroupStore messageGroupStore) { + AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean(); + aggregatorFactoryBean.setCorrelationStrategy(message -> message.getPayload().getClass().getName()); + aggregatorFactoryBean.setReleaseStrategy(new MessageCountReleaseStrategy(this.properties.getBatchSize())); + if (this.properties.getIdleTimeout() >= 0) { + aggregatorFactoryBean.setGroupTimeoutExpression(new ValueExpression<>(this.properties.getIdleTimeout())); + } + aggregatorFactoryBean.setMessageStore(messageGroupStore); + aggregatorFactoryBean.setProcessorBean(new DefaultAggregatingMessageGroupProcessor()); + aggregatorFactoryBean.setExpireGroupsUponCompletion(true); + aggregatorFactoryBean.setSendPartialResultOnExpiry(true); + return aggregatorFactoryBean; + } + + @Bean + MessageGroupStore messageGroupStore() { + SimpleMessageStore messageGroupStore = new SimpleMessageStore(); + messageGroupStore.setTimeoutOnIdle(true); + messageGroupStore.setCopyOnGet(false); + return messageGroupStore; + } + + @Bean + public JdbcMessageHandler jdbcMessageHandler(DataSource dataSource) { + final MultiValueMap columnExpressionVariations = new LinkedMultiValueMap<>(); + for (Map.Entry entry : this.properties.getColumnsMap().entrySet()) { + String value = entry.getValue(); + columnExpressionVariations.add(entry.getKey(), this.spelExpressionParser.parseExpression(value)); + if (!value.startsWith("payload")) { + String qualified = "payload." + value; + try { + columnExpressionVariations.add(entry.getKey(), + this.spelExpressionParser.parseExpression(qualified)); + } + catch (SpelParseException e) { + logger.info("failed to parse qualified fallback expression " + qualified + + "; be sure your expression uses the 'payload.' prefix where necessary"); + } + } + } + JdbcMessageHandler jdbcMessageHandler = new JdbcMessageHandler(dataSource, + generateSql(this.properties.getTableName(), columnExpressionVariations.keySet())) { + + @Override + protected void handleMessageInternal(final Message message) { + Message convertedMessage = message; + if (message.getPayload() instanceof byte[] || message.getPayload() instanceof Iterable) { + + final String contentType = message.getHeaders().containsKey(MessageHeaders.CONTENT_TYPE) + ? message.getHeaders().get(MessageHeaders.CONTENT_TYPE).toString() + : MimeTypeUtils.APPLICATION_JSON_VALUE; + if (message.getPayload() instanceof Iterable) { + Stream messageStream = + StreamSupport.stream(((Iterable) message.getPayload()).spliterator(), false) + .map(payload -> { + if (payload instanceof byte[]) { + return convertibleContentType(contentType) ? + new String(((byte[]) payload)) : payload; + } + else { + return payload; + } + }); + convertedMessage = new MutableMessage<>(messageStream.collect(Collectors.toList()), + message.getHeaders()); + } + else { + if (convertibleContentType(contentType)) { + convertedMessage = new MutableMessage<>(new String(((byte[]) message.getPayload())), + message.getHeaders()); + } + } + } + super.handleMessageInternal(convertedMessage); + } + }; + SqlParameterSourceFactory parameterSourceFactory = + new ParameterFactory(columnExpressionVariations, this.evaluationContext); + jdbcMessageHandler.setSqlParameterSourceFactory(parameterSourceFactory); + return jdbcMessageHandler; + } + + @ConditionalOnProperty("jdbc.consumer.initialize") + @Bean + public DataSourceInitializer nonBootDataSourceInitializer(DataSource dataSource, ResourceLoader resourceLoader) { + DataSourceInitializer dataSourceInitializer = new DataSourceInitializer(); + dataSourceInitializer.setDataSource(dataSource); + ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); + databasePopulator.setIgnoreFailedDrops(true); + dataSourceInitializer.setDatabasePopulator(databasePopulator); + if ("true".equals(properties.getInitialize())) { + databasePopulator.addScript( + new DefaultInitializationScriptResource(this.properties.getTableName(), + this.properties.getColumnsMap().keySet())); + } + else { + databasePopulator.addScript(resourceLoader.getResource(this.properties.getInitialize())); + } + return dataSourceInitializer; + } + + @Bean + public static ShorthandMapConverter shorthandMapConverter() { + return new ShorthandMapConverter(); + } + + private static boolean convertibleContentType(String contentType) { + return contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple"); + } + + private static String generateSql(String tableName, Set columns) { + StringBuilder builder = new StringBuilder("INSERT INTO "); + StringBuilder questionMarks = new StringBuilder(") VALUES ("); + builder.append(tableName).append("("); + int i = 0; + + for (String column : columns) { + if (i++ > 0) { + builder.append(", "); + questionMarks.append(", "); + } + builder.append(column); + questionMarks.append(':').append(column); + } + builder.append(questionMarks).append(")"); + return builder.toString(); + } + + private static final class ParameterFactory implements SqlParameterSourceFactory { + + private final MultiValueMap columnExpressions; + + private final EvaluationContext context; + + ParameterFactory(MultiValueMap columnExpressions, EvaluationContext context) { + this.columnExpressions = columnExpressions; + this.context = context; + } + + @Override + public SqlParameterSource createParameterSource(Object o) { + if (!(o instanceof Message)) { + throw new IllegalArgumentException("Unable to handle type " + o.getClass().getName()); + } + Message message = (Message) o; + MapSqlParameterSource parameterSource = new MapSqlParameterSource(); + for (Map.Entry> entry : this.columnExpressions.entrySet()) { + String key = entry.getKey(); + List spels = entry.getValue(); + Object value = NOT_SET; + EvaluationException lastException = null; + for (Expression spel : spels) { + try { + value = spel.getValue(context, message); + break; + } + catch (EvaluationException e) { + lastException = e; + } + } + if (value == NOT_SET) { + if (lastException != null) { + logger.info("Could not find value for column '" + key + "': " + lastException.getMessage()); + } + parameterSource.addValue(key, null); + } + else { + if (value instanceof JsonPropertyAccessor.ToStringFriendlyJsonNode) { + // Need to do some reflection until we have a getter for the Node + DirectFieldAccessor dfa = new DirectFieldAccessor(value); + JsonNode node = (JsonNode) dfa.getPropertyValue("node"); + Object valueToUse; + if (node == null || node.isNull()) { + valueToUse = null; + } + else if (node.isNumber()) { + valueToUse = node.numberValue(); + } + else if (node.isBoolean()) { + valueToUse = node.booleanValue(); + } + else { + valueToUse = node.textValue(); + } + parameterSource.addValue(key, valueToUse); + } + else { + parameterSource.addValue(key, value); + } + } + } + return parameterSource; + } + + } + +} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java new file mode 100644 index 00000000..3afae4a1 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java @@ -0,0 +1,109 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Eric Bottard + * @author Artem Bilan + * @author Oliver Flasch + */ +@ConfigurationProperties("jdbc.consumer") +public class JdbcConsumerProperties { + + @Autowired + private ShorthandMapConverter shorthandMapConverter; + + /** + * The name of the table to write into. + */ + private String tableName = "messages"; + + /** + * The comma separated colon-based pairs of column names and SpEL expressions for values to insert/update. + * Names are used at initialization time to issue the DDL. + */ + private String columns = "payload:payload.toString()"; + + /** + * 'true', 'false' or the location of a custom initialization script for the table. + */ + private String initialize = "false"; + + /** + * Threshold in number of messages when data will be flushed to database table. + */ + private int batchSize = 1; + + /** + * Idle timeout in milliseconds when data is automatically flushed to database table. + */ + private long idleTimeout = -1L; + + private Map columnsMap; + + public String getTableName() { + return this.tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getColumns() { + return this.columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + + public String getInitialize() { + return this.initialize; + } + + public void setInitialize(String initialize) { + this.initialize = initialize; + } + + public int getBatchSize() { + return this.batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public long getIdleTimeout() { + return this.idleTimeout; + } + + public void setIdleTimeout(long idleTimeout) { + this.idleTimeout = idleTimeout; + } + + Map getColumnsMap() { + if (this.columnsMap == null) { + this.columnsMap = this.shorthandMapConverter.convert(this.columns); + } + return this.columnsMap; + } +} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java new file mode 100644 index 00000000..ec925b85 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.util.Assert; + +/** + * A Converter from String to Map that accepts csv {@literal key:value} pairs + * (similar to what comes out of the box in Spring Core) but also simple + * {@literal key} items, in which case the value is assumed to be equal to the key. + *

+ *

Additionally, commas and colons can be escaped by using a backslash, which is + * useful if said mappings are to be used for SpEL for example.

+ * + * @author Eric Bottard + * @author Artem Bilan + */ +public class ShorthandMapConverter implements Converter> { + + @Override + public Map convert(String source) { + Map result = new LinkedHashMap<>(); + + // Split on comma if not preceded by backslash + String[] mappings = source.split("(? message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + } + Awaitility.await().until(() -> jdbcOperations + .queryForObject("select count(*) from messages", Integer.class), value -> value == numberOfInserts); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java new file mode 100644 index 00000000..f7857f93 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a") +public class DataReceivedAsByteArrayTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertionWhenDataReceivedAsByteArray() { + String hello = "{\"a\": \"hello\"}"; + final Message message = MessageBuilder.withPayload(hello.getBytes()).build(); + jdbcConsumer.accept(message); + final Integer count = + jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class, "hello"); + assertThat(count).isEqualTo(1); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java new file mode 100644 index 00000000..375db87a --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = { "jdbc.consumer.tableName=foobar", + "jdbc.consumer.initialize=classpath:explicit-script.sql", + "jdbc.consumer.columns=a,b" }) +public class ExplicitTableCreationTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = + jdbcOperations.query("select a, b from foobar", new BeanPropertyRowMapper<>(Payload.class)) + .get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java new file mode 100644 index 00000000..c5cc13f8 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a: headers[foo]") +public class HeaderInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testHeaderInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent) + .setHeader("foo", "bar").build(); + jdbcConsumer.accept(message); + assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ?", + Integer.class, "bar")).isEqualTo(1); + } +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java new file mode 100644 index 00000000..3915f29f --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = { + "jdbc.consumer.tableName=no_script", + "jdbc.consumer.initialize=true", + "jdbc.consumer.columns=a,b" }) +public class ImplicitTableCreationTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = jdbcOperations + .query("select a, b from no_script", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java new file mode 100644 index 00000000..e4bff00a --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import java.util.function.Consumer; + +import org.junit.jupiter.api.AfterEach; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +/** + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class JdbcConsumerApplicationTests { + + @Autowired + Consumer> jdbcConsumer; + + @Autowired + JdbcOperations jdbcOperations; + + @Autowired + JdbcTemplate jdbcTemplate; + + @AfterEach + public void cleanup() { + jdbcOperations.execute("DROP TABLE MESSAGES IF EXISTS"); + } + + static class Payload { + + private String a; + + private Integer b; + + public Payload() { + } + + public Payload(String a, Integer b) { + this.a = a; + this.b = b; + } + + public String getA() { + return a; + } + + public Integer getB() { + return b; + } + + public void setA(String a) { + this.a = a; + } + + public void setB(Integer b) { + this.b = b; + } + + @Override + public String toString() { + return a + b; + } + + } + + @SpringBootApplication + static class TestApplication {} + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java new file mode 100644 index 00000000..0ffcc35e --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class JsonStringPayloadInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + String stringA = "{\"a\": \"hello1\", \"b\": 42}"; + String stringB = "{\"a\": \"hello2\", \"b\": null}"; + String stringC = "{\"a\": \"hello3\"}"; + final Message message1 = MessageBuilder.withPayload(stringA).build(); + jdbcConsumer.accept(message1); + final Message message2 = MessageBuilder.withPayload(stringB).build(); + jdbcConsumer.accept(message2); + final Message message3 = MessageBuilder.withPayload(stringC).build(); + jdbcConsumer.accept(message3); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b = ?", + Integer.class, "hello1", 42)).isEqualTo(1); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b IS NULL", + Integer.class, "hello2")).isEqualTo(1); + assertThat(jdbcOperations.queryForObject( + "select count(*) from messages where a = ? and b IS NULL", + Integer.class, "hello3")).isEqualTo(1); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java new file mode 100644 index 00000000..e5b38384 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class MapPayloadInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + NamedParameterJdbcOperations namedParameterJdbcOperations = new NamedParameterJdbcTemplate(jdbcOperations); + Map mapA = new HashMap<>(); + mapA.put("a", "hello1"); + mapA.put("b", 42); + Map mapB = new HashMap<>(); + mapB.put("a", "hello2"); + mapB.put("b", null); + Map mapC = new HashMap<>(); + mapC.put("a", "hello3"); + final Message> message1 = MessageBuilder.withPayload(mapA).build(); + jdbcConsumer.accept(message1); + final Message> message2 = MessageBuilder.withPayload(mapB).build(); + jdbcConsumer.accept(message2); + final Message> message3 = MessageBuilder.withPayload(mapC).build(); + jdbcConsumer.accept(message3); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b = :b", mapA, Integer.class)).isEqualTo(1); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b IS NULL", mapB, Integer.class)).isEqualTo(1); + assertThat(namedParameterJdbcOperations.queryForObject( + "select count(*) from messages where a = :a and b IS NULL", mapC, Integer.class)).isEqualTo(1); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java new file mode 100644 index 00000000..ecf8e287 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.batchSize=1000") +public class SimpleBatchInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testBatchInsertion() { + final int numberOfInserts = 5000; + Payload sent = new Payload("hello", 42); + for (int i = 0; i < numberOfInserts; i++) { + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + } + int result = jdbcOperations.queryForObject("select count(*) from messages", Integer.class); + assertThat(result).isEqualTo(numberOfInserts); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java new file mode 100644 index 00000000..f0826573 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +public class SimpleInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testSimpleInsert() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + String result = jdbcOperations.queryForObject("select payload from messages", String.class); + assertThat(result).isEqualTo(("hello42")); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java new file mode 100644 index 00000000..428a266b --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class SimpleMappingTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(sent); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java new file mode 100644 index 00000000..9eb8fd9b --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +// annotation below relies on java.util.Properties so backslash needs to be doubled +@TestPropertySource(properties = "jdbc.consumer.columns=a: a.substring(0\\\\, 4), b: b + 624") +public class SpELTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload sent = new Payload("hello", 42); + final Message message = MessageBuilder.withPayload(sent).build(); + jdbcConsumer.accept(message); + Payload expected = new Payload("hell", 666); + Payload result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); + assertThat(result).isEqualToComparingFieldByField(expected); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java new file mode 100644 index 00000000..9fb69a97 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a: new StringBuilder(payload.a).reverse().toString(), b") +public class UnqualifiableColumnExpressionTests extends JdbcConsumerApplicationTests { + + @Test + public void doesNotFailParsingUnqualifiableExpression() { + // if the app initializes, the test condition passes, but go ahead and apply the column expression anyway + jdbcConsumer.accept(MessageBuilder.withPayload(new Payload("desrever", 123)).build()); + assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?", + Integer.class, "reversed", 123)).isEqualTo(1); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java new file mode 100644 index 00000000..bb48f807 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; + +/** + * @author Eric Bottard + * @author Thomas Risberg + * @author Artem Bilan + * @author Robert St. John + * @author Oliver Flasch + * @author Soby Chacko + * @author Szabolcs Stremler + */ +@TestPropertySource(properties = "jdbc.consumer.columns=a,b") +public class VaryingInsertTests extends JdbcConsumerApplicationTests { + + @Test + public void testInsertion() { + Payload a = new Payload("hello", 42); + Payload b = new Payload("world", 12); + Payload c = new Payload("bonjour", null); + Payload d = new Payload(null, 22); + final Message message1 = MessageBuilder.withPayload(a).build(); + jdbcConsumer.accept(message1); + final Message message2 = MessageBuilder.withPayload(b).build(); + jdbcConsumer.accept(message2); + final Message message3 = MessageBuilder.withPayload(c).build(); + jdbcConsumer.accept(message3); + final Message message4 = MessageBuilder.withPayload(d).build(); + jdbcConsumer.accept(message4); + List result = jdbcOperations + .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)); + Assertions.assertThat(result).extracting("a").containsExactly("hello", "world", "bonjour", null); + Assertions.assertThat(result).extracting("b").contains(42, 12, 22, null); + } + +} diff --git a/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql b/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql new file mode 100644 index 00000000..ac07d2d0 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql @@ -0,0 +1,6 @@ +-- Used in test for explicit script + +create table foobar( + a varchar(2000), + b VARCHAR (2000) +); diff --git a/functions/consumer/jdbc-consumer/src/test/resources/schema.sql b/functions/consumer/jdbc-consumer/src/test/resources/schema.sql new file mode 100644 index 00000000..65e85283 --- /dev/null +++ b/functions/consumer/jdbc-consumer/src/test/resources/schema.sql @@ -0,0 +1,7 @@ +-- Run by default by Boot infrastructure + +create table messages( + a varchar(2000), + b VARCHAR (2000), + payload VARCHAR (2000) +); diff --git a/functions/consumer/log-consumer/.gitignore b/functions/consumer/log-consumer/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/functions/consumer/log-consumer/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/functions/consumer/log-consumer/pom.xml b/functions/consumer/log-consumer/pom.xml new file mode 100644 index 00000000..384dd2c7 --- /dev/null +++ b/functions/consumer/log-consumer/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + log-consumer + 1.0.0.BUILD-SNAPSHOT + log-consumer + Log Consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + io.pivotal.java.function + payload-converter-function + ${project.version} + + + + org.springframework.boot + spring-boot-starter-integration + + + + org.hibernate.validator + hibernate-validator + true + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.integration + spring-integration-test + test + + + + diff --git a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java new file mode 100644 index 00000000..3b17411e --- /dev/null +++ b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import java.util.function.Consumer; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.messaging.Message; + +/** + * The Configuration class for {@link Consumer} which logs incoming data. + * For the logging logic a Spring Integration {@link org.springframework.integration.handler.LoggingHandler} + * is used. + * If incoming payload is a {@code byte[]} and incoming message {@code contentType} header is text-compatible + * (e.g. {@code application/json}), it is converted into a {@link String}. + * Otherwise the payload is passed to logger as is. + * + * @author Artem Bilan + */ +@Configuration +@EnableConfigurationProperties(LogConsumerProperties.class) +public class LogConsumerConfiguration { + + @Bean + IntegrationFlow logConsumerFlow(LogConsumerProperties logSinkProperties) { + return IntegrationFlows.from(MessageConsumer.class, (gateway) -> gateway.beanName("logConsumer")) + .handle((payload, headers) -> payload) + .log(logSinkProperties.getLevel(), logSinkProperties.getName(), logSinkProperties.getExpression()) + .get(); + } + + private interface MessageConsumer extends Consumer> {} + +} diff --git a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java new file mode 100644 index 00000000..ff9b9528 --- /dev/null +++ b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import static org.springframework.integration.handler.LoggingHandler.Level.INFO; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.handler.LoggingHandler; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the Log Sink app. + * + * @author Gary Russell + * @author Eric Bottard + * @author Chris Schaefer + * @author Artem Bilan + */ +@ConfigurationProperties("log") +@Validated +public class LogConsumerProperties { + + /** + * The name of the logger to use. + */ + @Value("${spring.application.name:log.consumer}") + private String name; + + /** + * A SpEL expression (against the incoming message) to evaluate as the logged message. + */ + private String expression = "payload"; + + /** + * The level at which to log messages. + */ + private LoggingHandler.Level level = INFO; + + @NotBlank + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @NotBlank + public String getExpression() { + return expression; + } + + public void setExpression(String expression) { + this.expression = expression; + } + + @NotNull + public LoggingHandler.Level getLevel() { + return level; + } + + public void setLevel(LoggingHandler.Level level) { + this.level = level; + } + +} diff --git a/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java b/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java new file mode 100644 index 00000000..cf2b837f --- /dev/null +++ b/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.log; + +import java.util.function.Consumer; + +import org.apache.commons.logging.Log; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.handler.LoggingHandler; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.MimeType; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author Artem Bilan + */ +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@SpringBootTest({ "log.name=foo", "log.level=warn", "log.expression=payload.toUpperCase()" }) +class LogConsumerApplicationTests { + + @Autowired + private Consumer> logConsumer; + + @Autowired + @Qualifier("logConsumerFlow.logging-channel-adapter#0") + private LoggingHandler loggingHandler; + + @Test + public void testJsonContentType() { + Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") + .setHeader("contentType", new MimeType("json")) + .build(); + testMessage(message, "{\"foo\":\"bar\"}"); + } + + private void testMessage(Message message, String expectedPayload) { + assertThat(this.loggingHandler.getLevel()).isEqualTo(LoggingHandler.Level.WARN); + Log logger = TestUtils.getPropertyValue(this.loggingHandler, "messageLogger", Log.class); + assertThat(TestUtils.getPropertyValue(logger, "logger.name")).isEqualTo("foo"); + logger = spy(logger); + new DirectFieldAccessor(this.loggingHandler).setPropertyValue("messageLogger", logger); + this.logConsumer.accept(message); + ArgumentCaptor captor = ArgumentCaptor.forClass(Object.class); + verify(logger).warn(captor.capture()); + assertThat(captor.getValue()).isEqualTo(expectedPayload.toUpperCase()); + this.loggingHandler.setLogExpressionString("#this"); + this.logConsumer.accept(message); + verify(logger, times(2)).warn(captor.capture()); + + Message captorMessage = (Message) captor.getAllValues().get(2); + assertThat(captorMessage.getPayload()).isEqualTo(expectedPayload); + + MessageHeaders messageHeaders = captorMessage.getHeaders(); + assertThat(messageHeaders).hasSize(3); + + assertThat(messageHeaders) + .containsEntry(MessageHeaders.CONTENT_TYPE, message.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/consumer/mongodb-consumer/.gitignore b/functions/consumer/mongodb-consumer/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/functions/consumer/mongodb-consumer/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/functions/consumer/mongodb-consumer/pom.xml b/functions/consumer/mongodb-consumer/pom.xml new file mode 100644 index 00000000..c4cca0ed --- /dev/null +++ b/functions/consumer/mongodb-consumer/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + mongodb-consumer + 1.0.0.BUILD-SNAPSHOT + mongodb-consumer + Mongo DB consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-mongodb + + + org.mongodb + mongodb-driver-reactivestreams + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + io.projectreactor + reactor-test + test + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java new file mode 100644 index 00000000..e2315858 --- /dev/null +++ b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import java.util.function.Function; + +import reactor.core.publisher.Mono; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.ReactiveMongoTemplate; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.integration.mongodb.outbound.ReactiveMongoDbStoringMessageHandler; +import org.springframework.messaging.Message; +import org.springframework.messaging.ReactiveMessageHandler; + +/** + * A configuration for MongoDB Consumer function. Uses a + * {@link ReactiveMongoDbStoringMessageHandler} to save payload contents to Mongo DB. + * + * @author Artem Bilan + * @author David Turanski + * + */ +@Configuration +@EnableConfigurationProperties({ MongoDbConsumerProperties.class }) +public class MongoDbConsumerConfiguration { + + private final MongoDbConsumerProperties properties; + + private final ReactiveMongoTemplate mongoTemplate; + + public MongoDbConsumerConfiguration(MongoDbConsumerProperties properties, ReactiveMongoTemplate mongoTemplate) { + this.properties = properties; + this.mongoTemplate = mongoTemplate; + } + + @Bean + public Function, Mono> mongodbConsumer(ReactiveMessageHandler mongoConsumerMessageHandler) { + return mongoConsumerMessageHandler::handleMessage; + } + + @Bean + public ReactiveMessageHandler mongoConsumerMessageHandler() { + ReactiveMongoDbStoringMessageHandler mongoDbMessageHandler = new ReactiveMongoDbStoringMessageHandler( + this.mongoTemplate); + Expression collectionExpression = this.properties.getCollectionExpression(); + if (collectionExpression == null) { + collectionExpression = new LiteralExpression(this.properties.getCollection()); + } + mongoDbMessageHandler.setCollectionNameExpression(collectionExpression); + return mongoDbMessageHandler; + } +} diff --git a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java new file mode 100644 index 00000000..838cbf4c --- /dev/null +++ b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; +/** + * @author Artem Bilan + * @author David Turanski + * + */ +@ConfigurationProperties("mongodb.consumer") +@Validated +public class MongoDbConsumerProperties { + + /** + * The MongoDB collection to store data + */ + private String collection; + + /** + * The SpEL expression to evaluate MongoDB collection + */ + private Expression collectionExpression; + + public void setCollection(String collection) { + this.collection = collection; + } + + public String getCollection() { + return this.collection; + } + + public void setCollectionExpression(Expression collectionExpression) { + this.collectionExpression = collectionExpression; + } + + public Expression getCollectionExpression() { + return collectionExpression; + } + + @AssertTrue(message = "One of 'collection' or 'collectionExpression' is required") + private boolean isValid() { + return StringUtils.hasText(this.collection) || this.collectionExpression != null; + } +} diff --git a/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java b/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java new file mode 100644 index 00000000..d3c1a3fa --- /dev/null +++ b/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.mongo; + +import java.time.Duration; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import java.util.function.Function; +import org.bson.Document; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.mongodb.core.ReactiveMongoTemplate; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author David Turanski + */ +@SpringBootTest(properties = { + "spring.data.mongodb.port=0", + "mongodb.consumer.collection=testing"}) +class MongoDbConsumerApplicationTests { + + @Autowired + private MongoDbConsumerProperties properties; + + @Autowired + private Function, Mono> mongoDbConsumer; + + @Autowired + private ReactiveMongoTemplate mongoTemplate; + + @Test + void testMongodbConsumer() { + Map data1 = new HashMap<>(); + data1.put("foo", "bar"); + + Map data2 = new HashMap<>(); + data2.put("firstName", "Foo"); + data2.put("lastName", "Bar"); + + Flux> messages = Flux.just( + new GenericMessage<>(data1), + new GenericMessage<>(data2), + new GenericMessage<>("{\"my_data\": \"THE DATA\"}") + ); + + messages.flatMap(mongoDbConsumer::apply).blockLast(Duration.ofSeconds(10)); + + StepVerifier.create(this.mongoTemplate.findAll(Document.class, properties.getCollection()) + .sort(Comparator.comparing(d -> d.get("_id").toString()))) + .assertNext(document -> { + assertThat(document.get("foo")).isEqualTo("bar"); + }) + .assertNext(document-> { + assertThat(document.get("firstName")).isEqualTo("Foo"); + assertThat(document.get("lastName")).isEqualTo("Bar"); + }) + .assertNext(document-> { + assertThat(document.get("my_data")).isEqualTo("THE DATA"); + }) + .verifyComplete(); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/consumer/rabbit-consumer/pom.xml b/functions/consumer/rabbit-consumer/pom.xml new file mode 100644 index 00000000..4443782d --- /dev/null +++ b/functions/consumer/rabbit-consumer/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + rabbit-consumer + 1.0.0.BUILD-SNAPSHOT + rabbit-consumer + Rabbit consumer + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-amqp + + + org.springframework.boot + spring-boot-starter-amqp + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + javax.validation + validation-api + + + + diff --git a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java new file mode 100644 index 00000000..dd89486e --- /dev/null +++ b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java @@ -0,0 +1,150 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.rabbit; + +import java.util.function.Function; + +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionNameStrategy; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.Expression; +import org.springframework.integration.amqp.dsl.Amqp; +import org.springframework.integration.amqp.dsl.AmqpOutboundChannelAdapterSpec; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; + +@EnableConfigurationProperties(RabbitConsumerProperties.class) +@Configuration +public class RabbitConsumerConfiguration implements DisposableBean { + + @Autowired + private RabbitProperties bootProperties; + + @Autowired + private ObjectProvider connectionNameStrategy; + + @Autowired + private RabbitConsumerProperties properties; + + @Value("#{${rabbit.converterBeanName:null}}") + private MessageConverter messageConverter; + + private CachingConnectionFactory ownConnectionFactory; + + @Bean + public Function, Object> rabbitConsumer(@Qualifier("amqpChannelAdapter") MessageHandler messageHandler) { + return o -> { + messageHandler.handleMessage(o); + return ""; + }; + } + + @Bean + public MessageHandler amqpChannelAdapter(ConnectionFactory rabbitConnectionFactory) + throws Exception { + + AmqpOutboundChannelAdapterSpec handler = Amqp + .outboundAdapter(rabbitTemplate(this.properties.isOwnConnection() + ? buildLocalConnectionFactory() : rabbitConnectionFactory)) + .mappedRequestHeaders(properties.getMappedRequestHeaders()) + .defaultDeliveryMode(properties.getPersistentDeliveryMode() + ? MessageDeliveryMode.PERSISTENT + : MessageDeliveryMode.NON_PERSISTENT); + + Expression exchangeExpression = this.properties.getExchangeExpression(); + if (exchangeExpression != null) { + handler.exchangeNameExpression(exchangeExpression); + } + else { + handler.exchangeName(this.properties.getExchange()); + } + + Expression routingKeyExpression = this.properties.getRoutingKeyExpression(); + if (routingKeyExpression != null) { + handler.routingKeyExpression(routingKeyExpression); + } + else { + handler.routingKey(this.properties.getRoutingKey()); + } + return handler.get(); + } + + private ConnectionFactory buildLocalConnectionFactory() throws Exception { + this.ownConnectionFactory = new AutoConfig.Creator().rabbitConnectionFactory( + this.bootProperties, this.connectionNameStrategy); + return this.ownConnectionFactory; + } + + @Bean + public RabbitTemplate rabbitTemplate(ConnectionFactory rabbitConnectionFactory) { + RabbitTemplate rabbitTemplate = new RabbitTemplate(rabbitConnectionFactory); + if (this.messageConverter != null) { + rabbitTemplate.setMessageConverter(this.messageConverter); + } + return rabbitTemplate; + } + + @Bean + @ConditionalOnProperty(name = "rabbit.converterBeanName", + havingValue = RabbitConsumerProperties.JSON_CONVERTER) + public Jackson2JsonMessageConverter jsonConverter() { + return new Jackson2JsonMessageConverter(); + } + + @Override + public void destroy() { + if (this.ownConnectionFactory != null) { + this.ownConnectionFactory.destroy(); + } + } + +} + +class AutoConfig extends RabbitAutoConfiguration { + + static class Creator extends RabbitConnectionFactoryCreator { + + @Override + public CachingConnectionFactory rabbitConnectionFactory(RabbitProperties config, + ObjectProvider connectionNameStrategy) + throws Exception { + CachingConnectionFactory cf = super.rabbitConnectionFactory(config, + connectionNameStrategy); + cf.setConnectionNameStrategy( + connectionFactory -> "rabbit.sink.own.connection"); + cf.afterPropertiesSet(); + return cf; + } + + } + +} diff --git a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java new file mode 100644 index 00000000..654a9c2b --- /dev/null +++ b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.consumer.rabbit; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.validation.annotation.Validated; + +@ConfigurationProperties("rabbit") +@Validated +public class RabbitConsumerProperties { + + public static final String JSON_CONVERTER = "jsonConverter"; + + /** + * Exchange name - overridden by exchangeNameExpression, if supplied. + */ + private String exchange = ""; + + /** + * A SpEL expression that evaluates to an exchange name. + */ + private Expression exchangeExpression; + + /** + * Routing key - overridden by routingKeyExpression, if supplied. + */ + private String routingKey; + + /** + * A SpEL expression that evaluates to a routing key. + */ + private Expression routingKeyExpression; + + /** + * Default delivery mode when 'amqp_deliveryMode' header is not present, + * true for PERSISTENT. + */ + private boolean persistentDeliveryMode; + + /** + * Headers that will be mapped. + */ + private String[] mappedRequestHeaders = { "*" }; + + /** + * The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. + * If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you. + */ + private String converterBeanName; + + /** + * When true, use a separate connection based on the boot properties. + */ + private boolean ownConnection; + + public String getExchange() { + return this.exchange; + } + + public void setExchange(String exchange) { + this.exchange = exchange; + } + + public Expression getExchangeExpression() { + return this.exchangeExpression; + } + + public void setExchangeExpression(Expression exchangeExpression) { + this.exchangeExpression = exchangeExpression; + } + + public String getRoutingKey() { + return this.routingKey; + } + + public void setRoutingKey(String routingKey) { + this.routingKey = routingKey; + } + + public Expression getRoutingKeyExpression() { + return this.routingKeyExpression; + } + + public void setRoutingKeyExpression(Expression routingKeyExpression) { + this.routingKeyExpression = routingKeyExpression; + } + + public boolean getPersistentDeliveryMode() { + return this.persistentDeliveryMode; + } + + public void setPersistentDeliveryMode(boolean persistentDeliveryMode) { + this.persistentDeliveryMode = persistentDeliveryMode; + } + + public String[] getMappedRequestHeaders() { + return this.mappedRequestHeaders; + } + + public void setMappedRequestHeaders(String[] mappedRequestHeaders) { + this.mappedRequestHeaders = mappedRequestHeaders; + } + + public String getConverterBeanName() { + return this.converterBeanName; + } + + public void setConverterBeanName(String converterBeanName) { + this.converterBeanName = converterBeanName; + } + + @AssertTrue(message = "routingKey or routingKeyExpression is required") + public boolean isRoutingKeyProvided() { + return this.routingKey != null || this.routingKeyExpression != null; + } + + public boolean isOwnConnection() { + return this.ownConnection; + } + + public void setOwnConnection(boolean ownConnection) { + this.ownConnection = ownConnection; + } + +} \ No newline at end of file diff --git a/functions/function/filter-function/.gitignore b/functions/function/filter-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/functions/function/filter-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/functions/function/filter-function/pom.xml b/functions/function/filter-function/pom.xml new file mode 100644 index 00000000..5e5b72d5 --- /dev/null +++ b/functions/function/filter-function/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + filter-function + 1.0.0.BUILD-SNAPSHOT + filter-function + Spring Native Function for applying filter SpEL expressions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.fn + spel-function + 1.0.0.BUILD-SNAPSHOT + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java b/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java new file mode 100644 index 00000000..9859446c --- /dev/null +++ b/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.filter; + +import java.util.Optional; +import java.util.function.Function; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.cloud.fn.spel.SpelFunctionConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; + +@Configuration +@Import(SpelFunctionConfiguration.class) +public class FilterFunctionConfiguration { + + @Bean + public Function, Message> filterFunction( + @Qualifier("spelFunction") Function, Message> spelFunction) { + + return message -> + Optional.of(message) + .filter(m -> (Boolean) spelFunction.apply(m).getPayload()) + .orElse(null); + } + +} diff --git a/functions/function/filter-function/src/main/resources/application.properties b/functions/function/filter-function/src/main/resources/application.properties new file mode 100644 index 00000000..86e445ae --- /dev/null +++ b/functions/function/filter-function/src/main/resources/application.properties @@ -0,0 +1 @@ +spel.function.expression=true diff --git a/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java b/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java new file mode 100644 index 00000000..23afb9be --- /dev/null +++ b/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.filter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(properties = "spel.function.expression=payload.length() > 5") +@DirtiesContext +public class FilterFunctionApplicationTests { + + @Autowired + @Qualifier("filterFunction") + Function, Message> filter; + + @Test + public void testFilter() { + Message filtered = this.filter.apply(new GenericMessage<>("hello")); + assertThat(filtered).isNull(); + filtered = this.filter.apply(new GenericMessage<>("hello world")); + assertThat(filtered).isNotNull() + .extracting(Message::getPayload) + .isEqualTo("hello world"); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/function/payload-converter-function/pom.xml b/functions/function/payload-converter-function/pom.xml new file mode 100644 index 00000000..846b9c54 --- /dev/null +++ b/functions/function/payload-converter-function/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + payload-converter-function + 1.0.0.BUILD-SNAPSHOT + payload-converter-function + Utility message conversion functions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework + spring-messaging + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + diff --git a/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java b/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java new file mode 100644 index 00000000..2aecc905 --- /dev/null +++ b/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functions; + +import java.util.function.Function; + +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.MimeTypeUtils; + +/** + * + * @author Christian Tzolov + */ +public class ByteArrayTextToString implements Function, Message> { + + @Override + public Message apply(Message message) { + + if (message.getPayload() instanceof byte[]) { + final MessageHeaders headers = message.getHeaders(); + String contentType = headers.containsKey(MessageHeaders.CONTENT_TYPE) + ? headers.get(MessageHeaders.CONTENT_TYPE).toString() + : MimeTypeUtils.APPLICATION_JSON_VALUE; + + if (contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple")) { + message = MessageBuilder.withPayload(new String(((byte[]) message.getPayload()))) + .copyHeaders(message.getHeaders()) + .build(); + } + } + + return message; + } +} + diff --git a/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java b/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java new file mode 100644 index 00000000..24b8a93e --- /dev/null +++ b/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functions; + +import java.util.Collections; +import java.util.function.Function; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ByteArrayTextToStringTests { + + private static final String MESSAGE = "hello world"; + private static Function, Message> converter; + + @BeforeAll + static void before() { + converter = new ByteArrayTextToString(); + } + + @Test + public void testDefaultNoContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes())); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + + converted = converter.apply(new GenericMessage<>(MESSAGE)); // String + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + } + + @Test + public void testApplicationJsonContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo("hello world"); + } + + @Test + public void testPlainTextContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); + } + + @Test + public void testOctetContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); + } + + @Test + public void testRandomNonTextContentType() { + Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), + Collections.singletonMap(MessageHeaders.CONTENT_TYPE, "Random Content Type"))); + assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); + } + +} diff --git a/functions/function/spel-function/.gitignore b/functions/function/spel-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/functions/function/spel-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/functions/function/spel-function/pom.xml b/functions/function/spel-function/pom.xml new file mode 100644 index 00000000..f11c25e7 --- /dev/null +++ b/functions/function/spel-function/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + spel-function + 1.0.0.BUILD-SNAPSHOT + spel-function + Spring Native Function for applying SpEL expressions + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.cloud.function + payload-converter-function + ${project.version} + + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java new file mode 100644 index 00000000..9e682b1e --- /dev/null +++ b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import java.util.function.Function; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.transformer.ExpressionEvaluatingTransformer; +import org.springframework.messaging.Message; + +@Configuration +@EnableConfigurationProperties(SpelFunctionProperties.class) +public class SpelFunctionConfiguration { + + @Bean + public Function, Message> spelFunction( + ExpressionEvaluatingTransformer expressionEvaluatingTransformer) { + + return message -> expressionEvaluatingTransformer.transform(message); + } + + @Bean + public ExpressionEvaluatingTransformer expressionEvaluatingTransformer( + SpelFunctionProperties spelFunctionProperties) { + + return new ExpressionEvaluatingTransformer(new SpelExpressionParser() + .parseExpression(spelFunctionProperties.getExpression())); + } + +} diff --git a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java new file mode 100644 index 00000000..89bfec3f --- /dev/null +++ b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +/** + * Configuration properties for the SpEL function. + * + * @author Gary Russell + * @author Artem Bilan + */ +@ConfigurationProperties("spel.function") +public class SpelFunctionProperties { + + private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); + + /** + * A SpEL expression to apply. + */ + private String expression = DEFAULT_EXPRESSION.getExpressionString(); + + public void setExpression(String expression) { + this.expression = expression; + } + + public String getExpression() { + return this.expression; + } + +} diff --git a/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java b/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java new file mode 100644 index 00000000..1ef016c3 --- /dev/null +++ b/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.spel; + +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.util.MimeTypeUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = "spel.function.expression=payload.toUpperCase()") +@DirtiesContext +public class SpelFunctionApplicationTests { + + @Autowired + Function, Message> transformer; + + @Test + public void testTransform() { + final Message transformed = this.transformer.apply(new GenericMessage<>("hello,world")); + assertThat(transformed.getPayload()).isEqualTo("HELLO,WORLD"); + } + + @Test + public void testJson() { + Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON).build(); + final Message transformed = this.transformer.apply(message); + assertThat(transformed.getPayload()).isEqualTo("{\"FOO\":\"BAR\"}"); + } + + @SpringBootApplication + static class TestApplication { + + } +} diff --git a/functions/function/splitter-function/.gitignore b/functions/function/splitter-function/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/functions/function/splitter-function/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/functions/function/splitter-function/pom.xml b/functions/function/splitter-function/pom.xml new file mode 100644 index 00000000..fcdeaf2d --- /dev/null +++ b/functions/function/splitter-function/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + splitter-function + 1.0.0.BUILD-SNAPSHOT + splitter-function + Spring Native Function for Splitter + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-validation + + + + org.springframework.integration + spring-integration-file + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java new file mode 100644 index 00000000..7d1897c7 --- /dev/null +++ b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import java.nio.charset.Charset; +import java.util.List; +import java.util.function.Function; + +import org.reactivestreams.Publisher; + +import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.channel.ReactiveStreamsSubscribableChannel; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.splitter.AbstractMessageSplitter; +import org.springframework.integration.splitter.DefaultMessageSplitter; +import org.springframework.integration.splitter.ExpressionEvaluatingSplitter; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; + +import reactor.core.publisher.Flux; + +@Configuration +@EnableConfigurationProperties(SplitterFunctionProperties.class) +public class SplitterFunctionConfiguration { + + @Bean + public Function, List>> splitterFunction(AbstractMessageSplitter messageSplitter, + SplitterFunctionProperties splitterFunctionProperties) { + + messageSplitter.setApplySequence(splitterFunctionProperties.isApplySequence()); + ThreadLocalFluxSinkMessageChannel outputChannel = new ThreadLocalFluxSinkMessageChannel(); + messageSplitter.setOutputChannel(outputChannel); + return message -> { + messageSplitter.handleMessage(message); + return outputChannel.publisherThreadLocal.get(); + }; + } + + @Bean + @ConditionalOnProperty(prefix = "splitter", name = "expression") + public AbstractMessageSplitter expressionSplitter(SplitterFunctionProperties splitterFunctionProperties) { + return new ExpressionEvaluatingSplitter( + new SpelExpressionParser() + .parseExpression(splitterFunctionProperties.getExpression())); + } + + @Bean + @ConditionalOnMissingBean + @Conditional(FileSplitterCondition.class) + public AbstractMessageSplitter fileSplitter(SplitterFunctionProperties splitterFunctionProperties) { + Boolean markers = splitterFunctionProperties.getFileMarkers(); + String charset = splitterFunctionProperties.getCharset(); + if (markers == null) { + markers = false; + } + FileSplitter fileSplitter = new FileSplitter(true, markers, splitterFunctionProperties.getMarkersJson()); + if (charset != null) { + fileSplitter.setCharset(Charset.forName(charset)); + } + return fileSplitter; + } + + + @Bean + @ConditionalOnMissingBean + public AbstractMessageSplitter defaultSplitter(SplitterFunctionProperties splitterFunctionProperties) { + DefaultMessageSplitter defaultMessageSplitter = new DefaultMessageSplitter(); + defaultMessageSplitter.setDelimiters(splitterFunctionProperties.getDelimiters()); + return defaultMessageSplitter; + } + + static class FileSplitterCondition extends AnyNestedCondition { + + FileSplitterCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(prefix = "splitter", name = "charset") + static class Charset { } + + @ConditionalOnProperty(prefix = "splitter", name = "fileMarkers") + static class FileMarkers { } + + } + + private static final class ThreadLocalFluxSinkMessageChannel + implements MessageChannel, ReactiveStreamsSubscribableChannel { + + private final ThreadLocal>> publisherThreadLocal = new ThreadLocal<>(); + + @Override + @SuppressWarnings("unchecked") + public void subscribeTo(Publisher> publisher) { + this.publisherThreadLocal.set(Flux.from(publisher).collectList().cast(List.class).block()); + } + + @Override + public boolean send(Message message, long l) { + throw new UnsupportedOperationException("This channel only supports a reactive 'subscribeTo()' "); + } + + } + +} diff --git a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java new file mode 100644 index 00000000..9749f95f --- /dev/null +++ b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java @@ -0,0 +1,128 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import javax.validation.constraints.AssertTrue; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the Splitter Processor app. + * + * @author Gary Russell + * @author Artem Bilan + */ +@ConfigurationProperties("splitter") +@Validated +public class SplitterFunctionProperties { + + /** + * A SpEL expression for splitting payloads. + */ + private String expression; + + /** + * When expression is null, delimiters to use when tokenizing + * {@link String} payloads. + */ + private String delimiters; + + /** + * Set to true or false to use a {@code FileSplitter} (to split + * text-based files by line) that includes + * (or not) beginning/end of file markers. + */ + private Boolean fileMarkers; + + /** + * When 'fileMarkers == true', specify if they should be produced + * as FileSplitter.FileMarker objects or JSON. + */ + private boolean markersJson = true; + + /** + * The charset to use when converting bytes in text-based files + * to String. + */ + private String charset; + + /** + * Add correlation/sequence information in headers to facilitate later + * aggregation. + */ + private boolean applySequence = true; + + public void setExpression(String expression) { + this.expression = expression; + } + + public String getExpression() { + return this.expression; + } + + public String getDelimiters() { + return this.delimiters; + } + + public void setDelimiters(String delimiters) { + this.delimiters = delimiters; + } + + public Boolean getFileMarkers() { + return this.fileMarkers; + } + + public void setFileMarkers(Boolean fileMarkers) { + this.fileMarkers = fileMarkers; + } + + public boolean getMarkersJson() { + return this.markersJson; + } + + public void setMarkersJson(boolean markersJson) { + this.markersJson = markersJson; + } + + public String getCharset() { + return this.charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public boolean isApplySequence() { + return this.applySequence; + } + + public void setApplySequence(boolean applySequence) { + this.applySequence = applySequence; + } + + @AssertTrue(message = "'delimiters' is not allowed when an 'expression' is provided") + public boolean isDelimitersAllowed() { + return this.expression == null || this.delimiters == null; + } + + @AssertTrue(message = "File properties are not allowed when an 'expression' or 'delimiters' property is provided") + public boolean isFilePropsAllowed() { + return !(this.expression != null || this.delimiters != null) || this.fileMarkers == null && this.charset == null; + } + +} diff --git a/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java b/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java new file mode 100644 index 00000000..6a8a5f94 --- /dev/null +++ b/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011-2020 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.splitter; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.annotation.DirtiesContext; + +@SpringBootTest(properties = "splitter.expression=payload.split(',')") +@DirtiesContext +public class SplitterFunctionApplicationTests { + + @Autowired + Function, List>> splitter; + + @Test + public void testExpressionSplitter() { + List> messageList = this.splitter.apply(new GenericMessage<>("hello,world")); + assertThat(messageList).extracting(m -> m.getPayload().toString()).contains("hello", "world"); + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/pom.xml b/functions/pom.xml new file mode 100644 index 00000000..cfd3fd98 --- /dev/null +++ b/functions/pom.xml @@ -0,0 +1,205 @@ + + + 4.0.0 + org.springframework.cloud.fn + java-functions-parent + 1.0.0.BUILD-SNAPSHOT + java-functions-parent + Pivotal Java Functions Parent + pom + + + 1.8 + 3.1.1 + 3.2.1 + 2.22.2 + UTF-8 + UTF-8 + ${java.version} + ${java.version} + + + + consumer/cassandra-consumer + consumer/counter-consumer + consumer/file-consumer + consumer/jdbc-consumer + consumer/log-consumer + consumer/mongodb-consumer + consumer/rabbit-consumer + + function/filter-function + function/spel-function + function/payload-converter-function + function/splitter-function + + supplier/file-supplier + supplier/http-supplier + supplier/jdbc-supplier + supplier/mongodb-supplier + supplier/time-supplier + + spring-functions-parent + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + Copyright 2014-2020 the original author or authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. + + See the License for the specific language governing permissions and + limitations under the License. + + + + scm:git:git://github.com/pivotal/java-functions.git + scm:git:ssh://git@github.com/pivotal/java-functions.git + https://github.com/pivotal/java-functions + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + javadoc + package + + jar + + + + + true + + + + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + package + + jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + **/*Tests.java + **/*Test.java + + + **/Abstract*.java + + + + + + + + repo.spring.io + Spring Release Repository + https://repo.spring.io/libs-release-local + + + repo.spring.io + Spring Snapshot Repository + https://repo.spring.io/libs-snapshot-local + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + + + + + milestone + + + repo.spring.io + Spring Milestone Repository + https://repo.spring.io/libs-milestone-local + + + + + central + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + + + + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + + + diff --git a/functions/spring-functions-parent/pom.xml b/functions/spring-functions-parent/pom.xml new file mode 100644 index 00000000..b645f407 --- /dev/null +++ b/functions/spring-functions-parent/pom.xml @@ -0,0 +1,40 @@ + + + + java-functions-parent + org.springframework.cloud.fn + 1.0.0.BUILD-SNAPSHOT + .. + + 4.0.0 + + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + pom + + + 2.3.0.M4 + 3.0.3.RELEASE + + + + + org.springframework.boot + spring-boot-starter-parent + ${spring-boot.version} + import + pom + + + org.springframework.cloud + spring-cloud-function-dependencies + ${spring-cloud-function.version} + import + pom + + + + + \ No newline at end of file diff --git a/functions/supplier/file-supplier/pom.xml b/functions/supplier/file-supplier/pom.xml new file mode 100644 index 00000000..43d085de --- /dev/null +++ b/functions/supplier/file-supplier/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + file-supplier + 1.0.0.BUILD-SNAPSHOT + file-supplier + file supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-file + + + org.springframework.boot + spring-boot-starter-integration + + + org.springframework.boot + spring-boot-starter-json + true + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + + diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java new file mode 100644 index 00000000..b5c08234 --- /dev/null +++ b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * TODO: This will be used in other apps like (S)FTP and S3. Therefore, it might be moved to a common place. + * + * @author David Turanski + * @author Artem Bilan + */ +@ConfigurationProperties("file.consumer") +@Validated +public class FileConsumerProperties { + + /** + * The FileReadingMode to use for file reading sources. + * Values are 'ref' - The File object, + * 'lines' - a message per line, or + * 'contents' - the contents as bytes. + */ + private FileReadingMode mode = FileReadingMode.contents; + + /** + * Set to true to emit start of file/end of file marker messages before/after the data. + * Only valid with FileReadingMode 'lines'. + */ + private Boolean withMarkers = null; + + /** + * When 'fileMarkers == true', specify if they should be produced + * as FileSplitter.FileMarker objects or JSON. + */ + private boolean markersJson = true; + + @NotNull + public FileReadingMode getMode() { + return this.mode; + } + + public void setMode(FileReadingMode mode) { + this.mode = mode; + } + + public Boolean getWithMarkers() { + return this.withMarkers; + } + + public void setWithMarkers(Boolean withMarkers) { + this.withMarkers = withMarkers; + } + + public boolean getMarkersJson() { + return this.markersJson; + } + + public void setMarkersJson(boolean markersJson) { + this.markersJson = markersJson; + } + + @AssertTrue(message = "withMarkers can only be supplied when FileReadingMode is 'lines'") + public boolean isWithMarkersValid() { + return this.withMarkers == null || FileReadingMode.lines == this.mode; + } +} diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java new file mode 100644 index 00000000..3d357733 --- /dev/null +++ b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileReadingMode.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +/** + * Defines the supported modes of reading and processing files. + * + * @author Gunnar Hillert + * @author David Turanski + */ +public enum FileReadingMode { + ref, + lines, + contents; +} diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java new file mode 100644 index 00000000..ff4139a6 --- /dev/null +++ b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java @@ -0,0 +1,104 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.util.function.Supplier; + +import org.reactivestreams.Publisher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.file.FileReadingMessageSource; +import org.springframework.integration.file.dsl.FileInboundChannelAdapterSpec; +import org.springframework.integration.file.dsl.Files; +import org.springframework.messaging.Message; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * @author Artem Bilan + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties({FileSupplierProperties.class, FileConsumerProperties.class}) +public class FileSupplierConfiguration { + + private final FileSupplierProperties fileSupplierProperties; + + private final FileConsumerProperties fileConsumerProperties; + + @Autowired + @Lazy + @Qualifier("fileMessageSource") + private FileReadingMessageSource fileMessageSource; + + public FileSupplierConfiguration(FileSupplierProperties fileSupplierProperties, + FileConsumerProperties fileConsumerProperties) { + this.fileSupplierProperties = fileSupplierProperties; + this.fileConsumerProperties = fileConsumerProperties; + } + + @Bean + public FileInboundChannelAdapterSpec fileMessageSource() { + final FileInboundChannelAdapterSpec fileInboundChannelAdapterSpec = + Files.inboundAdapter(this.fileSupplierProperties.getDirectory()); + if (StringUtils.hasText(this.fileSupplierProperties.getFilenamePattern())) { + fileInboundChannelAdapterSpec.patternFilter(this.fileSupplierProperties.getFilenamePattern()); + } + else if (this.fileSupplierProperties.getFilenameRegex() != null) { + fileInboundChannelAdapterSpec.regexFilter(this.fileSupplierProperties.getFilenameRegex().pattern()); + } + fileInboundChannelAdapterSpec.preventDuplicates(this.fileSupplierProperties.isPreventDuplicates()); + return fileInboundChannelAdapterSpec; + } + + @Bean + public Flux> fileMessageFlux() { + return Mono.>create(monoSink -> + monoSink.onRequest(value -> + monoSink.success(this.fileMessageSource.receive()))) + .subscribeOn(Schedulers.boundedElastic()) + .repeatWhenEmpty(it -> it.delayElements(this.fileSupplierProperties.getDelayWhenEmpty())) + .repeat(); + } + + @Bean + @ConditionalOnExpression("environment['file.consumer.mode'] != 'ref'") + public Publisher> fileReadingFlow() { + IntegrationFlowBuilder flowBuilder = IntegrationFlows.from(fileMessageFlux()); + return FileUtils.enhanceFlowForReadingMode(flowBuilder, this.fileConsumerProperties) + .toReactivePublisher(); + } + + @Bean + public Supplier>> fileSupplier() { + if (this.fileConsumerProperties.getMode() == FileReadingMode.ref) { + return this::fileMessageFlux; + } + else { + return () -> Flux.from(fileReadingFlow()); + } + } +} diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java new file mode 100644 index 00000000..3e14582b --- /dev/null +++ b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java @@ -0,0 +1,112 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.File; +import java.time.Duration; +import java.util.regex.Pattern; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Properties for the file supplier. + * + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@ConfigurationProperties("file.supplier") +@Validated +public class FileSupplierProperties { + + private static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + + File.separator + "file-supplier"; + + /** + * The directory to poll for new files. + */ + private File directory = new File(DEFAULT_DIR); + + /** + * Set to true to include an AcceptOnceFileListFilter which prevents duplicates. + */ + private boolean preventDuplicates = true; + + /** + * A simple ant pattern to match files. + */ + private String filenamePattern; + + /** + * A regex pattern to match files. + */ + private Pattern filenameRegex; + + /** + * Duration of delay when no new files are detected. + */ + private Duration delayWhenEmpty = Duration.ofSeconds(1); + + public File getDirectory() { + return this.directory; + } + + public void setDirectory(File directory) { + this.directory = directory; + } + + public boolean isPreventDuplicates() { + return this.preventDuplicates; + } + + public void setPreventDuplicates(boolean preventDuplicates) { + this.preventDuplicates = preventDuplicates; + } + + public String getFilenamePattern() { + return this.filenamePattern; + } + + public void setFilenamePattern(String filenamePattern) { + this.filenamePattern = filenamePattern; + } + + public Pattern getFilenameRegex() { + return this.filenameRegex; + } + + public void setFilenameRegex(Pattern filenameRegex) { + this.filenameRegex = filenameRegex; + } + + //@AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") + + public boolean isExclusivePatterns() { + return !(this.filenamePattern != null && this.filenameRegex != null); + } + + public Duration getDelayWhenEmpty() { + return delayWhenEmpty; + } + + public void setDelayWhenEmpty(Duration delayWhenEmpty) { + this.delayWhenEmpty = delayWhenEmpty; + } + + +} diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java new file mode 100644 index 00000000..7e1e04e0 --- /dev/null +++ b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileUtils.java @@ -0,0 +1,103 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.util.Collections; + +import org.springframework.integration.dsl.IntegrationFlowBuilder; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.file.transformer.FileToByteArrayTransformer; +import org.springframework.integration.transformer.StreamTransformer; +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.MimeTypeUtils; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Christian Tzolov + * + */ +public class FileUtils { + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new FileToByteArrayTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_JSON_VALUE)); + break; + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode."); + } + return flowBuilder; + } + + /** + * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on + * {@link FileConsumerProperties}; used for streaming sources. + * @param flowBuilder the flow builder. + * @param fileConsumerProperties the properties. + * @return the updated flow builder. + */ + public static IntegrationFlowBuilder enhanceStreamFlowForReadingMode(IntegrationFlowBuilder flowBuilder, + FileConsumerProperties fileConsumerProperties) { + switch (fileConsumerProperties.getMode()) { + case contents: + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) + .transform(new StreamTransformer()); + break; + case lines: + Boolean withMarkers = fileConsumerProperties.getWithMarkers(); + if (withMarkers == null) { + withMarkers = false; + } + flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, + MimeTypeUtils.TEXT_PLAIN_VALUE)) + .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); + break; + case ref: + default: + throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + + " is not a supported file reading mode when streaming."); + } + return flowBuilder; + } + +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java new file mode 100644 index 00000000..56c3f4e8 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.nio.file.Path; +import java.util.function.Supplier; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import reactor.core.publisher.Flux; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@SpringBootTest +@DirtiesContext +public class AbstractFileSupplierTests { + + @TempDir + static Path tempDir; + + @Autowired + Supplier>> fileSupplier; + + @BeforeAll + public static void beforeAll() { + System.setProperty("file.supplier.directory", tempDir.toAbsolutePath().toString()); + } + + @AfterAll + public static void afterAll() { + System.clearProperty("file.supplier.directory"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java new file mode 100644 index 00000000..b933e9a3 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.FileHeaders; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +public class DefaultFileSupplierTests extends AbstractFileSupplierTests { + + @Test + public void testBasicFlow() throws IOException { + + Path firstFile = tempDir.resolve("first.file"); + Files.write(firstFile, "first.file".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + //create file after subscription + Path tempFile = tempDir.resolve("test.file"); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo("first.file".getBytes()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); + } + ) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo("testing".getBytes()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "test.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "test.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, tempFile.toFile()); + }) + .thenCancel() + .verifyLater(); + Files.write(tempFile, "testing".getBytes()); + stepVerifier.verify(); + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java new file mode 100644 index 00000000..c420f74b --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.FileHeaders; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.mode=ref") +public class FileModeRefTests extends AbstractFileSupplierTests { + + @Test + public void testBasicFlow() throws IOException { + + Path firstFile = tempDir.resolve("first.file"); + Files.write(firstFile, "first.file".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(firstFile.toAbsolutePath().toFile()); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.FILENAME, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); + assertThat(message.getHeaders()) + .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); + } + ) + .thenCancel() + .verify(); + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java new file mode 100644 index 00000000..e1c81697 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenamePattern = *.txt"}) +public class FilePayloadWithPatternTests extends AbstractFileSupplierTests { + + @Test + public void testPattern() throws IOException { + + Path txtFile1 = tempDir.resolve("test1.txt"); + Files.write(txtFile1, "one".getBytes()); + Path nonTxtExtension = tempDir.resolve("hello.bin"); + Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); + Path txtFile2 = tempDir.resolve("test2.txt"); + Files.write(txtFile2, "two".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile1.toAbsolutePath().toFile()); + } + ) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile2.toAbsolutePath().toFile()); + }) + .expectNoEvent(Duration.ofSeconds(1)) + .thenCancel() + .verify(); + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java new file mode 100644 index 00000000..0bf999c1 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenameRegex = t.*.txt"}) +public class FilePayloadWithRegexTests extends AbstractFileSupplierTests { + + @Test + public void testRegexPattern() throws IOException { + + Path txtFile1 = tempDir.resolve("test1.txt"); + Files.write(txtFile1, "one".getBytes()); + Path nonTxtExtension = tempDir.resolve("hello.bin"); + Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); + Path txtFile2 = tempDir.resolve("abc.txt"); + Files.write(txtFile2, "two".getBytes()); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> { + assertThat(message.getPayload()) + .isEqualTo(txtFile1.toAbsolutePath().toFile()); + } + ) + .expectNoEvent(Duration.ofSeconds(1)) + .expectNoEvent(Duration.ofSeconds(1)) + .thenCancel() + .verifyLater(); + + stepVerifier.verify(); + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java new file mode 100644 index 00000000..f03bea85 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.junit.jupiter.api.Test; +import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.json.JsonPathUtils; +import org.springframework.integration.support.json.JsonObjectMapperProvider; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = {"file.consumer.mode=lines", "file.consumer.withMarkers = true"}) +public class LinesAndMarkersAsJsonPayloadTests extends AbstractFileSupplierTests { + + @Test + public void testLinesWithMarkers() throws Exception { + Path firstFile = tempDir.resolve("test.file"); + Files.write(firstFile, "first line\n".getBytes()); + Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> { + try { + final Object evaluate = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); + assertThat(evaluate).isEqualTo(FileSplitter.FileMarker.Mark.START.name()); + } catch (IOException e) { + // passt through + } + } + ) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) + .assertNext((message) -> { + try { + final Object fileMarker = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); + assertThat(fileMarker).isEqualTo(FileSplitter.FileMarker.Mark.END.name()); + FileSplitter.FileMarker fileMarker1 = JsonObjectMapperProvider.newInstance() + .fromJson(fileMarker, FileSplitter.FileMarker.class); + assertThat(FileSplitter.FileMarker.Mark.END).isEqualTo(fileMarker1.getMark()); + assertThat(firstFile.toAbsolutePath()).isEqualTo(fileMarker1.getFilePath()); + assertThat(fileMarker1.getLineCount()).isEqualTo(2); + } catch (IOException e) { + // passt through + } + } + ) + .thenCancel() + .verify(); + } +} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java new file mode 100644 index 00000000..a70a0841 --- /dev/null +++ b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.junit.jupiter.api.Test; +import org.springframework.messaging.Message; +import org.springframework.test.context.TestPropertySource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Gary Russell + * @author Artem Bilan + * @author Soby Chacko + */ +@TestPropertySource(properties = "file.consumer.mode=lines") +public class LinesPayloadTests extends AbstractFileSupplierTests { + + @Test + public void testLines() throws IOException { + Path firstFile = tempDir.resolve("test.file"); + Files.write(firstFile, "first line\n".getBytes()); + Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); + + final Flux> messageFlux = fileSupplier.get(); + + StepVerifier.create(messageFlux) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) + .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) + .thenCancel() + .verify(); + + } +} diff --git a/functions/supplier/http-supplier/.gitignore b/functions/supplier/http-supplier/.gitignore new file mode 100644 index 00000000..4a453031 --- /dev/null +++ b/functions/supplier/http-supplier/.gitignore @@ -0,0 +1,28 @@ +/target/ +!.mvn/wrapper/maven-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +/build/ + +### VS Code ### +.vscode/ diff --git a/functions/supplier/http-supplier/pom.xml b/functions/supplier/http-supplier/pom.xml new file mode 100644 index 00000000..0e4ab57b --- /dev/null +++ b/functions/supplier/http-supplier/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + http-supplier + 1.0.0.BUILD-SNAPSHOT + http-supplier + HTTP Supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter-webflux + + + + org.springframework.integration + spring-integration-webflux + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + javax.validation + validation-api + + + org.hibernate.validator + hibernate-validator + 6.1.0.Final + + + + diff --git a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java new file mode 100644 index 00000000..4210d5bb --- /dev/null +++ b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSourceProperties.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.http; + +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.integration.http.support.DefaultHttpHeaderMapper; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.cors.CorsConfiguration; + +/** + * Configuration properties for the HTTP Supplier. + * + * @author Artem Bilan + */ +@ConfigurationProperties("http") +@Validated +public class HttpSourceProperties { + + /** + * HTTP endpoint path mapping. + */ + private String pathPattern = "/"; + + /** + * Headers that will be mapped. + */ + private String[] mappedRequestHeaders = { DefaultHttpHeaderMapper.HTTP_REQUEST_HEADER_NAME_PATTERN }; + + /** + * CORS properties. + */ + private Cors cors = new Cors(); + + @NotEmpty + public String getPathPattern() { + return this.pathPattern; + } + + public void setPathPattern(String pathPattern) { + this.pathPattern = pathPattern; + } + + public String[] getMappedRequestHeaders() { + return this.mappedRequestHeaders; + } + + public void setMappedRequestHeaders(String[] mappedRequestHeaders) { + this.mappedRequestHeaders = mappedRequestHeaders; + } + + public Cors getCors() { + return this.cors; + } + + public void setCors(Cors cors) { + this.cors = cors; + } + + public static class Cors { + + /** + * List of allowed origins, e.g. "http://domain1.com". + */ + private String[] allowedOrigins = { CorsConfiguration.ALL }; + + /** + * List of request headers that can be used during the actual request. + */ + private String[] allowedHeaders = { CorsConfiguration.ALL }; + + /** + * Whether the browser should include any cookies associated with the domain of the request being annotated. + */ + private Boolean allowCredentials; + + @NotEmpty + public String[] getAllowedOrigins() { + return this.allowedOrigins; + } + + public void setAllowedOrigins(String[] allowedOrigins) { + this.allowedOrigins = allowedOrigins; + } + + @NotEmpty + public String[] getAllowedHeaders() { + return this.allowedHeaders; + } + + public void setAllowedHeaders(String[] allowedHeaders) { + this.allowedHeaders = allowedHeaders; + } + + public Boolean getAllowCredentials() { + return allowCredentials; + } + + public void setAllowCredentials(Boolean allowCredentials) { + this.allowCredentials = allowCredentials; + } + + } + +} diff --git a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java new file mode 100644 index 00000000..a59bd7d6 --- /dev/null +++ b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2011-2018 Pivotal Software Inc, All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.http; + +import java.util.function.Supplier; + +import org.reactivestreams.Publisher; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.integration.expression.ValueExpression; +import org.springframework.integration.http.support.DefaultHttpHeaderMapper; +import org.springframework.integration.mapping.HeaderMapper; +import org.springframework.integration.webflux.dsl.WebFlux; +import org.springframework.integration.webflux.inbound.WebFluxInboundEndpoint; +import org.springframework.messaging.Message; + +import reactor.core.publisher.Flux; + +/** + * Configuration for the HTTP Supplier. + * + * @author Artem Bilan + */ +@EnableConfigurationProperties(HttpSourceProperties.class) +@Configuration +public class HttpSupplierConfiguration { + + @Bean + public Publisher> httpSupplierFlow(HttpSourceProperties httpSourceProperties) { + return IntegrationFlows.from( + WebFlux.inboundChannelAdapter(httpSourceProperties.getPathPattern()) + .requestPayloadType(byte[].class) + .statusCodeExpression(new ValueExpression<>(HttpStatus.ACCEPTED)) + .mappedRequestHeaders(httpSourceProperties.getMappedRequestHeaders()) + .crossOrigin(crossOrigin -> + crossOrigin.origin(httpSourceProperties.getCors().getAllowedOrigins()) + .allowedHeaders(httpSourceProperties.getCors().getAllowedHeaders()) + .allowCredentials(httpSourceProperties.getCors().getAllowCredentials())) + .autoStartup(false)) + .toReactivePublisher(); + } + + @Bean + public HeaderMapper httpHeaderMapper() { + return DefaultHttpHeaderMapper.inboundMapper(); + } + + @Bean + public Supplier>> httpSupplier( + Publisher> httpRequestPublisher, + WebFluxInboundEndpoint webFluxInboundEndpoint) { + + return () -> Flux.from(httpRequestPublisher) + .doOnSubscribe((subscription) -> webFluxInboundEndpoint.start()) + .doOnTerminate(webFluxInboundEndpoint::stop); + } + +} diff --git a/functions/supplier/http-supplier/src/main/resources/application.properties b/functions/supplier/http-supplier/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/functions/supplier/http-supplier/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java b/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java new file mode 100644 index 00000000..c7d229c7 --- /dev/null +++ b/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java @@ -0,0 +1,123 @@ +package org.springframework.cloud.fn.supplier.http; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.http.MediaType; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.integration.http.HttpHeaders; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.web.reactive.function.client.WebClient; + +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import reactor.core.publisher.Flux; +import reactor.netty.http.client.HttpClient; +import reactor.test.StepVerifier; + +/** + * The test for HTTP Supplier. + * + * @author Artem Bilan + */ +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "server.ssl.key-store=classpath:test.jks", + "server.ssl.key-password=password", + "server.ssl.trust-store=classpath:test.jks", + "server.ssl.client-auth=want" + }) +public class HttpSupplierApplicationTests { + + @Autowired + private Supplier>> httpSupplier; + + @LocalServerPort + private int port; + + @Test + public void testHttpSupplier() { + Flux> messageFlux = this.httpSupplier.get(); + + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .isEqualTo("test1".getBytes())) + .satisfies((msg) -> assertThat(msg.getHeaders()) + .containsEntry(MessageHeaders.CONTENT_TYPE, + new MediaType("text", "plain", StandardCharsets.UTF_8)) + .extractingByKey(HttpHeaders.REQUEST_URL).asString() + .startsWith("https://")) + ) + .assertNext((message) -> + assertThat(message) + .extracting(Message::getPayload) + .isEqualTo("{\"name\":\"test2\"}".getBytes())) + .assertNext((message) -> + assertThat(message) + .extracting(Message::getPayload) + .isEqualTo("{\"name\":\"test3\"}".getBytes())) + .thenCancel() + .verifyLater(); + + HttpClient httpClient = + HttpClient.create() + .secure(sslSpec -> + sslSpec.sslContext(SslContextBuilder.forClient() + .sslProvider(SslProvider.JDK) + .trustManager(InsecureTrustManagerFactory.INSTANCE))); + + WebClient webClient = + WebClient.builder() + .clientConnector(new ReactorClientHttpConnector(httpClient)) + .baseUrl("https://localhost:" + port) + .build(); + + WebClient.RequestBodySpec requestBodySpec = webClient.post().uri("/"); + requestBodySpec.bodyValue("test1").exchange().block(Duration.ofSeconds(10)); + requestBodySpec.bodyValue(new TestPojo("test2")).exchange().block(Duration.ofSeconds(10)); + requestBodySpec.bodyValue(new TestPojo("test3")).exchange().block(Duration.ofSeconds(10)); + + stepVerifier.verify(); + } + + private static class TestPojo { + + private String name; + + public TestPojo() { + } + + public TestPojo(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + } + + @SpringBootApplication + static class TestApplication { } + +} diff --git a/functions/supplier/http-supplier/src/test/resources/test.jks b/functions/supplier/http-supplier/src/test/resources/test.jks new file mode 100644 index 00000000..0fc3e802 Binary files /dev/null and b/functions/supplier/http-supplier/src/test/resources/test.jks differ diff --git a/functions/supplier/jdbc-supplier/pom.xml b/functions/supplier/jdbc-supplier/pom.xml new file mode 100644 index 00000000..ac7d52be --- /dev/null +++ b/functions/supplier/jdbc-supplier/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + jdbc-supplier + 1.0.0.BUILD-SNAPSHOT + jdbc-supplier + JDBC supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-jdbc + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.cloud + spring-cloud-function-context + ${spring-cloud-function.version} + + + io.pivotal.java.function + splitter-function + ${project.version} + + + org.springframework.boot + spring-boot-configuration-processor + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + com.h2database + h2 + test + + + io.projectreactor + reactor-test + test + + + + diff --git a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java new file mode 100644 index 00000000..ec6a59ee --- /dev/null +++ b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import io.pivotal.java.function.splitter.function.SplitterFunctionConfiguration; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.sql.DataSource; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.function.context.PollableBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.integration.core.MessageSource; +import org.springframework.integration.jdbc.JdbcPollingChannelAdapter; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@Configuration +@EnableConfigurationProperties(JdbcSupplierProperties.class) +@Import(SplitterFunctionConfiguration.class) +public class JdbcSupplierConfiguration { + + private final JdbcSupplierProperties properties; + + private final DataSource dataSource; + + public JdbcSupplierConfiguration(JdbcSupplierProperties properties, DataSource dataSource) { + this.properties = properties; + this.dataSource = dataSource; + } + + @Bean + public MessageSource jdbcMessageSource() { + JdbcPollingChannelAdapter jdbcPollingChannelAdapter = + new JdbcPollingChannelAdapter(this.dataSource, this.properties.getQuery()); + jdbcPollingChannelAdapter.setMaxRows(this.properties.getMaxRows()); + jdbcPollingChannelAdapter.setUpdateSql(this.properties.getUpdate()); + return jdbcPollingChannelAdapter; + } + + @Bean(name = "jdbcSupplier") + @PollableBean(splittable = true) + @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", matchIfMissing = true) + public Supplier>> splittedSupplier(Function, List>> splitterFunction) { + return () -> { + Message received = jdbcMessageSource().receive(); + if (received != null) { + return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> + } + else { + return Flux.empty(); + } + }; + } + + @Bean + @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", havingValue = "false") + public Supplier> jdbcSupplier() { + return () -> jdbcMessageSource().receive(); + } + +} diff --git a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java new file mode 100644 index 00000000..6b6d14fc --- /dev/null +++ b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@ConfigurationProperties("jdbc.supplier") +@Validated +public class JdbcSupplierProperties { + + /** + * The query to use to select data. + */ + private String query; + + /** + * An SQL update statement to execute for marking polled messages as 'seen'. + */ + private String update; + + /** + * Whether to split the SQL result as individual messages. + */ + private boolean split = true; + + /** + * Max numbers of rows to process for query. + */ + private int maxRows = 0; + + @NotNull + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public String getUpdate() { + return update; + } + + public void setUpdate(String update) { + this.update = update; + } + + public boolean isSplit() { + return split; + } + + public void setSplit(boolean split) { + this.split = split; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(int maxRows) { + this.maxRows = maxRows; + } + +} diff --git a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java new file mode 100644 index 00000000..49e3a24a --- /dev/null +++ b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.jdbc; + +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = "jdbc.supplier.query=select id, name from test order by id") +@DirtiesContext +public class DefaultJdbcSupplierTests { + + @Autowired + Supplier>> jdbcSupplier; + + @Test + void testExtraction() { + final Flux> messageFlux = jdbcSupplier.get(); + StepVerifier stepVerifier = + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(1L) && map.get("NAME").equals("Bob"); + }) + )) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(2L) && map.get("NAME").equals("Jane"); + }) + )) + .assertNext((message) -> + assertThat(message) + .satisfies((msg) -> assertThat(msg) + .extracting(Message::getPayload) + .matches(o -> { + Map map = (Map)o; + return map.get("ID").equals(3L) && map.get("NAME").equals("John"); + }) + )) + .thenCancel() + .verifyLater(); + stepVerifier.verify(); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java new file mode 100644 index 00000000..60e000b0 --- /dev/null +++ b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.fn.supplier.jdbc; + +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import org.springframework.test.annotation.DirtiesContext; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = {"jdbc.supplier.query=select id, name from test order by id", "jdbc.supplier.split=false"}) +@DirtiesContext +public class NonSplitJdbcSupplierTests { + + @Autowired + Supplier> jdbcSupplier; + + @Test + void testExtraction() { + final Message message = jdbcSupplier.get(); + final List> payload = (List>) message.getPayload(); + assertThat(payload.size()).isEqualTo(3); + Map map = payload.get(0); + assertThat(map.get("ID")).isEqualTo(1L); + assertThat(map.get("NAME")).isEqualTo("Bob"); + map = payload.get(1); + assertThat(map.get("ID")).isEqualTo(2L); + assertThat(map.get("NAME")).isEqualTo("Jane"); + map = payload.get(2); + assertThat(map.get("ID")).isEqualTo(3L); + assertThat(map.get("NAME")).isEqualTo("John"); + } + + @SpringBootApplication + static class TestApplication { + } +} diff --git a/functions/supplier/jdbc-supplier/src/test/resources/schema.sql b/functions/supplier/jdbc-supplier/src/test/resources/schema.sql new file mode 100644 index 00000000..bb5835cb --- /dev/null +++ b/functions/supplier/jdbc-supplier/src/test/resources/schema.sql @@ -0,0 +1,10 @@ +-- Run by default by Boot infrastructure + +create table test( + id bigint, + name varchar (2000), + tag char(1) +); +insert into test values (1, 'Bob', NULL); +insert into test values (2, 'Jane', NULL); +insert into test values (3, 'John', NULL); \ No newline at end of file diff --git a/functions/supplier/mongodb-supplier/.gitignore b/functions/supplier/mongodb-supplier/.gitignore new file mode 100644 index 00000000..a2a3040a --- /dev/null +++ b/functions/supplier/mongodb-supplier/.gitignore @@ -0,0 +1,31 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/** +!**/src/test/** + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ + +### VS Code ### +.vscode/ diff --git a/functions/supplier/mongodb-supplier/pom.xml b/functions/supplier/mongodb-supplier/pom.xml new file mode 100644 index 00000000..15845cc1 --- /dev/null +++ b/functions/supplier/mongodb-supplier/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + mongodb-supplier + 1.0.0.BUILD-SNAPSHOT + mongodb-supplier + Mongo DB supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.integration + spring-integration-mongodb + + + org.mongodb + mongodb-driver-sync + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.cloud + spring-cloud-function-context + ${spring-cloud-function.version} + + + org.springframework.cloud.fn + splitter-function + ${project.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + io.projectreactor + reactor-test + test + + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + diff --git a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java new file mode 100644 index 00000000..a82a8c68 --- /dev/null +++ b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration; +import org.springframework.context.annotation.Configuration; +import reactor.core.publisher.Flux; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.function.context.PollableBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.expression.Expression; +import org.springframework.expression.common.LiteralExpression; +import org.springframework.integration.mongodb.inbound.MongoDbMessageSource; +import org.springframework.messaging.Message; + +/** + * A configuration for MongoDB Source applications. Produces + * {@link MongoDbMessageSource} which polls collection with the query after startup + * according to the polling properties. + * + * @author Adam Zwickey + * @author Artem Bilan + * @author David Turanski + * + */ +@Configuration +@EnableConfigurationProperties({ MongodbSupplierProperties.class }) +@Import(SplitterFunctionConfiguration.class) +public class MongodbSupplierConfiguration { + + private final MongodbSupplierProperties properties; + + private final MongoTemplate mongoTemplate; + + public MongodbSupplierConfiguration(MongodbSupplierProperties properties, MongoTemplate mongoTemplate) { + this.properties = properties; + this.mongoTemplate = mongoTemplate; + } + + @Bean(name = "mongodbSupplier") + @PollableBean(splittable = true) + @ConditionalOnProperty(prefix = "mongodb", name = "split", matchIfMissing = true) + public Supplier>> splittedSupplier(Function, List>> splitterFunction) { + return () -> { + Message received = mongoSource().receive(); + if (received != null) { + return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> + } + else { + return Flux.empty(); + } + }; + } + + @Bean + @ConditionalOnProperty(prefix = "mongodb", name = "split", havingValue = "false") + public Supplier> mongodbSupplier() { + return () -> mongoSource().receive(); + } + + /** + * The inheritors can consider to override this method for their purpose or just adjust + * options for the returned instance + * @return a {@link MongoDbMessageSource} instance + */ + @Bean + public MongoDbMessageSource mongoSource() { + Expression queryExpression = (this.properties.getQueryExpression() != null + ? this.properties.getQueryExpression() + : new LiteralExpression(this.properties.getQuery())); + MongoDbMessageSource mongoDbMessageSource = new MongoDbMessageSource(this.mongoTemplate, queryExpression); + mongoDbMessageSource.setCollectionNameExpression(new LiteralExpression(this.properties.getCollection())); + mongoDbMessageSource.setEntityClass(String.class); + return mongoDbMessageSource; + } + +} diff --git a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java new file mode 100644 index 00000000..31ba05e6 --- /dev/null +++ b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java @@ -0,0 +1,91 @@ +/* + * Copyright 2016-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.expression.Expression; +import org.springframework.validation.annotation.Validated; + +/** + * @author Adam Zwickey + * @author Artem Bilan + * @author Chris Schaefer + * @author David Turanski + * + */ +@ConfigurationProperties("mongodb.supplier") +@Validated +public class MongodbSupplierProperties { + + /** + * The MongoDB collection to query + */ + private String collection; + + /** + * The MongoDB query + */ + private String query = "{ }"; + + /** + * The SpEL expression in MongoDB query DSL style + */ + private Expression queryExpression; + + /** + * Whether to split the query result as individual messages. + */ + private boolean split = true; + + @NotEmpty(message = "Query is required") + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Expression getQueryExpression() { + return queryExpression; + } + + public void setQueryExpression(Expression queryExpression) { + this.queryExpression = queryExpression; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + @NotBlank(message = "Collection name is required") + public String getCollection() { + return collection; + } + + public boolean isSplit() { + return split; + } + + public void setSplit(boolean split) { + this.split = split; + } + +} diff --git a/functions/supplier/mongodb-supplier/src/main/resources/application.properties b/functions/supplier/mongodb-supplier/src/main/resources/application.properties new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/functions/supplier/mongodb-supplier/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java b/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java new file mode 100644 index 00000000..0c332137 --- /dev/null +++ b/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2019-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.mongo; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.messaging.Message; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +@SpringBootTest(properties = { + "spring.data.mongodb.port=0", + "mongodb.supplier.collection=testing"}) +class MongodbSupplierApplicationTests { + + private ObjectMapper objectMapper = new ObjectMapper(); + + @Autowired + private Supplier>> mongodbSupplier; + + @Autowired + private MongoClient mongo; + + @BeforeEach + public void setUp() { + MongoDatabase database = this.mongo.getDatabase("test"); + database.createCollection("testing"); + MongoCollection collection = database.getCollection("testing"); + collection.insertOne( + new Document("greeting", "hello") + .append("name", "foo")); + collection.insertOne( + new Document("greeting", "hola") + .append("name", "bar")); + } + + @Test + void testMongodbSupplier() { + Flux> messageFlux = this.mongodbSupplier.get(); + StepVerifier.create(messageFlux) + .assertNext((message) -> + assertThat(payload(message)).contains( + entry("greeting","hello"), + entry("name", "foo"))) + .assertNext((message) -> + assertThat(payload(message)).contains( + entry("greeting","hola"), + entry("name", "bar"))) + .thenCancel() + .verify(); + } + + private Map payload(Message message) { + Map map = null; + try { + map = objectMapper.readValue(message.getPayload().toString(),HashMap.class); + } + catch (Exception e) { + e.printStackTrace(); + } + return map; + } + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/supplier/time-supplier/pom.xml b/functions/supplier/time-supplier/pom.xml new file mode 100644 index 00000000..6b871e62 --- /dev/null +++ b/functions/supplier/time-supplier/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + time-supplier + 1.0.0.BUILD-SNAPSHOT + time-supplier + time supplier + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0.BUILD-SNAPSHOT + ../../spring-functions-parent + + + + + org.springframework.boot + spring-boot-starter + + + org.apache.commons + commons-lang3 + + + + org.hibernate.validator + hibernate-validator + true + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + + diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java new file mode 100644 index 00000000..77ff2986 --- /dev/null +++ b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.text.SimpleDateFormat; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; + +/** + * The annotated String must be a valid {@link java.text.SimpleDateFormat} pattern. + * + * @author Eric Bottard + * @author Soby Chacko + */ +@Target({ ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE, ElementType.CONSTRUCTOR, + ElementType.PARAMETER }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Constraint(validatedBy = { DateFormat.DateFormatValidator.class }) +public @interface DateFormat { + + String DEFAULT_MESSAGE = ""; + + String message() default DEFAULT_MESSAGE; + + Class[] groups() default { }; + + Class[] payload() default { }; + + public static class DateFormatValidator implements ConstraintValidator { + + private String message; + + @Override + public void initialize(DateFormat constraintAnnotation) { + this.message = constraintAnnotation.message(); + } + + @Override + public boolean isValid(CharSequence value, ConstraintValidatorContext context) { + if (value == null) { + return true; + } + try { + new SimpleDateFormat(value.toString()); + } + catch (IllegalArgumentException e) { + if (DEFAULT_MESSAGE.equals(this.message)) { + context.disableDefaultConstraintViolation(); + context.buildConstraintViolationWithTemplate(e.getMessage()).addConstraintViolation(); + } + return false; + } + return true; + } + + } + +} diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java new file mode 100644 index 00000000..3c59377c --- /dev/null +++ b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * @author Soby Chacko + */ +@ConfigurationProperties("time") +@Validated +public class TimeProperties { + + /** + * Format for the date value. + */ + private String dateFormat = "MM/dd/yy HH:mm:ss"; + + @DateFormat + public String getDateFormat() { + return this.dateFormat; + } + + public void setDateFormat(String dateFormat) { + this.dateFormat = dateFormat; + } + +} diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java new file mode 100644 index 00000000..40224405 --- /dev/null +++ b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.util.Date; +import java.util.function.Supplier; + +import org.apache.commons.lang3.time.FastDateFormat; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(TimeProperties.class) +public class TimeSupplierConfiguration { + + @Bean + public Supplier timeSupplier(TimeProperties timeProperties) { + FastDateFormat fastDateFormat = FastDateFormat.getInstance(timeProperties.getDateFormat()); + return () -> fastDateFormat.format(new Date()); + } + +} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java new file mode 100644 index 00000000..4e426863 --- /dev/null +++ b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.junit.jupiter.api.Test; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +public class SimpleTimeSupplierTests extends TimeSupplierApplicationTests { + + @Test + public void testTimeSupplier() { + final String time = timeSupplier.get(); + SimpleDateFormat dateFormat = new SimpleDateFormat(new TimeProperties().getDateFormat()); + assertThatCode(() -> { + Date date = dateFormat.parse(time); + assertThat(date).isNotNull(); + }).doesNotThrowAnyException(); + } + +} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java new file mode 100644 index 00000000..df379308 --- /dev/null +++ b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import java.util.function.Supplier; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest +public abstract class TimeSupplierApplicationTests { + + @Autowired + Supplier timeSupplier; + + @Autowired + TimeProperties timeProperties; + + protected abstract void testTimeSupplier(); + + @SpringBootApplication + static class TestApplication {} +} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java new file mode 100644 index 00000000..bcbe99d8 --- /dev/null +++ b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.time; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author Soby Chacko + * @author Artem Bilan + */ +@SpringBootTest({ "time.dateFormat=MMddyyyy HH:mm:ss" }) +public class VariationToSimpleTests extends TimeSupplierApplicationTests { + + @Test + public void testTimeSupplier() { + final String time = timeSupplier.get(); + SimpleDateFormat dateFormat = new SimpleDateFormat(timeProperties.getDateFormat()); + assertThatCode(() -> { + Date date = dateFormat.parse(time); + assertThat(date).isNotNull(); + }).doesNotThrowAnyException(); + } + + @Test + public void testInvalidDateFormat() { + TimeProperties timeProperties = new TimeProperties(); + timeProperties.setDateFormat("AA/dd/yyyy HH:mm:ss"); + assertThatIllegalArgumentException().isThrownBy(() -> new SimpleDateFormat(timeProperties.getDateFormat())); + } + +} diff --git a/mvnw b/mvnw new file mode 100755 index 00000000..a08b219e --- /dev/null +++ b/mvnw @@ -0,0 +1,253 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +echo "Running version check" +VERSION=$( sed '\!//' -e 's!.*$!!' ) +echo "The found version is [${VERSION}]" + +if echo $VERSION | egrep -q 'M|RC'; then + echo Activating \"milestone\" profile for version=\"$VERSION\" + echo $MAVEN_ARGS | grep -q milestone || MAVEN_ARGS="$MAVEN_ARGS -Pmilestone" +else + echo Deactivating \"milestone\" profile for version=\"$VERSION\" + echo $MAVEN_ARGS | grep -q milestone && MAVEN_ARGS=$(echo $MAVEN_ARGS | sed -e 's/-Pmilestone//') +fi + +if echo $VERSION | egrep -q 'RELEASE'; then + echo Activating \"central\" profile for version=\"$VERSION\" + echo $MAVEN_ARGS | grep -q milestone || MAVEN_ARGS="$MAVEN_ARGS -Pcentral" +else + echo Deactivating \"central\" profile for version=\"$VERSION\" + echo $MAVEN_ARGS | grep -q central && MAVEN_ARGS=$(echo $MAVEN_ARGS | sed -e 's/-Pcentral//') +fi + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} ${MAVEN_ARGS} "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 00000000..b26ab24f --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..93cb7271 --- /dev/null +++ b/pom.xml @@ -0,0 +1,16 @@ + + + 4.0.0 + org.springframework.cloud.stream.app + stream-applications + 3.0.0.BUILD-SNAPSHOT + stream-applications + Functions and Infrastructure for stream applications + pom + + + functions + applications + + +