diff --git a/.gitignore b/.gitignore index f2564bc..cc0bf82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,34 @@ +*~ +.#* +*# +*.sw* +_site/ +.factorypath +.gradletasknamecache .DS_Store +.checkstyle +/application.yml +/application.properties +asciidoctor.css +atlassian-ide-plugin.xml +bin/ +build/ +dump.rdb +out +spring-shell.log target/ +test-output + +# Eclipse artifacts, including WTP generated manifests +.classpath +.project +.settings/ +.springBeans +spring-*/src/main/java/META-INF/MANIFEST.MF + +# IDEA artifacts and output dirs +*.iml +*.ipr +*.iws +.idea/* +rebel.xml diff --git a/javadsl/.mvn/wrapper/maven-wrapper.jar b/javadsl/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..9cc84ea Binary files /dev/null and b/javadsl/.mvn/wrapper/maven-wrapper.jar differ diff --git a/javadsl/.mvn/wrapper/maven-wrapper.properties b/javadsl/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..9dda3b6 --- /dev/null +++ b/javadsl/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip diff --git a/javadsl/mvnw b/javadsl/mvnw new file mode 100755 index 0000000..5bf251c --- /dev/null +++ b/javadsl/mvnw @@ -0,0 +1,225 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/javadsl/mvnw.cmd b/javadsl/mvnw.cmd new file mode 100644 index 0000000..019bd74 --- /dev/null +++ b/javadsl/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/javadsl/pom.xml b/javadsl/pom.xml new file mode 100644 index 0000000..13f6df6 --- /dev/null +++ b/javadsl/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + com.example + scdfdsl + 0.0.1-SNAPSHOT + jar + + scdfdsl + Demo project for Spring Boot + + + + org.springframework.boot + spring-boot-starter-parent + 1.5.8.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.cloud + spring-cloud-dataflow-rest-client + 1.3.0.BUILD-SNAPSHOT + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-milestones + Spring Milestones + http://repo.spring.io/libs-milestone-local + + false + + + + + + diff --git a/javadsl/src/main/java/com/example/scdfdsl/ScdfdslApplication.java b/javadsl/src/main/java/com/example/scdfdsl/ScdfdslApplication.java new file mode 100644 index 0000000..1f4c028 --- /dev/null +++ b/javadsl/src/main/java/com/example/scdfdsl/ScdfdslApplication.java @@ -0,0 +1,142 @@ +package com.example.scdfdsl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.dataflow.rest.client.DataFlowOperations; +import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate; +import org.springframework.cloud.dataflow.rest.client.dsl.Stream; +import org.springframework.cloud.dataflow.rest.client.dsl.StreamApplication; +import org.springframework.context.annotation.Bean; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +@SpringBootApplication +public class ScdfdslApplication implements ApplicationRunner { + + public static void main(String[] args) { + SpringApplication.run(ScdfdslApplication.class, args); + } + + @Autowired + private StreamApplication source; + + @Autowired + private StreamApplication processor; + + @Autowired + private StreamApplication sink; + + // Using @Bean defintions makes it easier to reuse an application in multiple streams + @Bean + public StreamApplication source() { + return new StreamApplication("http").addProperty("server.port", 9900); + } + + @Bean + public StreamApplication processor() { + return new StreamApplication("splitter") + .addProperty("producer.partitionKeyExpression", "payload"); + } + + @Bean + public StreamApplication sink() { + return new StreamApplication("log") + .addDeploymentProperty("count", 2); + } + + @Override + public void run(ApplicationArguments applicationArguments) throws Exception { + DataFlowOperations dataFlowOperations = createDataFlowOperations(applicationArguments); + + if (applicationArguments.containsOption("style")) { + String style = applicationArguments.getOptionValues("style").get(0); + if (style.equalsIgnoreCase("definition")) { + // DEFINITION STYLE + definitionStyle(dataFlowOperations); + } else if (style.equalsIgnoreCase("fluent")) { + // FLUENT STYLE + fluentApplicationSytle(dataFlowOperations); + } else { + System.out.println("Style [" + style + "] not supported"); + } + } else { + definitionStyle(dataFlowOperations); + } + } + + + private void definitionStyle(DataFlowOperations dataFlowOperations) throws InterruptedException { + Map deploymentProperties = createDeploymentProperties(); + + System.out.println("Deploying stream."); + + Stream woodchuck = Stream.builder(dataFlowOperations) + .name("woodchuck") + .definition( + "http --server.port=9900 | splitter --expression=payload.split(' ') | log") + .create() + .deploy(deploymentProperties); + + waitAndDestroy(woodchuck); + } + + + + private void fluentApplicationSytle(DataFlowOperations dataFlowOperations) throws InterruptedException { + Map deploymentProperties = createDeploymentProperties(); + + System.out.println("Deploying stream."); + + Stream woodchuck = Stream.builder(dataFlowOperations).name("woodchuck") + .source(source) + .processor(processor) + .sink(sink) + .create() + .deploy(deploymentProperties); + + waitAndDestroy(woodchuck); + } + + private Map createDeploymentProperties() { + Map deploymentProperties = new HashMap<>(); + deploymentProperties.put("app.splitter.producer.partitionKeyExpression", "payload"); + deploymentProperties.put("deployer.log.count", "2"); + return deploymentProperties; + } + + + private DataFlowOperations createDataFlowOperations(ApplicationArguments applicationArguments) { + DataFlowOperations dataFlowOperations; + if (applicationArguments.containsOption("uri")) { + URI dataFlowUri = URI.create(applicationArguments.getOptionValues("uri").get(0)); + dataFlowOperations = new DataFlowTemplate(dataFlowUri); + } else { + dataFlowOperations = new DataFlowTemplate(URI.create("http://localhost:9393")); + } + + dataFlowOperations.appRegistryOperations().importFromResource( + "http://bit.ly/Celsius-RC1-stream-applications-rabbit-maven", true); + return dataFlowOperations; + } + + private void waitAndDestroy(Stream woodchuck) throws InterruptedException { + while(!woodchuck.getStatus().equals("deployed")){ + System.out.println("Wating for deployment of stream."); + Thread.sleep(5000); + } + + System.out.println("Letting the stream run for 2 minutes."); + // Let woodchuck run for 2 minutes + Thread.sleep(120000); + + System.out.println("Destroying stream"); + woodchuck.destroy(); + } + +} diff --git a/javadsl/src/main/resources/application.properties b/javadsl/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 diff --git a/javadsl/src/test/java/com/example/scdfdsl/SimpleUsageTests.java b/javadsl/src/test/java/com/example/scdfdsl/SimpleUsageTests.java new file mode 100644 index 0000000..e963a4e --- /dev/null +++ b/javadsl/src/test/java/com/example/scdfdsl/SimpleUsageTests.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.scdfdsl; + +import org.junit.Test; +import org.springframework.cloud.dataflow.rest.client.DataFlowOperations; +import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate; +import org.springframework.cloud.dataflow.rest.client.dsl.Stream; +import org.springframework.cloud.dataflow.rest.client.dsl.StreamBuilder; + +import java.net.URI; + +/** + * @author Mark Pollack + */ +public class SimpleUsageTests { + + @Test + public void doNothing() { + + } + + public void doStuff() { + URI dataFlowUri = URI.create("http://localhost:9393"); + DataFlowOperations dataFlowOperations = new DataFlowTemplate(dataFlowUri); + StreamBuilder streamBuilder = Stream.builder(dataFlowOperations).name("ticktock").definition("time | log").create(); + Stream stream = streamBuilder.deploy(); + } + + +} diff --git a/src/main/asciidoc/javadsl/main.adoc b/src/main/asciidoc/javadsl/main.adoc new file mode 100644 index 0000000..92355d0 --- /dev/null +++ b/src/main/asciidoc/javadsl/main.adoc @@ -0,0 +1,106 @@ +[[spring-cloud-data-flow-samples-javadsl]] +:docs_dir: ../.. +=== Deploying a stream programmaticaly +This sample shows the two usage styles of the Java DSL to create and deploy a stream. +You should look in the https://github.com/spring-cloud/spring-cloud-dataflow-samples/tree/master/batch/javadsl/src/main[source code] to get a feel for the different styles. + +1) Build the sample application + +[source,bash] +---- +./mvnw clean package +---- +With no command line options, the application will deploy the stream `http --server.port=9900 | splitter --expression=payload.split(' ') | log` using the URI `http://localhost:9393` to connect to the Data Flow server. +There is also a command line option `--style` whose value can be either `definition` or `fluent`. +This options picks which JavaDSL style will execute. +Both are identical in terms of behavior. +The `definition` style has code of the style +[source,java,options="nowrap"] +---- +Stream woodchuck = Stream.builder(dataFlowOperations) + .name("woodchuck") + .definition("http --server.port=9900 | splitter --expression=payload.split(' ') | log") + .create() + .deploy(deploymentProperties); +---- +while the `fluent` style has code of the style +[source,java] +---- +Stream woodchuck = Stream.builder(dataFlowOperations).name("woodchuck") + .source(source) + .processor(processor) + .sink(sink) + .create() + .deploy(deploymentProperties); +---- +where `source`, `processor`, and `sink` variables were defined as `@Bean`s of the type `StreamApplication` +[source,java] +---- +@Bean +public StreamApplication source() { + return new StreamApplication("http").addProperty("server.port", 9900); +} +---- +2) Run a local Data Flow Server and run the sample application. +This sample demonstrates the use of the local Data Flow Server, but you can pass in the option `--uri` to point to another Data Flow server instance that is running elsewhere. +[source,bash] +---- +$ java -jar target/scdfdsl-0.0.1-SNAPSHOT.jar +---- +You will then see the following output. +[source,bash] +---- +Deploying stream. +Wating for deployment of stream. +Wating for deployment of stream. +Wating for deployment of stream. +Wating for deployment of stream. +Wating for deployment of stream. +Letting the stream run for 2 minutes. +---- +To verify that the application has been deployed successfully, will tail the logs of one of the log sinks and post some data to the http source. +You can find the location for the logs of one of the log sink applications by looking in the Data Flow server's log file. + +3) Post some data to the server + +``` +curl http://localhost:9900 -H "Content-Type:text/plain" -X POST -d "how much wood would a woodchuck chuck if a woodchuck could chuck wood" +``` + +5) Verify the output +Tailing the log file of the first instance +[source,bash,options="nowrap"] +---- +cd /tmp/spring-cloud-dataflow-4323595028663837160/woodchuck-1511390696355/woodchuck.log +tail -f stdout_0.log +---- +[source,bash,options="nowrap"] +---- +2017-11-22 18:04:08.631 INFO 26652 --- [r.woodchuck-0-1] log-sink : how +2017-11-22 18:04:08.632 INFO 26652 --- [r.woodchuck-0-1] log-sink : chuck +2017-11-22 18:04:08.634 INFO 26652 --- [r.woodchuck-0-1] log-sink : chuck +---- + +Tailing the log file of the second instance +[source,bash,options="nowrap"] +---- +cd /tmp/spring-cloud-dataflow-4323595028663837160/woodchuck-1511390696355/woodchuck.log +tail -f stdout_1.log +---- + +You should see the output +[source,bash,options="nowrap"] +---- +$ tail -f stdout_1.log +2017-11-22 18:04:08.636 INFO 26655 --- [r.woodchuck-1-1] log-sink : much +2017-11-22 18:04:08.638 INFO 26655 --- [r.woodchuck-1-1] log-sink : wood +2017-11-22 18:04:08.639 INFO 26655 --- [r.woodchuck-1-1] log-sink : would +2017-11-22 18:04:08.640 INFO 26655 --- [r.woodchuck-1-1] log-sink : a +2017-11-22 18:04:08.641 INFO 26655 --- [r.woodchuck-1-1] log-sink : woodchuck +2017-11-22 18:04:08.642 INFO 26655 --- [r.woodchuck-1-1] log-sink : if +2017-11-22 18:04:08.644 INFO 26655 --- [r.woodchuck-1-1] log-sink : a +2017-11-22 18:04:08.645 INFO 26655 --- [r.woodchuck-1-1] log-sink : woodchuck +2017-11-22 18:04:08.646 INFO 26655 --- [r.woodchuck-1-1] log-sink : could +2017-11-22 18:04:08.647 INFO 26655 --- [r.woodchuck-1-1] log-sink : wood +---- +Note that the partitioning is done based on the hash of the `java.lang.String` object. diff --git a/src/main/asciidoc/overview.adoc b/src/main/asciidoc/overview.adoc index 7a712ad..9b5094b 100644 --- a/src/main/asciidoc/overview.adoc +++ b/src/main/asciidoc/overview.adoc @@ -2,6 +2,9 @@ == Overview This guide contains samples and demonstrations of how to build data pipelines with https://cloud.spring.io/spring-cloud-dataflow/[Spring Cloud Data Flow]. +== Java DSL +include::javadsl/main.adoc[] + == Streaming include::streaming/cassandra/http-to-cassandra/main.adoc[] include::streaming/jdbc/http-mysql/main.adoc[]