diff --git a/batch/file-ingest/pom.xml b/batch/file-ingest/pom.xml index 007e59e..aaacc2a 100644 --- a/batch/file-ingest/pom.xml +++ b/batch/file-ingest/pom.xml @@ -8,12 +8,12 @@ org.springframework.boot spring-boot-starter-parent - 2.1.0.BUILD-SNAPSHOT + 2.1.5.RELEASE 1.8 3.7.0 - 2.1.0.BUILD-SNAPSHOT + 2.1.1.RELEASE checkstyle.xml 2.17 springcloud diff --git a/dataflow-website/recipes/file-ingest/file-to-jdbc/pom.xml b/dataflow-website/recipes/file-ingest/file-to-jdbc/pom.xml index 019b12c..aaacc2a 100644 --- a/dataflow-website/recipes/file-ingest/file-to-jdbc/pom.xml +++ b/dataflow-website/recipes/file-ingest/file-to-jdbc/pom.xml @@ -8,7 +8,7 @@ org.springframework.boot spring-boot-starter-parent - 2.1.3.RELEASE + 2.1.5.RELEASE 1.8 diff --git a/src/main/asciidoc/batch/file-ingest-sftp/kubernetes.adoc b/src/main/asciidoc/batch/file-ingest-sftp/kubernetes.adoc index 7f486fe..d144ba2 100644 --- a/src/main/asciidoc/batch/file-ingest-sftp/kubernetes.adoc +++ b/src/main/asciidoc/batch/file-ingest-sftp/kubernetes.adoc @@ -1,6 +1,6 @@ [[sftp-file-ingest-kubernetes]] :docs_dir: ../.. -==== Using the Kubernetes Server +==== Running on Kubernetes ===== Additional Prerequisites * A Kubernetes cluster @@ -10,7 +10,7 @@ NOTE: For this example, we use an NFS host configured to allow https://www.tldp.org/HOWTO/NFS-HOWTO/server.html[read-write access]. -* The Spring Cloud Data Flow Kubernetes Server +* Spring Cloud Data Flow installed on Kubernetes + include::{docs_dir}/kubernetes-server.adoc[] + @@ -105,9 +105,9 @@ In the Spring Cloud Data Flow shell: + [source,console,options=nowrap] ---- -dataflow:>dataflow:>app register --name sftp --type source --uri docker:springcloud/sftp-dataflow-source-kafka --metadata-uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-kafka:jar:metadata:2.1.0.BUILD-SNAPSHOT +dataflow:>app register --name sftp --type source --uri docker:springcloud/sftp-dataflow-source-kafka --metadata-uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-kafka:jar:metadata:2.1.0.RELEASE Successfully registered application 'source:sftp' -dataflow:>app register --name task-launcher --type sink --uri docker:springcloud/task-launcher-dataflow-sink-kafka --metadata-uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-kafka:jar:metadata:1.0.0.BUILD-SNAPSHOT +dataflow:>app register --name task-launcher --type sink --uri docker:springcloud/task-launcher-dataflow-sink-kafka --metadata-uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-kafka:jar:metadata:1.0.1.RELEASE Successfully registered application 'sink:task-launcher' ---- + @@ -125,7 +125,7 @@ Once deployed, the stream will start polling the SFTP server and, when new files + [source,console,options=nowrap] ---- -dataflow:>stream create inboundSftp --definition "sftp --host= --username= --password= --allow-unknown-keys=true --remote-dir=/remote-files --local-dir=/staging/shared-files --task.launch.request.taskName=fileIngestTask --task.launch.request.deployment-properties='deployer.*.kubernetes.volumes=[{\"name\":\"staging\",\"persistentVolumeClaim\":{\"claimName\":\"nfs\"}}],deployer.*.kubernetes.volumeMounts=[{\"mountPath\":\"/staging\",\"name\":\"staging\"}]'| task-launcher --spring.cloud.dataflow.client.server-uri=http:// --spring.cloud.dataflow.client.authentication.basic.username=user --spring.cloud.dataflow.client.authentication.basic.password=password" +dataflow:>stream create inboundSftp --definition "sftp --host= --username= --password= --allow-unknown-keys=true --remote-dir=/remote-files --local-dir=/staging/shared-files --task.launch.request.taskName=fileIngestTask --task.launch.request.deployment-properties="deployer.*.kubernetes.volumes=[{'name':'staging','persistentVolumeClaim':{'claimName':'nfs'}}],deployer.*.kubernetes.volumeMounts=[{'mountPath':'/staging','name':'staging'}]"| task-launcher --spring.cloud.dataflow.client.server-uri=http://" ---- + NOTE: Replace ``, '`, and `` above. @@ -144,9 +144,8 @@ Now let's deploy the stream. + [source, console, options=nowrap] ---- -dataflow:>stream deploy inboundSftp --properties 'deployer.sftp.kubernetes.volumes=[{"name":"staging","persistentVolumeClaim":{"claimName":"nfs"}}],deployer.sftp.kubernetes.volumeMounts=[{"mountPath":"/staging","name":"staging"}],deployer.*.kubernetes.bootMajorVersion=2' +dataflow:>stream deploy inboundSftp --properties "deployer.sftp.kubernetes.volumes=[{'name':'staging','persistentVolumeClaim':{'claimName':'nfs'}}],deployer.sftp.kubernetes.volumeMounts=[{'mountPath':'/staging','name':'staging'}]" ---- -NOTE: The deployment property `deployer.*.kubernetes.bootMajorVersion=2` tells the deployer to configure the liveness and readiness probes for Spring Boot 2.x applications, as actuator endpoints have changed in Spring Boot 2.0. This will likely become the default in a future SCDF release. + . Verify Stream deployment + @@ -159,7 +158,7 @@ dataflow:>stream list ║Stream Name│ Stream Definition │ Status ║ ╠═══════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪════════════╣ ║inboundSftp│sftp │The stream ║ -║ │--task.launch.request.deployment-properties='deployer.*.kubernetes.volumes=[{"name":"staging","persistentVolumeClaim":{"claimName":"nfs"}}],deployer.*.kubernetes.volumeMounts=[{"mountPath":"/staging","name":"staging"}]'│has been ║ +║ │--task.launch.request.deployment-properties="deployer.*.kubernetes.volumes=[{'name':'staging','persistentVolumeClaim':{'claimName':'nfs'}}],deployer.*.kubernetes.volumeMounts=[{'mountPath':'/staging','name':'staging'}]"│has been ║ ║ │--password='******' --local-dir=/staging/shared-files --host= --remote-dir=/remote-files --task.launch.request.taskName=fileIngestTask --allow-unknown-keys=true --username= | task-launcher │successfully║ ║ │--spring.cloud.dataflow.client.server-uri=http:// --spring.cloud.dataflow.client.authentication.basic.username=user --spring.cloud.dataflow.client.authentication.basic.password='******' │deployed ║ ╚═══════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╧════════════╝ diff --git a/src/main/asciidoc/batch/file-ingest-sftp/local.adoc b/src/main/asciidoc/batch/file-ingest-sftp/local.adoc index d03e73b..25237ff 100644 --- a/src/main/asciidoc/batch/file-ingest-sftp/local.adoc +++ b/src/main/asciidoc/batch/file-ingest-sftp/local.adoc @@ -1,9 +1,10 @@ [[sftp-file-ingest-local]] -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A running local Data Flow Server +* Spring Cloud Data Flow installed locally + include::{docs_dir}/local-server.adoc[] * Running instance of link:http://kafka.apache.org/downloads.html[Kafka] @@ -49,9 +50,9 @@ In the Spring Cloud Data Flow shell: + [source,console,options=nowrap] ---- -dataflow:>app register --name sftp --type source --uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-kafka:2.0.3.BUILD-SNAPSHOT +dataflow:>app register --name sftp --type source --uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-kafka:2.1.0.RELEASE Successfully registered application 'source:sftp' -dataflow:>app register --name task-launcher --type sink --uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-kafka:1.0.0.BUILD-SNAPSHOT +dataflow:>app register --name task-launcher --type sink --uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-kafka:1.0.1.RELEASE Successfully registered application 'sink:task-launcher' ---- + diff --git a/src/main/asciidoc/batch/file-ingest-sftp/main.adoc b/src/main/asciidoc/batch/file-ingest-sftp/main.adoc index a10702c..5e3e78b 100644 --- a/src/main/asciidoc/batch/file-ingest-sftp/main.adoc +++ b/src/main/asciidoc/batch/file-ingest-sftp/main.adoc @@ -25,14 +25,14 @@ The Batch File Ingest - SFTP Demo processes a single file with 5000+ items. What The sftp source will process them immediately, generating 100 task launch requests. The Dataflow Server launches tasks asynchronously so this could potentially overwhelm the resources of the runtime platform. For example, when running the Data Flow server on your local machine, each launched task creates a new JVM. In Cloud Foundry, each task creates a new container instance. -Fortunately, Spring Cloud Data Flow 1.7 introduced new features to manage concurrently running tasks, including a new configuration parameter, `spring.cloud.dataflow.task.maximum-concurrent-tasks`, to http://docs.spring.io/spring-cloud-dataflow/docs/current/reference/htmlsingle/#spring-cloud-dataflow-task-limit-concurrent-executions[limit the number of concurrently running tasks]. +Fortunately, Spring Cloud Data Flow provides configuration settings to https://docs.spring.io/spring-cloud-dataflow/docs/current/reference/htmlsingle/#spring-cloud-dataflow-task-limit-concurrent-executions[limit the number of concurrently running tasks]. We can use this demo to see how this works. ===== Configuring the SCDF server Set the maximum concurrent tasks to 3. -For the local server, restart the server, adding a command line argument `--spring.cloud.dataflow.task.maximum-concurrent-tasks=3`. +For running tasks on a local server, restart the server, adding a command line argument `spring.cloud.dataflow.task.platform.local.accounts[default].maximum-concurrent-tasks=3`. -For the Cloud Foundry server, `cf set-env SPRING_CLOUD_DATAFLOW_TASK_MAXIMUM_CONCURRENT_TASKS 3`, and restage. +If running on Cloud Foundry, `cf set-env SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_LOCAL_ACCOUNTS[DEFAULT].MAXIMUM_CONCURRENT_TASKS 3`, and restage. ===== Running the demo Follow the main demo instructions but change the `Add Data` step, as described below. diff --git a/src/main/asciidoc/batch/file-ingest-sftp/pcf.adoc b/src/main/asciidoc/batch/file-ingest-sftp/pcf.adoc index d2e3242..c2a6d01 100644 --- a/src/main/asciidoc/batch/file-ingest-sftp/pcf.adoc +++ b/src/main/asciidoc/batch/file-ingest-sftp/pcf.adoc @@ -1,14 +1,14 @@ [sftp-file-ingest-pcf]] -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites NOTE: Running this demo in Cloud Foundry requires a shared file system that is accessed by apps running in different containers. -This feature is provided by https://docs.pivotal.io/pivotalcf/2-3/devguide/services/using-vol-services.html[NFS Volume Services]. +This feature is provided by https://docs.pivotal.io/pivotalcf/2-5/devguide/services/using-vol-services.html[NFS Volume Services]. To use Volume Services with SCDF, it is required that we provide `nfs` configuration via `cf create-service` rather than `cf bind-service`. Cloud Foundry introduced the `cf create-service` configuration option for Volume Services in version 2.3. -* A Cloud Foundry instance v2.3+ with NFS Volume Services https://docs.pivotal.io/pivotalcf/2-3/opsguide/enable-vol-services.html[enabled] +* A Cloud Foundry instance v2.3+ with NFS Volume Services https://docs.pivotal.io/pivotalcf/2-5/opsguide/enable-vol-services.html[enabled] * An SFTP server accessible from the Cloud Foundry instance * An `nfs` service instance properly configured @@ -23,16 +23,21 @@ $ cf create-service nfs Existing nfs -c '{"share":"/export","uid":" * A `rabbit` service instance * https://github.com/pivotal-cf/PivotalMySQLWeb[PivotalMySQLWeb] or another database tool to view the data -* The Spring Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry + include::{docs_dir}/cloudfoundry-server.adoc[] -===== Configuring the SCDF server +===== Configuring the SCDF and Skipper servers For convenience, we will configure the SCDF server to bind all stream and task apps to the `nfs` service. Using the Cloud Foundry CLI, set the following environment variables (or set them in the manifest): ``` -cf set-env SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_SERVICES rabbitmq,nfs -cf set-env SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_TASK_SERVICES mysql,nfs +cf set-env SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_CLOUDFOUNDRY_ACCOUNTS[default]_DEPLOYMENT_SERVICES: mysql,nfs +``` + +For the Skipper server: +``` +cf set-env SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_CLOUDFOUNDRY_ACCOUNTS[default]_DEPLOYMENT_SERVICES: rabbit,nfs ``` NOTE: Normally, for security and operational efficiency, we may want more fine grained control of which apps bind to the nfs service. @@ -40,7 +45,6 @@ One way to do this is to set deployment properties when creating and deploying t ===== Running the Demo - The source code for the <> batch job is located in `batch/file-ingest`. The resulting executable jar file must be available in a location that is accessible to your Cloud Foundry instance, such as an HTTP server or Maven repository. For convenience, the jar is published to the https://repo.spring.io/libs-snapshot-local/io/spring/cloud/dataflow/ingest/ingest/1.0.0.BUILD-SNAPSHOT/[Spring Maven repository] @@ -73,9 +77,9 @@ In the Spring Cloud Data Flow shell: + [source,console,options=nowrap] ---- -dataflow:>app register --name sftp --type source --uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-rabbit:2.0.3.BUILD-SNAPSHOT +dataflow:>app register --name sftp --type source --uri maven://org.springframework.cloud.stream.app:sftp-dataflow-source-rabbit:2.1.0.RELEASE Successfully registered application 'source:sftp' -dataflow:>app register --name task-launcher --type sink --uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-rabbit:1.0.0.BUILD-SNAPSHOT +dataflow:>app register --name task-launcher --type sink --uri maven://org.springframework.cloud.stream.app:task-launcher-dataflow-sink-rabbit:1.0.1.RELEASE Successfully registered application 'sink:task-launcher' ---- + @@ -102,6 +106,9 @@ dataflow:> app info --name task-launcher --type sink ╔══════════════════════════════╤══════════════════════════════╤══════════════════════════════╤══════════════════════════════╗ ║ Option Name │ Description │ Default │ Type ║ ╠══════════════════════════════╪══════════════════════════════╪══════════════════════════════╪══════════════════════════════╣ +║platform-name │The Spring Cloud Data Flow │default │java.lang.String ║ +║ │platform to use for launching │ │ ║ +║ │tasks. | ║ ║spring.cloud.dataflow.client.a│The login username. │ │java.lang.String ║ ║uthentication.basic.username │ │ │ ║ ║spring.cloud.dataflow.client.a│The login password. │ │java.lang.String ║ @@ -167,17 +174,19 @@ $ cf apps Getting apps in org cf_org / space cf_space as cf_user... OK -name requested state instances memory disk urls -dataflow-server started 1/1 2G 2G dataflow-server.app.io -dataflow-server-N5RYLDj-inboundSftp-sftp started 1/1 1G 1G dataflow-server-N5RYLDj-inboundSftp-sftp.dataflow-server.app.io -dataflow-server-N5RYLDj-inboundSftp-task-launcher-cloudfoundry started 1/1 1G 1G dataflow-server-N5RYLDj-inboundSftp-task-launcher-cloudfoundry.dataflow-server.app.io +name requested state instances memory disk urls +skipper-server started 1/1 1G 1G skipper-server.cfapps.io +data-flow-server started 1/1 2G 2G data-flow-server.cfapps.io +fileIngestTask stopped 0/1 1G 1G +bxZZ5Yv-inboundSftp-task-launcher-v1 started 1/1 2G 1G bxZZ5Yv-inboundSftp-task-launcher-v1.cfapps.io +bxZZ5Yv-inboundSftp-sftp-v1 started 1/1 2G 1G bxZZ5Yv-inboundSftp-sftp-v1.cfapps.io ---- + In this example, the logs for the `sftp` application can be viewed by: + [source, console, options=nowrap] ---- -cf logs dataflow-server-N5RYLDj-inboundSftp-sftp --recent +cf logs bxZZ5Yv-inboundSftp-sftp-v1 --recent ---- + The log files of this application would be useful to debug issues such as SFTP connection failures. @@ -185,7 +194,7 @@ The log files of this application would be useful to debug issues such as SFTP c Additionally, the logs for the `task-launcher` application can be viewed by: + ``` -cf logs dataflow-server-N5RYLDj-inboundSftp-task-launcher --recent +cf logs bxZZ5Yv-inboundSftp-task-launcher-v1 --recent ``` . Add data diff --git a/src/main/asciidoc/cloudfoundry-server.adoc b/src/main/asciidoc/cloudfoundry-server.adoc index 572817f..4e78206 100644 --- a/src/main/asciidoc/cloudfoundry-server.adoc +++ b/src/main/asciidoc/cloudfoundry-server.adoc @@ -1,66 +1,2 @@ -The Cloud Foundry Data Flow Server is Spring Boot application available for http://cloud.spring.io/spring-cloud-dataflow/#platform-implementations/[download] or you can https://github.com/spring-cloud/spring-cloud-dataflow-server-cloudfoundry[build] it yourself. -If you build it yourself, the executable jar will be in `spring-cloud-dataflow-server-cloudfoundry/target` - -NOTE: Although you can run the Data Flow Cloud Foundry Server locally and configure it to deploy to any Cloud Foundry instance, we will -deploy the server to Cloud Foundry as recommended. - -. Verify that CF instance is reachable (Your endpoint urls will be different from what is shown here). -+ - -``` -$ cf api -API endpoint: https://api.system.io (API version: ...) - -$ cf apps -Getting apps in org [your-org] / space [your-space] as user... -OK - -No apps found -``` -. Follow the instructions to deploy the https://docs.spring.io/spring-cloud-dataflow-server-cloudfoundry/docs/current/reference/htmlsingle[Spring Cloud Data Flow Cloud Foundry server]. Don't worry about creating a Redis service. We won't need it. If you are familiar with Cloud Foundry -application manifests, we recommend creating a manifest for the the Data Flow server as shown https://docs.spring.io/spring-cloud-dataflow-server-cloudfoundry/docs/current/reference/htmlsingle/#sample-manifest-template[here]. -+ -WARNING: As of this writing, there is a typo on the `SPRING_APPLICATION_JSON` entry in the sample manifest. `SPRING_APPLICATION_JSON` must be followed by `:` and the JSON string must be -wrapped in single quotes. Alternatively, you can replace that line with `MAVEN_REMOTE_REPOSITORIES_REPO1_URL: https://repo.spring.io/libs-snapshot`. If your Cloud Foundry installation is behind a firewall, you may need to install the stream apps used in this sample in your internal Maven repository and https://docs.spring.io/spring-cloud-dataflow/docs/1.3.0.M2/reference/htmlsingle/#getting-started-maven-configuration[configure] the server to access that repository. -. Once you have successfully executed `cf push`, verify the dataflow server is running -+ - -``` -$ cf apps -Getting apps in org [your-org] / space [your-space] as user... -OK - -name requested state instances memory disk urls -dataflow-server started 1/1 1G 1G dataflow-server.app.io -``` - -. Notice that the `dataflow-server` application is started and ready for interaction via the url endpoint - -. Connect the `shell` with `server` running on Cloud Foundry, e.g., `http://dataflow-server.app.io` -+ -``` -$ cd -$ java -jar spring-cloud-dataflow-shell-.jar - - ____ ____ _ __ - / ___| _ __ _ __(_)_ __ __ _ / ___| | ___ _ _ __| | - \___ \| '_ \| '__| | '_ \ / _` | | | | |/ _ \| | | |/ _` | - ___) | |_) | | | | | | | (_| | | |___| | (_) | |_| | (_| | - |____/| .__/|_| |_|_| |_|\__, | \____|_|\___/ \__,_|\__,_| - ____ |_| _ __|___/ __________ - | _ \ __ _| |_ __ _ | ___| | _____ __ \ \ \ \ \ \ - | | | |/ _` | __/ _` | | |_ | |/ _ \ \ /\ / / \ \ \ \ \ \ - | |_| | (_| | || (_| | | _| | | (_) \ V V / / / / / / / - |____/ \__,_|\__\__,_| |_| |_|\___/ \_/\_/ /_/_/_/_/_/ - - -Welcome to the Spring Cloud Data Flow shell. For assistance hit TAB or type "help". -server-unknown:> -``` -+ -``` -server-unknown:>dataflow config server http://dataflow-server.app.io -Successfully targeted http://dataflow-server.app.io -dataflow:> -``` +Follow the https://dataflow.spring.io/docs/installation/cloudfoundry/[installation] instructions to run Spring Cloud Data Flow on Cloud Foundry. diff --git a/src/main/asciidoc/kubernetes-server.adoc b/src/main/asciidoc/kubernetes-server.adoc index 16a110a..22fd986 100644 --- a/src/main/asciidoc/kubernetes-server.adoc +++ b/src/main/asciidoc/kubernetes-server.adoc @@ -1,107 +1 @@ -The Kubernetes Data Flow Server is a Spring Boot application. Operating The Data Flow Server in a Kubernetes cluster requires a cluster with sufficient resources to run the server, the distributed applications that the server will deploy to the cluster, and the requisite services. -Additional services include, at a minimum, a message broker (e.g., Kafka or Rabbit MQ) and a database. The Spring Cloud Data Flow Kubernetes reference incudes detailed https://docs.spring.io/spring-cloud-dataflow-server-kubernetes/docs/current/reference/htmlsingle/#kubernetes-getting-started[installation instructions]. -+ -For running these samples, we will follow these steps to set up a simple baseline configuration. -+ -. Install and set https://kubernetes.io/docs/tasks/tools/install-kubectl/[kubectl] -+ -. Verify that you are configured to access your Kubernetes cluster. -+ -``` -$kubectl config current-context -``` -+ -should display your cluster name. -For a newly installed cluster, the `kubectl get all` command should look something like this: -+ -``` -$kubectl get all -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/kubernetes ClusterIP 10.100.200.1 443/TCP 5m -``` -+ -. Install and configure Kubernetes Data Flow Server. -We will clone the git repo and configure Kafka message broker. -+ -``` -$git clone https://github.com/spring-cloud/spring-cloud-dataflow-server-kubernetes.git -$cd spring-cloud-dataflow-server-kubernetes -$git checkout v1.7.1.RELEASE -$kubectl create -f src/kubernetes/kafka/ -$kubectl create -f src/kubernetes/mysql/ -$kubectl create -f src/kubernetes/redis/ -$kubectl create -f src/kubernetes/server/server-roles.yaml -$kubectl create -f src/kubernetes/server/server-rolebinding.yaml -$kubectl create -f src/kubernetes/server/service-account.yaml -$kubectl create -f src/kubernetes/server/server-config-kafka.yaml -$kubectl create -f src/kubernetes/server/server-svc.yaml -$kubectl create -f src/kubernetes/server/server-deployment.yaml -``` -+ -. Verify the installation -+ -Now, if you run `kubectl get all`, you should see something like: -+ -[source, console, options=nowrap] ----- -$kubectl get all -NAME READY STATUS RESTARTS AGE -pod/kafka-broker-696786c8f7-4chnn 1/1 Running 0 7m -pod/kafka-zk-5f9bff7d5-4tbb7 1/1 Running 0 7m -pod/mysql-f878678df-ml5vd 1/1 Running 0 7m -pod/redis-748db48b4f-zz2ht 1/1 Running 0 1m -pod/scdf-server-64fb996ffb-dmwpj 1/1 Running 0 5m - -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/kafka ClusterIP 10.100.200.97 9092/TCP 7m -service/kafka-zk ClusterIP 10.100.200.244 2181/TCP,2888/TCP,3888/TCP 7m -service/kubernetes ClusterIP 10.100.200.1 443/TCP 19d -service/mysql ClusterIP 10.100.200.152 3306/TCP 7m -service/redis ClusterIP 10.100.200.8 6379/TCP 1m -service/scdf-server LoadBalancer 10.100.200.32 44.333.222.11 80:31486/TCP 5m - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -deployment.apps/kafka-broker 1 1 1 1 7m -deployment.apps/kafka-zk 1 1 1 1 7m -deployment.apps/mysql 1 1 1 1 7m -deployment.apps/redis 1 1 1 1 1m -deployment.apps/scdf-server 1 1 1 1 5m - -NAME DESIRED CURRENT READY AGE -replicaset.apps/kafka-broker-696786c8f7 1 1 1 7m -replicaset.apps/kafka-zk-5f9bff7d5 1 1 1 7m -replicaset.apps/mysql-f878678df 1 1 1 7m -replicaset.apps/redis-748db48b4f 1 1 1 1m -replicaset.apps/scdf-server-64fb996ffb 1 1 1 5m ----- -+ -. Connect the `shell` with `server` running on Kubernetes. -Note the `EXTERNAL-IP` entry for `service/scdf-server`. -Connect using the default username and password. -+ -``` -$ cd -$ java -jar spring-cloud-dataflow-shell-.jar - - ____ ____ _ __ - / ___| _ __ _ __(_)_ __ __ _ / ___| | ___ _ _ __| | - \___ \| '_ \| '__| | '_ \ / _` | | | | |/ _ \| | | |/ _` | - ___) | |_) | | | | | | | (_| | | |___| | (_) | |_| | (_| | - |____/| .__/|_| |_|_| |_|\__, | \____|_|\___/ \__,_|\__,_| - ____ |_| _ __|___/ __________ - | _ \ __ _| |_ __ _ | ___| | _____ __ \ \ \ \ \ \ - | | | |/ _` | __/ _` | | |_ | |/ _ \ \ /\ / / \ \ \ \ \ \ - | |_| | (_| | || (_| | | _| | | (_) \ V V / / / / / / / - |____/ \__,_|\__\__,_| |_| |_|\___/ \_/\_/ /_/_/_/_/_/ - - -Welcome to the Spring Cloud Data Flow shell. For assistance hit TAB or type "help". -server-unknown:> -``` -+ -``` -server-unknown:>dataflow config server --uri http:// --username user --password password -... -dataflow:> -``` - +Follow the https://dataflow.spring.io/docs/installation/kubernetes/[installation] instructions to run Spring Cloud Data Flow on Kubernetes. \ No newline at end of file diff --git a/src/main/asciidoc/local-server.adoc b/src/main/asciidoc/local-server.adoc index 759b20a..0f8feb7 100644 --- a/src/main/asciidoc/local-server.adoc +++ b/src/main/asciidoc/local-server.adoc @@ -1,9 +1,2 @@ -The Local Data Flow Server is Spring Boot application available for http://cloud.spring.io/spring-cloud-dataflow/#platform-implementations[download] or you can https://github.com/spring-cloud/spring-cloud-dataflow[build] it yourself. -If you build it yourself, the executable jar will be in `spring-cloud-dataflow-server-local/target` - -To run the Local Data Flow server Open a new terminal session: -``` -$cd -$java -jar spring-cloud-dataflow-server-local-.jar -``` +Follow the https://dataflow.spring.io/docs/installation/local/[installation] instructions to run Spring Cloud Data Flow on a local host. diff --git a/src/main/asciidoc/streaming/cassandra/http-to-cassandra/local.adoc b/src/main/asciidoc/streaming/cassandra/http-to-cassandra/local.adoc index 4717fd1..7aa88f8 100644 --- a/src/main/asciidoc/streaming/cassandra/http-to-cassandra/local.adoc +++ b/src/main/asciidoc/streaming/cassandra/http-to-cassandra/local.adoc @@ -1,9 +1,9 @@ [[http-cassandra-local]] -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A running local Data Flow Server +* Spring Cloud Data Flow installed locally include::{docs_dir}/local-server.adoc[] * Running instance of link:http://kafka.apache.org/downloads.html[Kafka] diff --git a/src/main/asciidoc/streaming/cassandra/http-to-cassandra/pcf.adoc b/src/main/asciidoc/streaming/cassandra/http-to-cassandra/pcf.adoc index af65ecb..bafe64d 100644 --- a/src/main/asciidoc/streaming/cassandra/http-to-cassandra/pcf.adoc +++ b/src/main/asciidoc/streaming/cassandra/http-to-cassandra/pcf.adoc @@ -1,5 +1,5 @@ [[http-cassandra-cf]] -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites @@ -18,7 +18,8 @@ CREATE TABLE book ( title text ); ``` -* The Spring Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry + include::{docs_dir}/cloudfoundry-server.adoc[] ===== Running the Demo diff --git a/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/local.adoc b/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/local.adoc index 6bf2779..ed56eaf 100644 --- a/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/local.adoc +++ b/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/local.adoc @@ -1,7 +1,7 @@ [[gemfire-cq-log-local]] -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A Running Data Flow Server +* Spring Cloud Data Flow installed locally include::{docs_dir}/local-server.adoc[] * A running instance of https://www.rabbitmq.com[Rabbit MQ] diff --git a/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/pcf.adoc b/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/pcf.adoc index c11961c..4ae513e 100644 --- a/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/pcf.adoc +++ b/src/main/asciidoc/streaming/gemfire/gemfire-cq-log/pcf.adoc @@ -1,5 +1,5 @@ [[gemfire-cq-log-cf]] -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites * A Cloud Foundry instance @@ -8,7 +8,7 @@ * Running instance of the https://docs.pivotal.io/p-cloud-cache/1-0/developer.html[Pivotal Cloud Cache for PCF] (PCC) service `cloudcache` in Cloud Foundry. -* The Spring Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry include::{docs_dir}/cloudfoundry-server.adoc[] diff --git a/src/main/asciidoc/streaming/gemfire/gemfire-log/local.adoc b/src/main/asciidoc/streaming/gemfire/gemfire-log/local.adoc index a0bc8e5..a53ad66 100644 --- a/src/main/asciidoc/streaming/gemfire/gemfire-log/local.adoc +++ b/src/main/asciidoc/streaming/gemfire/gemfire-log/local.adoc @@ -1,9 +1,10 @@ [[gemfire-log-local]] -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A Running Data Flow Server +* Spring Cloud Data Flow installed locally include::{docs_dir}/local-server.adoc[] + * A running instance of https://www.rabbitmq.com[Rabbit MQ] ===== Building and Running the Demo diff --git a/src/main/asciidoc/streaming/gemfire/gemfire-log/pcf.adoc b/src/main/asciidoc/streaming/gemfire/gemfire-log/pcf.adoc index f05ce11..2a9baf7 100644 --- a/src/main/asciidoc/streaming/gemfire/gemfire-log/pcf.adoc +++ b/src/main/asciidoc/streaming/gemfire/gemfire-log/pcf.adoc @@ -1,5 +1,5 @@ [[gemfire-log-cf]] -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites * A Cloud Foundry instance @@ -7,7 +7,7 @@ * Running instance of the https://docs.pivotal.io/p-cloud-cache/1-0/developer.html[Pivotal Cloud Cache for PCF] (PCC) service `cloudcache` in Cloud Foundry. -* The Spring Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry include::{docs_dir}/cloudfoundry-server.adoc[] diff --git a/src/main/asciidoc/streaming/gemfire/http-gemfire/local.adoc b/src/main/asciidoc/streaming/gemfire/http-gemfire/local.adoc index 47ba884..0bedd91 100644 --- a/src/main/asciidoc/streaming/gemfire/http-gemfire/local.adoc +++ b/src/main/asciidoc/streaming/gemfire/http-gemfire/local.adoc @@ -1,9 +1,10 @@ [[gemfire-http-local]] -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A running local Data Flow Server +* Spring Cloud Data Flow installed locally include::{docs_dir}/local-server.adoc[] + * A running instance of https://www.rabbitmq.com[Rabbit MQ] ===== Building and Running the Demo diff --git a/src/main/asciidoc/streaming/gemfire/http-gemfire/pcf.adoc b/src/main/asciidoc/streaming/gemfire/http-gemfire/pcf.adoc index e027845..e53a9e1 100644 --- a/src/main/asciidoc/streaming/gemfire/http-gemfire/pcf.adoc +++ b/src/main/asciidoc/streaming/gemfire/http-gemfire/pcf.adoc @@ -1,5 +1,5 @@ [[gemfire-http-cf]] -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites * A Cloud Foundry instance @@ -8,7 +8,7 @@ * Running instance of the https://docs.pivotal.io/p-cloud-cache/1-0/developer.html[Pivotal Cloud Cache for PCF] (PCC) service `cloudcache` in Cloud Foundry. -* Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry include::{docs_dir}/cloudfoundry-server.adoc[] ===== Building and Running the Demo diff --git a/src/main/asciidoc/streaming/jdbc/http-mysql/local.adoc b/src/main/asciidoc/streaming/jdbc/http-mysql/local.adoc index fd03a19..03b0c10 100644 --- a/src/main/asciidoc/streaming/jdbc/http-mysql/local.adoc +++ b/src/main/asciidoc/streaming/jdbc/http-mysql/local.adoc @@ -1,8 +1,8 @@ -==== Using the Local Server +==== Running Locally ===== Additional Prerequisites -* A running local Data Flow Server +* Spring Cloud Data Flow installed locally include::{docs_dir}/local-server.adoc[] * Running instance of link:http://kafka.apache.org/downloads.html[Kafka] diff --git a/src/main/asciidoc/streaming/jdbc/http-mysql/pcf.adoc b/src/main/asciidoc/streaming/jdbc/http-mysql/pcf.adoc index ce4cff8..b0534c2 100644 --- a/src/main/asciidoc/streaming/jdbc/http-mysql/pcf.adoc +++ b/src/main/asciidoc/streaming/jdbc/http-mysql/pcf.adoc @@ -1,5 +1,5 @@ -==== Using the Cloud Foundry Server +==== Running on Cloud Foundry ===== Additional Prerequisites @@ -17,7 +17,8 @@ CREATE TABLE names ); ``` + -* The Spring Cloud Data Flow Cloud Foundry Server +* Spring Cloud Data Flow installed on Cloud Foundry + include::{docs_dir}/cloudfoundry-server.adoc[] ===== Building and Running the Demo diff --git a/src/main/asciidoc/tasks/simple-batch-job/main.adoc b/src/main/asciidoc/tasks/simple-batch-job/main.adoc index f292eca..4f5ef9c 100644 --- a/src/main/asciidoc/tasks/simple-batch-job/main.adoc +++ b/src/main/asciidoc/tasks/simple-batch-job/main.adoc @@ -11,7 +11,7 @@ In this demonstration, you will learn how to orchestrate short-lived data proces * Running instance of mysql in PCFDev * A Running Data Flow Shell include::{docs_dir}/shell.adoc[] -* The Spring Cloud Data Flow Cloud Foundry Server running in PCFDev +* Spring Cloud Data Flow installed on Cloud Foundry include::{docs_dir}/cloudfoundry-server.adoc[] ==== Building and Running the Demo