From f176e92fdf1072b95952459a0e2b0a122a66869e Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Tue, 3 May 2022 21:29:24 +0200 Subject: [PATCH 1/5] provide property for baseimage version. provide script for building images locally. --- applications/stream-applications-core/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/applications/stream-applications-core/pom.xml b/applications/stream-applications-core/pom.xml index f33b752b..3a77b1f0 100644 --- a/applications/stream-applications-core/pom.xml +++ b/applications/stream-applications-core/pom.xml @@ -19,13 +19,13 @@ 3.2.1-SNAPSHOT ${revision} 1.2.0 - springcloud/baseimage:1.0.0 + springcloud/baseimage:1.0.1 5.10 3.2.2 3.2.2 - 1.0.6 - 1.0.6 - 1.0.6 + 1.0.7-SNAPSHOT + 1.0.7-SNAPSHOT + 1.0.7-SNAPSHOT 2.1.2.RELEASE 1.3.0 From 617aeeba0f4346ffdeb2699cd4db02637934ab54 Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Tue, 3 May 2022 21:29:41 +0200 Subject: [PATCH 2/5] provide property for baseimage version. provide script for building images locally. --- build-images.sh | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 build-images.sh diff --git a/build-images.sh b/build-images.sh new file mode 100755 index 00000000..7718631a --- /dev/null +++ b/build-images.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +java --version +if [ $? -eq 0 ] +then + java_version=$(java --version) + echo "$java_version" + grep -e "1.8" "$java_version" + if [ $? -ne 0 ] + then + echo "Java 1.8 required not $java_version" + exit 2 + fi +fi +version="3.2.1-SNAPSHOT" +rootdir="$(pwd)" +processors="aggregator bridge filter groovy header-enricher http-request image-recognition object-detection script semantic-segmentation splitter transform twitter-trend" +echo "Build processors:$processor" +pushd applications/processor + for app in $processors + do + pushd "${app}-processor" + echo "$Building $(pwd)" + rm -rf apps + $rootdir/mvnw clean package -Pintegration + pushd apps + $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} + popd + popd + done +popd +sinks="analytics cassandra elasticsearch file ftp geode jdbc log mongodb mqtt pgcopy rabbit redis router rsocket s3 sftp tcp throughput twitter-message twitter-update wavefront websocket zeromq" +echo "Build sinks:$sinks" +pushd applications/sink + for app in $sinks + do + pushd "${app}-sink" + echo "$Building $(pwd)" + rm -rf apps + $rootdir/mvnw clean package -Pintegration + pushd apps + $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} + popd + popd + done +popd +sources="cdc-debezium file ftp geode http jdbc jms load-generator mail mongodb mqtt rabbit s3 sftp syslog tcp time twitter-message twitter-search twitter-stream websocket zeromq" +echo "Build sources:sources" +pushd applications/source + for app in $sources + do + pushd "${app}-source" + echo "$Building $(pwd)" + rm -rf apps + $rootdir/mvnw clean package -Pintegration + pushd apps + $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} + popd + popd + done +popd From 39a2afce8a736c39581c92f7f4f7dcd319b20d8d Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Wed, 4 May 2022 12:00:50 +0200 Subject: [PATCH 3/5] Convert snippets to Asciidoc format and added types. Fixes #241 --- .../src/main/asciidoc/overview.adoc | 80 +++++++++++-------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/stream-applications-release-train/stream-applications-docs/src/main/asciidoc/overview.adoc b/stream-applications-release-train/stream-applications-docs/src/main/asciidoc/overview.adoc index 2b0c8ec1..2e212ec7 100644 --- a/stream-applications-release-train/stream-applications-docs/src/main/asciidoc/overview.adoc +++ b/stream-applications-release-train/stream-applications-docs/src/main/asciidoc/overview.adoc @@ -97,21 +97,23 @@ Let's assume that you want to build JDBC Source application based on Kafka Binde Here is what you need to do. Assuming you built both functions and stream-applications-core as above. -``` +[source,shell] +---- ./mvnw clean package -pl :jdbc-source cd applications/source/jdbc-source/apps/jdbc-source-kafka ./mvnw clean package -``` +---- This will generate the Kafka binder based uber jar in the target folder. Similarly for the log sink, do the following. -``` +[source,shell] +---- ./mvnw clean package -pl :log-sink cd applications/sink/log-sink/apps/log-sink-rabbit ./mvnw clean package -``` +---- ==== Building a Docker image The apps use the https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin[Jib Maven Plugin] to build and publish the Docker image. @@ -119,25 +121,28 @@ If you have made some changes to an app, you may want to build the image and tes NOTE: If you plan to use the image with minikube, run the following command before building the image: -``` +[source,shell] +---- eval $(minikube docker-env) -``` +---- To build the image in your local registry: -``` +[source,shell] +---- ./mvn clean package jib:dockerBuild -``` +---- To publish the image to a remote registry: -``` -jib:build \ +[source,shell] +---- +./mvnw jib:build \ -Djib.to.image=myregistry/myimage:latest \ -Djib.to.auth.username=$USERNAME \ -Djib.to.auth.password=$PASSWORD -``` +---- == Patching Pre-built Applications @@ -149,15 +154,18 @@ To add `mysql` driver to `jdbc-sink` application: 1. Clone the GitHub repository at https://github.com/spring-cloud/stream-applications 2. Find the module that you want to patch and add the additional dependencies, `jdbc-sink` in this case. For example, you can add the following mysql dependency to the application generator plugin's configuration in the pom.xml: -``` +[source,xml] +---- mysql mysql-connector-java 5.1.37 -``` +---- This is how the complete plugin configuration should look like. -``` + +[source,xml] +---- org.springframework.cloud.stream.app.plugin spring-cloud-stream-app-maven-plugin @@ -182,23 +190,25 @@ This is how the complete plugin configuration should look like. -``` +---- Once the above changes are done, you can generate the binder based apps as below from the root of the repository. -``` +[source,shell] +---- ./mvnw clean install -pl :jdbc-sink -``` +---- This generates the binder based applications in the `apps` folder under `jdbc-sink` folder. In order to build the app with the binder flavor that you are interested in, you need to do the following step. -``` +[source,shell] +---- cd applications/sink/jdbc-sink -cd apps/jdbc-sink-kafka (or Rabbit if you are interested in that) +cd apps/jdbc-sink-kafka # (or Rabbit if you are interested in that) ./mvnw clean package cd target -``` +---- There you will find the binder based uber jar with your changes. @@ -210,10 +220,11 @@ If we have to update the binder dependencies from a new release of Spring Cloud Here are the steps (again, we are using `jdbc-sink-kafka` as an example). -``` +[source,shell] +---- ./mvnw clean install -pl :jdbc-sink cd applications/sink/jdbc-sink/apps/jdbc-sink-kafka -``` +---- Open the generated application's `pom.xml` and update the dependencies. If there is a new version of Spring Cloud Stream update available that contains the enhancements we are looking for, then it is easier to update the BOM itself. @@ -221,7 +232,8 @@ Find where the bom is declared in `pom.xml` and update the version. For example, if we have to update Spring Cloud Stream to `Horsham.SR10`, this version must be specified in the BOM declaration as below: -``` +[source,xml] +---- @@ -233,7 +245,7 @@ For example, if we have to update Spring Cloud Stream to `Horsham.SR10`, this ve -``` +---- We can also update any individual dependencies directly, but it is preferred to use the above `dependencyManagement` approach if there is a BOM available. This is because, when using a BOM, maven will properly use and align any transitive dependencies. @@ -248,10 +260,11 @@ First, add the resources to the classpath by placing them under `src/main/resour Then rebuild the application. -``` +[source,shell] +---- ./mvnw clean package cd target -``` +---- Here you can find the modified application jar file. @@ -268,7 +281,8 @@ cd applications/stream-applications-core We need to edit the pom.xml in this module. Find the following configuration where it defines the Kafka and RabbitMQ binders for the maven plugin. -``` +[source,xml] +---- @@ -289,12 +303,13 @@ Find the following configuration where it defines the Kafka and RabbitMQ binders -``` +---- Add the binder for which you want to generate new apps for. For example, if we want to generate applications for the https://github.com/spring-cloud/spring-cloud-stream-binder-aws-kinesis[Kinesis binder], then modify as below. -``` +[source,xml] +---- @@ -328,7 +343,7 @@ For example, if we want to generate applications for the https://github.com/spri -``` +---- Note that, we need to use the Kinesis binder version here explicitly, while both Kafka and RabbitMQ do not need them. This is because, those versions come from a dependency management while the Kinesis binder is not available through such mechanisms. @@ -343,9 +358,10 @@ If we go to the applications folder and look at the generated applications, we s For instance, if we follow the configuration above for adding the Kinesis binder, then we should see the Kinesis binder based app in the generated apps. Let's take `time-source` as an example. -``` +[source,shell] +---- cd applications/source/time-souce/apps -``` +---- Here, we should see three different binder based apps projects - `time-source-kafka`, `time-source-rabbit` and `time-source-kineses`. Similarly, this should happen for all the out of the box application projects. From 190e8dd50d1c8000d9a16283b89b308de3d58f1f Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Wed, 4 May 2022 15:10:06 +0200 Subject: [PATCH 4/5] Convert snippets to Asciidoc format and added types. Fixes #241 --- README.adoc | 10 ++-- applications/sink/router-sink/README.adoc | 8 ++-- .../source/cdc-debezium-source/README.adoc | 48 ++++++++++--------- applications/source/ftp-source/README.adoc | 5 +- docs/FunctionComposition.adoc | 10 ++-- .../cdc-debezium-boot-starter/README.adoc | 2 +- .../consumer/analytics-consumer/README.adoc | 10 ++-- 7 files changed, 50 insertions(+), 43 deletions(-) diff --git a/README.adoc b/README.adoc index e658c125..3b1b1fbc 100644 --- a/README.adoc +++ b/README.adoc @@ -203,21 +203,23 @@ Let's assume that you want to build a `jdbc-source` application based on Kafka B Here is what you need to do. Assuming that you already built both functions and stream-applications-core as above, -``` +[source,shell] +---- ./mvnw clean package -pl :jdbc-source cd applications/source/jdbc-source/apps/jdbc-source-kafka ./mvnw clean package -``` +---- This will generate the Kafka binder based uber jar in the target folder. Similarly, for the `log-sink`, do the following. -``` +[source,shell] +---- ./mvnw clean package -pl :log-sink cd applications/sink/log-sink/apps/log-sink-rabbit ./mvnw clean package -``` +---- === Additional Resources diff --git a/applications/sink/router-sink/README.adoc b/applications/sink/router-sink/README.adoc index 8e0f0337..4b8b3853 100644 --- a/applications/sink/router-sink/README.adoc +++ b/applications/sink/router-sink/README.adoc @@ -47,14 +47,14 @@ https://docs.spring.io/spring-integration/reference/html/xml.html#xpath-spel-fun For example for `text` content type one should use: -[source] +[source,java] ---- - new String(payload).contains('a') + new String(payload).contains('a'); ---- and for `json` content type SpEL expressions like this: -[source] +[source,text] ---- #jsonPath(payload, '$.person.name') ---- @@ -66,7 +66,7 @@ Instead of SpEL expressions, Groovy scripts can also be used. Let's create a Gro [source,groovy] ---- -println("Groovy processing payload '" + payload + "'"); +println("Groovy processing payload '" + payload + "'") if (payload.contains('a')) { return "foo" } diff --git a/applications/source/cdc-debezium-source/README.adoc b/applications/source/cdc-debezium-source/README.adoc index 633e48ee..4df8572c 100644 --- a/applications/source/cdc-debezium-source/README.adoc +++ b/applications/source/cdc-debezium-source/README.adoc @@ -27,7 +27,7 @@ $$connector$$:: $$Shortcut for the cdc.config.connector.class property. Either o $$name$$:: $$Unique name for this sourceConnector instance.$$ *($$String$$, default: `$$$$`)* $$schema$$:: $$Include the schema's as part of the outbound message.$$ *($$Boolean$$, default: `$$false$$`)* -=== cdc.flattening +==== cdc.flattening $$add-fields$$:: $$Comma separated list of metadata fields to add to the flattened message. The fields will be prefixed with "__" or "__[<]struct]__", depending on the specification of the struct.$$ *($$String$$, default: `$$$$`)* $$add-headers$$:: $$Comma separated list specify a list of metadata fields to add to the header of the flattened message. The fields will be prefixed with "__" or "__[struct]__".$$ *($$String$$, default: `$$$$`)* @@ -35,19 +35,23 @@ $$delete-handling-mode$$:: $$Options for handling deleted records: (1) none - pa $$drop-tombstones$$:: $$By default Debezium generates tombstone records to enable Kafka compaction on deleted records. The dropTombstones can suppress the tombstone records.$$ *($$Boolean$$, default: `$$true$$`)* $$enabled$$:: $$Enable flattening the source record events (https://debezium.io/docs/configuration/event-flattening).$$ *($$Boolean$$, default: `$$true$$`)* -=== cdc.offset +==== cdc.offset $$commit-timeout$$:: $$Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt.$$ *($$Duration$$, default: `$$5000ms$$`)* $$flush-interval$$:: $$Interval at which to try committing offsets. The default is 1 minute.$$ *($$Duration$$, default: `$$60000ms$$`)* $$policy$$:: $$Offset storage commit policy.$$ *($$OffsetPolicy$$, default: `$$$$`)* $$storage$$:: $$Kafka connector tracks the number processed records and regularly stores the count (as "offsets") in a preconfigured metadata storage. On restart the connector resumes the reading from the last recorded source offset.$$ *($$OffsetStorageType$$, default: `$$$$`, possible values: `memory`,`file`,`kafka`,`metadata`)* -=== cdc.stream.header +==== cdc.stream.header $$convert-connect-headers$$:: $$When true the {@link org.apache.kafka.connect.header.Header} are converted into message headers with the {@link org.apache.kafka.connect.header.Header#key()} as name and {@link org.apache.kafka.connect.header.Header#value()}.$$ *($$Boolean$$, default: `$$true$$`)* $$offset$$:: $$Serializes the source record's offset metadata into the outbound message header under cdc.offset.$$ *($$Boolean$$, default: `$$false$$`)* -=== metadata.store.dynamo-db +=== metadata.store + +$$type$$:: $$Indicates the type of metadata store to configure (default is 'memory'). You must include the corresponding Spring Integration dependency to use a persistent store.$$ *($$StoreType$$, default: `$$$$`, possible values: `mongodb`,`gemfire`,`redis`,`dynamodb`,`jdbc`,`zookeeper`,`hazelcast`,`memory`)* + +==== metadata.store.dynamo-db $$create-delay$$:: $$Delay between create table retries.$$ *($$Integer$$, default: `$$1$$`)* $$create-retries$$:: $$Retry number for create table request.$$ *($$Integer$$, default: `$$25$$`)* @@ -56,28 +60,24 @@ $$table$$:: $$Table name for metadata.$$ *($$String$$, default: `$$$$`)* $$time-to-live$$:: $$TTL for table entries.$$ *($$Integer$$, default: `$$$$`)* $$write-capacity$$:: $$Write capacity on the table.$$ *($$Long$$, default: `$$1$$`)* -=== metadata.store.gemfire +==== metadata.store.gemfire $$region$$:: $$Gemfire region name for metadata.$$ *($$String$$, default: `$$$$`)* -=== metadata.store.jdbc +==== metadata.store.jdbc $$region$$:: $$Unique grouping identifier for messages persisted with this store.$$ *($$String$$, default: `$$DEFAULT$$`)* $$table-prefix$$:: $$Prefix for the custom table name.$$ *($$String$$, default: `$$$$`)* -=== metadata.store.mongo-db +==== metadata.store.mongo-db $$collection$$:: $$MongoDB collection name for metadata.$$ *($$String$$, default: `$$metadataStore$$`)* -=== metadata.store.redis +==== metadata.store.redis $$key$$:: $$Redis key for metadata.$$ *($$String$$, default: `$$$$`)* -=== metadata.store - -$$type$$:: $$Indicates the type of metadata store to configure (default is 'memory'). You must include the corresponding Spring Integration dependency to use a persistent store.$$ *($$StoreType$$, default: `$$$$`, possible values: `mongodb`,`gemfire`,`redis`,`dynamodb`,`jdbc`,`zookeeper`,`hazelcast`,`memory`)* - -=== metadata.store.zookeeper +==== metadata.store.zookeeper $$connect-string$$:: $$Zookeeper connect string in form HOST:PORT.$$ *($$String$$, default: `$$127.0.0.1:2181$$`)* $$encoding$$:: $$Encoding to use when storing data in Zookeeper.$$ *($$Charset$$, default: `$$UTF-8$$`)* @@ -158,9 +158,9 @@ mysql> GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIE ---- ==== -Use following properties to connect the CDC Source to the MySQL DB: +Use following properties to connect the CDC Source to MySQL DB: -[source] +[source,properties] ---- cdc.connector=mysql # <1> @@ -176,6 +176,7 @@ cdc.config.database.port=3306 # <3> cdc.schema=true # <4> cdc.flattening.enabled=true # <5> ---- + <1> Configures the CDC Source to use https://debezium.io/docs/connectors/mysql/[MySqlConnector]. (equivalent to setting `cdc.config.connector.class=io.debezium.connector.mysql.MySqlConnector`). <2> Metadata used to identify and dispatch the incoming events. <3> Connection to the MySQL server running on `localhost:3306` as `debezium` user. @@ -198,9 +199,9 @@ You can connect to this server like this: psql -U postgres -h localhost -p 5432 ---- -Use following properties to connect the CDC Source to the PostgreSQL: +Use following properties to connect the CDC Source to PostgreSQL: -[source] +[source,properties] ---- cdc.connector=postgres # <1> cdc.offset.storage=memory #<2> @@ -251,9 +252,9 @@ In the `mongodb` terminal output, search for a log entry like `host: "3f95a8a651 Add `127.0.0.1 3f95a8a6516e` entry to your `/etc/hosts` -Use following properties to connect the CDC Source to the MongoDB: +Use following properties to connect the CDC Source to MongoDB: -[source] +[source,properties] ---- cdc.connector=mongodb # <1> cdc.offset.storage=memory #<2> @@ -294,9 +295,9 @@ wget https://raw.githubusercontent.com/debezium/debezium-examples/master/tutoria cat ./inventory.sql | docker exec -i sqlserver bash -c '/opt/mssql-tools/bin/sqlcmd -U sa -P $SA_PASSWORD' ---- -Use following properties to connect the CDC Source to the SQLServer: +Use following properties to connect the CDC Source to SQLServer: -[source] +[source,properties] ---- cdc.connector=sqlserver # <1> cdc.offset.storage=memory #<2> @@ -335,7 +336,8 @@ cat ./inventory.sql | docker exec -i dbz_oracle sqlplus debezium/dbz@//localhost == Run standalone -``` +[source,shell] +---- java -jar cdc-debezium-source.jar --cdc.connector=mysql --cdc.name=my-sql-connector --cdc.config.database.server.id=85744 --cdc.config.database.server.name=my-app-connector --cdc.config.database.user=debezium --cdc.config.database.password=dbz --cdc.config.database.hostname=localhost --cdc.config.database.port=3306 --cdc.schema=true --cdc.flattening.enabled=true -``` +---- diff --git a/applications/source/ftp-source/README.adoc b/applications/source/ftp-source/README.adoc index 9bc8a1cd..93b5f496 100644 --- a/applications/source/ftp-source/README.adoc +++ b/applications/source/ftp-source/README.adoc @@ -139,8 +139,9 @@ $$root$$:: $$Root node - store entries are children of this node.$$ *($$String$$ == Examples -``` +[source,shell] +---- java -jar ftp_source.jar --ftp.supplier.remote-dir=foo --file.consumer.mode=lines --ftp.factory.host=ftpserver \ --ftp.factory.username=user --ftp.factory.password=pw --ftp.local-dir=/foo -``` +---- //end::ref-doc[] diff --git a/docs/FunctionComposition.adoc b/docs/FunctionComposition.adoc index a328deb6..978696af 100644 --- a/docs/FunctionComposition.adoc +++ b/docs/FunctionComposition.adoc @@ -22,18 +22,20 @@ standalone processor. For example, the `time` source when it is running, as shown below, will perform a series of internal transformations to finally publish a task launch request every second to the rabbit exchange with the name `time-test`. -``` +[source,shell] +---- java -jar target/time-source-rabbit-3.0.0-SNAPSHOT.jar \ --spring.cloud.stream.bindings.output.destination=time-test \ --spring.cloud.stream.function.definition="timeSupplier|spelFunction|headerEnricherFunction|taskLaunchRequestFunction" \ --spel.function.expression="payload.length()" \ --header.enricher.headers=task-id=payload*2 \ --task.launch.request.task-name-expression="'task-'+headers['task-id']" -``` +---- Now, the transformed message would look like: -``` +[source,text] +---- headers: task-id: 34 content_type: application/json @@ -41,7 +43,7 @@ Payload 49 bytes Encoding: string {"args":[],"deploymentProps":{},"name":"task-34"} -``` +---- Let us unpack what is happening behind the scenes. We will start with the following function definition. diff --git a/functions/common/cdc-debezium-boot-starter/README.adoc b/functions/common/cdc-debezium-boot-starter/README.adoc index a274e652..af7320bf 100644 --- a/functions/common/cdc-debezium-boot-starter/README.adoc +++ b/functions/common/cdc-debezium-boot-starter/README.adoc @@ -79,7 +79,7 @@ mysql> GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIE Configure `cdc-debezium-boot-starter` for consuming cdc events from the MySQL: -[source] +[source,properties] ---- cdc.name=my-sql-connector # <1> diff --git a/functions/consumer/analytics-consumer/README.adoc b/functions/consumer/analytics-consumer/README.adoc index d3e771b9..c845085d 100644 --- a/functions/consumer/analytics-consumer/README.adoc +++ b/functions/consumer/analytics-consumer/README.adoc @@ -46,7 +46,7 @@ The https://docs.spring.io/spring-integration/reference/html/message.html#messag The `SpEL` expressions use the `headers` and `payload` keywords to access message’s headers and payload values. For example a counter metrics can have a value amount computed from the size of the input message payload add a `my_tag` tag, extracted from the `kind` header value: -[source] +[source,properties] ---- analytics.amount-expression=payload.lenght() analytics.tag.expression.my_tag=headers['kind'] @@ -74,13 +74,13 @@ To enable support for another monitoring system you have to add the spring-boot- Follow the https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/production-ready-features.html#production-ready-metrics-export[configuration instructions] for the selected monitoring system. All monitoring configuration properties start with a prefix: `management.metrics.export`. -## Configuration Options +== Configuration Options All `analytics-consumer` configuration properties use the `analytics` prefix. For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java[AnalyticsConsumerProperties]. All monitoring configuration properties start with a prefix `management.metrics.export`. For configuring a particular monitoring system follow the provided https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/production-ready-features.html#production-ready-metrics-export[configuration instructions]. -#### Sample Configuration +==== Sample Configuration Following examples show how to configure `counter` and `gauge` metrics over a series of stock-exchange messages like this: @@ -166,11 +166,11 @@ To enable one or more https://micrometer.io/docs[supported monitoring systems] y |=== -## Tests +== Tests See this link:src/test/java/org/springframework/cloud/fn/consumer/analytics[test suite] for the various ways, this consumer is used. -## Other usage +== Other usage * See the https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/analytics-sink/README.adoc[Analytics Sink README] where this consumer is used to create a Spring Cloud Stream application where it makes a Counter sink. From ad3d7cd8ac03d85059636a575a05f0f92bdc6acb Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Tue, 17 May 2022 21:09:45 +0200 Subject: [PATCH 5/5] Improve indentation in docs Added description for building local container images. Fixes #241 --- README.adoc | 7 +++++ .../source/cdc-debezium-source/README.adoc | 26 +++++++++---------- build-images.sh => local/build-images.sh | 0 3 files changed, 20 insertions(+), 13 deletions(-) rename build-images.sh => local/build-images.sh (100%) diff --git a/README.adoc b/README.adoc index 3b1b1fbc..675a6b82 100644 --- a/README.adoc +++ b/README.adoc @@ -221,6 +221,13 @@ cd applications/sink/log-sink/apps/log-sink-rabbit ./mvnw clean package ---- +Rebuild all container images. + +[source,shell] +---- +./local/build-images.sh +---- + === Additional Resources Here is a list of resources where you can find out more about using and developing functions and stream applications: diff --git a/applications/source/cdc-debezium-source/README.adoc b/applications/source/cdc-debezium-source/README.adoc index 4df8572c..be561704 100644 --- a/applications/source/cdc-debezium-source/README.adoc +++ b/applications/source/cdc-debezium-source/README.adoc @@ -27,7 +27,7 @@ $$connector$$:: $$Shortcut for the cdc.config.connector.class property. Either o $$name$$:: $$Unique name for this sourceConnector instance.$$ *($$String$$, default: `$$$$`)* $$schema$$:: $$Include the schema's as part of the outbound message.$$ *($$Boolean$$, default: `$$false$$`)* -==== cdc.flattening +=== cdc.flattening $$add-fields$$:: $$Comma separated list of metadata fields to add to the flattened message. The fields will be prefixed with "__" or "__[<]struct]__", depending on the specification of the struct.$$ *($$String$$, default: `$$$$`)* $$add-headers$$:: $$Comma separated list specify a list of metadata fields to add to the header of the flattened message. The fields will be prefixed with "__" or "__[struct]__".$$ *($$String$$, default: `$$$$`)* @@ -35,23 +35,19 @@ $$delete-handling-mode$$:: $$Options for handling deleted records: (1) none - pa $$drop-tombstones$$:: $$By default Debezium generates tombstone records to enable Kafka compaction on deleted records. The dropTombstones can suppress the tombstone records.$$ *($$Boolean$$, default: `$$true$$`)* $$enabled$$:: $$Enable flattening the source record events (https://debezium.io/docs/configuration/event-flattening).$$ *($$Boolean$$, default: `$$true$$`)* -==== cdc.offset +=== cdc.offset $$commit-timeout$$:: $$Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt.$$ *($$Duration$$, default: `$$5000ms$$`)* $$flush-interval$$:: $$Interval at which to try committing offsets. The default is 1 minute.$$ *($$Duration$$, default: `$$60000ms$$`)* $$policy$$:: $$Offset storage commit policy.$$ *($$OffsetPolicy$$, default: `$$$$`)* $$storage$$:: $$Kafka connector tracks the number processed records and regularly stores the count (as "offsets") in a preconfigured metadata storage. On restart the connector resumes the reading from the last recorded source offset.$$ *($$OffsetStorageType$$, default: `$$$$`, possible values: `memory`,`file`,`kafka`,`metadata`)* -==== cdc.stream.header +=== cdc.stream.header $$convert-connect-headers$$:: $$When true the {@link org.apache.kafka.connect.header.Header} are converted into message headers with the {@link org.apache.kafka.connect.header.Header#key()} as name and {@link org.apache.kafka.connect.header.Header#value()}.$$ *($$Boolean$$, default: `$$true$$`)* $$offset$$:: $$Serializes the source record's offset metadata into the outbound message header under cdc.offset.$$ *($$Boolean$$, default: `$$false$$`)* -=== metadata.store - -$$type$$:: $$Indicates the type of metadata store to configure (default is 'memory'). You must include the corresponding Spring Integration dependency to use a persistent store.$$ *($$StoreType$$, default: `$$$$`, possible values: `mongodb`,`gemfire`,`redis`,`dynamodb`,`jdbc`,`zookeeper`,`hazelcast`,`memory`)* - -==== metadata.store.dynamo-db +=== metadata.store.dynamo-db $$create-delay$$:: $$Delay between create table retries.$$ *($$Integer$$, default: `$$1$$`)* $$create-retries$$:: $$Retry number for create table request.$$ *($$Integer$$, default: `$$25$$`)* @@ -60,24 +56,28 @@ $$table$$:: $$Table name for metadata.$$ *($$String$$, default: `$$$$`)* $$time-to-live$$:: $$TTL for table entries.$$ *($$Integer$$, default: `$$$$`)* $$write-capacity$$:: $$Write capacity on the table.$$ *($$Long$$, default: `$$1$$`)* -==== metadata.store.gemfire +=== metadata.store.gemfire $$region$$:: $$Gemfire region name for metadata.$$ *($$String$$, default: `$$$$`)* -==== metadata.store.jdbc +=== metadata.store.jdbc $$region$$:: $$Unique grouping identifier for messages persisted with this store.$$ *($$String$$, default: `$$DEFAULT$$`)* $$table-prefix$$:: $$Prefix for the custom table name.$$ *($$String$$, default: `$$$$`)* -==== metadata.store.mongo-db +=== metadata.store.mongo-db $$collection$$:: $$MongoDB collection name for metadata.$$ *($$String$$, default: `$$metadataStore$$`)* -==== metadata.store.redis +=== metadata.store.redis $$key$$:: $$Redis key for metadata.$$ *($$String$$, default: `$$$$`)* -==== metadata.store.zookeeper +=== metadata.store + +$$type$$:: $$Indicates the type of metadata store to configure (default is 'memory'). You must include the corresponding Spring Integration dependency to use a persistent store.$$ *($$StoreType$$, default: `$$$$`, possible values: `mongodb`,`gemfire`,`redis`,`dynamodb`,`jdbc`,`zookeeper`,`hazelcast`,`memory`)* + +=== metadata.store.zookeeper $$connect-string$$:: $$Zookeeper connect string in form HOST:PORT.$$ *($$String$$, default: `$$127.0.0.1:2181$$`)* $$encoding$$:: $$Encoding to use when storing data in Zookeeper.$$ *($$Charset$$, default: `$$UTF-8$$`)* diff --git a/build-images.sh b/local/build-images.sh similarity index 100% rename from build-images.sh rename to local/build-images.sh