diff --git a/confluent-schema-registry-integration/README.adoc b/confluent-schema-registry-integration/README.adoc new file mode 100644 index 0000000..f73d54e --- /dev/null +++ b/confluent-schema-registry-integration/README.adoc @@ -0,0 +1,170 @@ +== Schema evolution with Confluent Schema Registry + +A collection of Spring Boot applications to demonstrate schema evolution using Spring Cloud Stream and Confluent Schema Registry. + +:fn-native-encoding: footnote:[By default, message encoding is performed transparently by the framework based on contentType. However, when native encoding is used, the default encoding is disabled and instead handled by the client library. It is the responsibility of the consumer to use an appropriate decoder to deserialize the inbound message and the responsibility of the producer to use an appropriate encoder to serialize the outbound message.] + +All components (producers and consumers) in this sample are configured to use native encoding{fn-native-encoding} which ensures their use of the Confluent provided Avro serializers, rather than the Spring Cloud Stream provided Avro serializers. The Confluent serializers leverage the Confluent schema registry to validate schemas during record production. An additional benefit is that these components are now cross compatible with external tools that use the Confluent Avro serializers such as the out-of-the box tools - `kafka-avro-console-consumer` and `kafka-avro-console-producer` - that come with Confluent Schema registry. + +=== Requirement +As a developer, I'd like to design my consumer to be resilient to differing payload schemas, and I want to use Confluent Schema Registry. + +=== Assumptions +There are a lot of online literature on schema evolution, so we are going to skip defining them here. For this sample, however, we will simply assume there are two producers producing events with different payload schemas and a consumer that is able to consume both of the payloads. + +[[build-apps]] +=== Building the applications +To build the applications simply execute the following command from the `confluent-schema-registry-integration` directory: +[source,bash] +---- +./mvnw clean install +---- +NOTE: The apps can be built and run from w/in an IDE (such as IntelliJ) but you will need to invoke the Maven `package` goal and then `refresh` the project as the Avro Maven plugin needs to execute so that it generates the required model classes - otherwise you will see compile failures around missing `Sensor` class. + +[[run-apps]] +=== Running the applications + +==== Pre-requisites +**** +* The components have all been built by following the <> steps. +* Apache Kafka broker available at `localhost:9092` +* Confluent Schema Registry available at `localhost:8081` + +TIP: The included link:../../../tools/kafka/docker-compose/README.adoc#_all_the_things[Kafka tools] can be used to easily start a broker and schema registry locally on the required coordinates +**** + +==== Steps +Make sure the above pre-requisites are satisfied and that you are in the `confluent-schema-registry-integration` directory and follow the steps below. + +===== Adjust schema compatibility +Execute the following command to set the schema compatibility mode to `NONE` on the Confluent schema registry: +[source,bash] +---- +curl -X PUT http://localhost:8081/config -d '{"compatibility": "NONE"}' -H "Content-Type:application/json" +---- +.Expected output +[source,json] +---- +{"compatibility":"NONE"} +---- + +===== Start the applications + +* Start `consumer` on another terminal session + +[source,bash] +---- +cd confluent-schema-registry-integration-consumer +java -jar target/confluent-schema-registry-integration-consumer-4.0.0-SNAPSHOT.jar +---- + +* Start `producer1` on another terminal session + +[source,bash] +---- +cd confluent-schema-registry-integration-producer1 +java -jar target/confluent-schema-registry-integration-producer1-4.0.0-SNAPSHOT.jar +---- + +* Start `producer2` on another terminal session + +[source,bash] +---- +cd confluent-schema-registry-integration-producer2 +java -jar target/confluent-schema-registry-integration-producer2-4.0.0-SNAPSHOT.jar +---- + +===== Send sample data +The producer apps each expose a REST endpoint which sends a sample Kafka event when invoked. Execute the following commands to send some sample data. +[source,bash] +---- +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9010/randomMessage +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9010/randomMessage +---- + +===== View consumer output +The consumer should log the results which should look something like: + +[source,bash,options=nowrap,subs=attributes] +---- +{"id": "10fd4598-0a12-4d9b-9512-3e2257d9b713-v1", "internalTemperature": 28.674625, "externalTemperature": 0.0, "acceleration": 5.3196855, "velocity": 41.38155} +{"id": "cc3555b0-0a52-44ef-9a06-89ffda361e0b-v2", "internalTemperature": 8.026814, "externalTemperature": 0.0, "acceleration": 7.5858965, "velocity": 79.71516} +{"id": "5a23d687-093f-4b13-a42f-19842041a96f-v1", "internalTemperature": 31.853527, "externalTemperature": 0.0, "acceleration": 6.006965, "velocity": 19.520967} +{"id": "3fe53513-def6-4376-81b9-a5b332d54fc2-v1", "internalTemperature": 33.353485, "externalTemperature": 0.0, "acceleration": 5.429616, "velocity": 82.439064} +{"id": "5813a495-748c-4485-ac85-cd6ad28a839c-v2", "internalTemperature": 6.988123, "externalTemperature": 0.0, "acceleration": 1.4945298, "velocity": 51.230377} +---- + +NOTE: The `id` value is appended with `-v1` or `-v2` indicating if it was sent from `producer1` or `producer2`, respectively. + +Alternatively, the Confluent command line tools can also be used to view the output with the following command: +[source,bash] +---- +/bin/kafka-avro-console-consumer \ + --topic sensor-topic \ + --bootstrap-server localhost:9092 \ + --from-beginning +---- + +==== What just happened? +The schema evolved on the `temperature` field. That field is now split into two separate fields, `internalTemperature` and `externalTemperature`. The `producer1` produces payload only with `temperature` and on the other hand, `producer2` produces payload with `internalTemperature` and `externalTemperature`. + +The `consumer` is coded against a base schema that include the split fields. + +The `consumer` app can happily deserialize the payload with `internalTemperature` and `externalTemperature` fields. However, when a `producer1` payload arrives (which includes only the single `temperature` field), the schema evolution and compatibility check are automatically applied. + +Because each payload also includes the payload version in the header, Spring Cloud Stream with the help of Schema +Registry server and Avro, the schema evolution occurs behind the scenes. The automatic mapping of `temperature` to +`internalTemperature` field is applied. + +=== Using Confluent Control Center +Confluent provides the https://docs.confluent.io/current/control-center/index.html[Confluent Control Center] UI - a web-based tool for managing and monitoring Apache Kafka®. + +TIP: The included link:../../../tools/kafka/docker-compose/README.adoc#_all_the_things[Kafka tools] can be used to easily start a Control Center UI locally + +Once the UI is running locally and you have followed the <> section: +**** +* Open the control center at `http://localhost:9021` and click on the provided cluster +* From the vertical menu select `Topics` tab +* From the list of topics select the `sensor-topic` (created by the samples) +* Click on the `Schema` tab to view the `Sensors` schema. +**** + +=== Confluent Schema Registry REST API +The Confluent Schema Registry also makes available a REST API for schema related operations. + +For example, the `http://localhost:8081/subjects` endpoint will list the schema names (e.g. subjects) currently defined in the registry. After you have run the samples you should be able to see a schema subject name `sensor-topic-value`. + + +==== Schema naming + +By default, Confluent uses the https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#subject-name-strategy[TopicNameStrategy] to create the name of the message payload schema. The name of the schema is your topic name (e.g. `spring.cloud.stream.bindings.:destination`) appended with `-value`. + +This means that by default you can use a single schema per topic. However, the subject naming strategy can be changed to `RecordNameStrategy` or `TopicRecordNameStrategy` by setting the following properties: + +.consumer +[source,yaml] +---- +spring: + cloud: + stream: + kafka: + binder: + consumerProperties: + value.subject.name.strategy: io.confluent.kafka.serializers.subject.RecordNameStrategy +---- +.producer +[source,yaml] +---- +spring: + cloud: + stream: + kafka: + binder: + producerProperties: + value.subject.name.strategy: io.confluent.kafka.serializers.subject.RecordNameStrategy +---- + +NOTE: Currently the Control Center seems to only recognize the subjects created with the `TopicNameStrategy` . If you configure the `RecordNameStrategy` the schema will not appear in the UI. diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/pom.xml b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/pom.xml new file mode 100644 index 0000000..0c2d073 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + confluent-schema-registry-integration-consumer + 4.1.0-SNAPSHOT + jar + confluent-schema-registry-integration-consumer + Confluent Schema Registry Consumer + + + io.spring.cloud.stream.sample + confluent-schema-registry-integration + 4.1.0-SNAPSHOT + .. + + + + + + org.apache.avro + avro-maven-plugin + + + + + diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java new file mode 100644 index 0000000..8dc55bc --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/java/sample/consumer/ConfluentAvroConsumerApplication.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.consumer; + +import java.util.function.Consumer; + +import com.example.Sensor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +@SpringBootApplication(proxyBeanMethods = false) +public class ConfluentAvroConsumerApplication { + + private final Logger logger = LoggerFactory.getLogger(ConfluentAvroConsumerApplication.class); + + public static void main(String[] args) { + SpringApplication.run(ConfluentAvroConsumerApplication.class, args); + } + + @Bean + Consumer process() { + return input -> logger.info("input: {}", input); + } + +} diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/application.yml b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/application.yml new file mode 100644 index 0000000..79ad859 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/application.yml @@ -0,0 +1,24 @@ +spring: + cloud: + stream: + bindings: + process-in-0: + destination: sensor-topic + consumer: + useNativeDecoding: true + kafka: +# binder: +# consumerProperties: +# value: +# subject: +# name: +# strategy: io.confluent.kafka.serializers.subject.RecordNameStrategy + bindings: + process-in-0: + consumer: + configuration: + value.deserializer: io.confluent.kafka.serializers.KafkaAvroDeserializer + schema.registry.url: http://localhost:8081 + specific.avro.reader: true + +server.port: 9999 diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/avro/sensor.avsc b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..95afdfc --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-consumer/src/main/resources/avro/sensor.avsc @@ -0,0 +1,12 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"internalTemperature", "type":"float", "default":0.0, "aliases":["temperature"]}, + {"name":"externalTemperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} \ No newline at end of file diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/pom.xml b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/pom.xml new file mode 100644 index 0000000..8876b80 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + confluent-schema-registry-integration-producer1 + 4.1.0-SNAPSHOT + jar + confluent-schema-registry-integration-producer1 + Confluent Schema Registry Producer1 + + + io.spring.cloud.stream.sample + confluent-schema-registry-integration + 4.1.0-SNAPSHOT + .. + + + + + + org.apache.avro + avro-maven-plugin + + + + + diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java new file mode 100644 index 0000000..59e45d0 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/java/sample/producer1/ConfluentAvroProducer1Application.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.producer1; + +import java.util.Random; +import java.util.UUID; + +import com.example.Sensor; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +@SpringBootApplication(proxyBeanMethods = false) +@RestController +public class ConfluentAvroProducer1Application { + + private Random random = new Random(); + + @Autowired + private StreamBridge streamBridge; + + public static void main(String[] args) { + SpringApplication.run(ConfluentAvroProducer1Application.class, args); + } + + @PostMapping("/randomMessage") + public String sendRandomMessage() { + streamBridge.send("supplier-out-0", randomSensor()); + return "ok, have fun with v1 payload!"; + } + + private Sensor randomSensor() { + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID() + "-v1"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + sensor.setTemperature(random.nextFloat() * 50); + return sensor; + } +} diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/application.yml b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/application.yml new file mode 100644 index 0000000..c8ce0dc --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/application.yml @@ -0,0 +1,23 @@ +spring: + cloud: + stream: + bindings: + supplier-out-0: + destination: sensor-topic + producer: + useNativeEncoding: true + kafka: +# binder: +# producerProperties: +# value: +# subject: +# name: +# strategy: io.confluent.kafka.serializers.subject.RecordNameStrategy + bindings: + supplier-out-0: + producer: + configuration: + value.serializer: io.confluent.kafka.serializers.KafkaAvroSerializer + schema.registry.url: http://localhost:8081 + +server.port: 9009 diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/avro/sensor.avsc b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..c0e060d --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer1/src/main/resources/avro/sensor.avsc @@ -0,0 +1,11 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"temperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/pom.xml b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/pom.xml new file mode 100644 index 0000000..1eafe8b --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + confluent-schema-registry-integration-producer2 + 4.1.0-SNAPSHOT + jar + confluent-schema-registry-integration-producer2 + Confluent Schema Registry Producer2 + + + io.spring.cloud.stream.sample + confluent-schema-registry-integration + 4.1.0-SNAPSHOT + .. + + + + + + org.apache.avro + avro-maven-plugin + + + + + diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java new file mode 100644 index 0000000..1c3bd90 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/java/sample/producer2/ConfluentAvroProducer2Application.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.producer2; + +import java.util.Random; +import java.util.UUID; + +import com.example.Sensor; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +@SpringBootApplication(proxyBeanMethods = false) +@RestController +public class ConfluentAvroProducer2Application { + + private Random random = new Random(); + + @Autowired + private StreamBridge streamBridge; + + public static void main(String[] args) { + SpringApplication.run(ConfluentAvroProducer2Application.class, args); + } + + @PostMapping("/randomMessage") + public String sendRandomMessage() { + streamBridge.send("supplier-out-0", randomSensor()); + return "ok, have fun with v2 payload!"; + } + + private Sensor randomSensor() { + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID().toString() + "-v2"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + sensor.setInternalTemperature(random.nextFloat() * 50); + sensor.setAccelerometer(null); + sensor.setMagneticField(null); + return sensor; + } +} diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/application.yml b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/application.yml new file mode 100644 index 0000000..a95415e --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/application.yml @@ -0,0 +1,23 @@ +spring: + cloud: + stream: + bindings: + supplier-out-0: + destination: sensor-topic + producer: + useNativeEncoding: true + kafka: +# binder: +# producerProperties: +# value: +# subject: +# name: +# strategy: io.confluent.kafka.serializers.subject.RecordNameStrategy + bindings: + supplier-out-0: + producer: + configuration: + value.serializer: io.confluent.kafka.serializers.KafkaAvroSerializer + schema.registry.url: http://localhost:8081 + +server.port: 9010 diff --git a/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/avro/sensor.avsc b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..8d2e605 --- /dev/null +++ b/confluent-schema-registry-integration/confluent-schema-registry-integration-producer2/src/main/resources/avro/sensor.avsc @@ -0,0 +1,25 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"internalTemperature", "type":"float", "default":0.0, "aliases":["temperature"]}, + {"name":"externalTemperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0}, + {"name":"accelerometer","type":[ + "null",{ + "type":"array", + "items":"float" + } + ]}, + {"name":"magneticField","type":[ + "null",{ + "type":"array", + "items":"float" + } + ]} + ] + +} \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw b/confluent-schema-registry-integration/mvnw similarity index 99% rename from kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw rename to confluent-schema-registry-integration/mvnw index 2d1e3cf..0ce08e9 100755 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw +++ b/confluent-schema-registry-integration/mvnw @@ -218,8 +218,9 @@ fi WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain -exec "$JAVACMD" \ +"$JAVACMD" \ $MAVEN_OPTS \ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" + diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw.cmd b/confluent-schema-registry-integration/mvnw.cmd similarity index 96% rename from kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw.cmd rename to confluent-schema-registry-integration/mvnw.cmd index 86846ae..7ecd01d 100644 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/mvnw.cmd +++ b/confluent-schema-registry-integration/mvnw.cmd @@ -80,6 +80,8 @@ goto error :init +set MAVEN_CMD_LINE_ARGS=%* + @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". @REM Fallback to current working directory if not found. @@ -116,10 +118,10 @@ for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do s SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% if ERRORLEVEL 1 goto error goto end diff --git a/confluent-schema-registry-integration/pom.xml b/confluent-schema-registry-integration/pom.xml new file mode 100644 index 0000000..bca941c --- /dev/null +++ b/confluent-schema-registry-integration/pom.xml @@ -0,0 +1,167 @@ + + + 4.0.0 + confluent-schema-registry-integration + 4.1.0-SNAPSHOT + pom + confluent-schema-registry-integration + Confluent Schema Registry Integration Sample App + + + io.spring.cloud.stream.sample + spring-cloud-stream-samples-parent + 0.0.1-SNAPSHOT + .. + + + + confluent-schema-registry-integration-consumer + confluent-schema-registry-integration-producer1 + confluent-schema-registry-integration-producer2 + + + + 1.11.0 + 7.0.1 + 2023.0.0-SNAPSHOT + + + + + + org.springframework.cloud + spring-cloud-stream-dependencies + ${project.version} + pom + import + + + + + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-sources + + schema + protocol + idl-protocol + + + src/main/resources/avro + + + + + + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + org.apache.avro + avro + ${avro.version} + + + io.confluent + kafka-avro-serializer + ${confluent.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + + + io.confluent + kafka-schema-registry-client + ${confluent.version} + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + confluent + https://packages.confluent.io/maven/ + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release-local + + false + + + + + diff --git a/kafka-binder-native-app/README.adoc b/kafka-binder-native-app/README.adoc new file mode 100644 index 0000000..34a6ff4 --- /dev/null +++ b/kafka-binder-native-app/README.adoc @@ -0,0 +1,90 @@ +== Kafka Binder based AOT/Native Sample Application + +This sample application demonstrates a Kafka binder based Spring Cloud Stream application running as a native image. + +=== Application Details + +The application consists of a supplier that is generating a random string every second, a function that receives the generated string and uppercase it, and then finally a consumer that simply logs the uppercased function. + +The application comes with a standard configuration properties yaml file that activates all the functions in the application. +It then specifies the necessary destinations for all the bindings. +Some bindings rely on the default binding destinations (see the `application.yml` for details). +In addition, the application expects Kafka to be available on `localhost:9092`. +If that is not the case, please ensure to update that in `application.yml`. + +=== Building and Verifying the Sample App + +Before we build and run the sample app as a native image, let us do a few preliminary verifications. + +Ensure that, you are using a graalVM compatible JVM. +For example, when running this app, we used the following JVM. + +``` + java -version + +openjdk version "19.0.1" 2022-10-18 +OpenJDK Runtime Environment GraalVM CE 22.3.0 (build 19.0.1+10-jvmci-22.3-b08) +OpenJDK 64-Bit Server VM GraalVM CE 22.3.0 (build 19.0.1+10-jvmci-22.3-b08, mixed mode, sharing) +``` + +==== Testing in regular JVM mode + +First, let us run this application in regular JVM mode without generating any AOT or native specific information. + +Make sure that you are in the directory of this sample app: `samples/kafka-binder-native-app` + +``` +./mvnw clean package +``` + +and then: + +``` +java -jar target/kafka-binder-native-app-0.0.1-SNAPSHOT.jar +``` + +You should see output similar to the following: + +``` +++++++Received:MUCH +++++++Received:WOOD +++++++Received:COULD +++++++Received:A +++++++Received:WOODCHUCK +``` + +=== Testing in AOT Mode + +Next, let us run the same app in AOT mode. + +``` +./mvnw clean package -Pnative +``` + +and then: + +``` +java -Dspring.aot.enabled=true -jar target/kafka-binder-native-app-0.0.1-SNAPSHOT.jar +``` + +You should see the similar output as above. + +=== Running the app as a native image + +Now that we verified that our app is working in regular and AOT mode, let us now take it to run it as a native executable. + +First, build the native image: + +``` +./mvnw -Pnative native:compile +``` + +Then, run the generated executable: + +``` +./target/kafka-binder-native-app +``` + +You should see output similar to what we got above. + + diff --git a/kafka-binder-native-app/mvnw b/kafka-binder-native-app/mvnw new file mode 100755 index 0000000..a16b543 --- /dev/null +++ b/kafka-binder-native-app/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/kafka-binder-native-app/mvnw.cmd b/kafka-binder-native-app/mvnw.cmd new file mode 100644 index 0000000..c8d4337 --- /dev/null +++ b/kafka-binder-native-app/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-binder-native-app/pom.xml b/kafka-binder-native-app/pom.xml new file mode 100644 index 0000000..0ceaf54 --- /dev/null +++ b/kafka-binder-native-app/pom.xml @@ -0,0 +1,139 @@ + + + 4.0.0 + + io.spring.cloud.stream.sample + spring-cloud-stream-samples-parent + 0.0.1-SNAPSHOT + .. + + + com.example + kafka-binder-native-app + 0.0.1 + kafka-binder-native-app + Native Sample App based on Kafka Binder + + 17 + 4.1.0-SNAPSHOT + + + + org.springframework.cloud + spring-cloud-stream + ${spring-cloud-stream.version} + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + ${spring-cloud-stream.version} + + + org.springframework.kafka + spring-kafka + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream-test-binder + ${spring-cloud-stream.version} + test + + + org.springframework.kafka + spring-kafka-test + test + + + + + + + maven-deploy-plugin + + true + + + + org.graalvm.buildtools + native-maven-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + confluent + https://packages.confluent.io/maven/ + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release-local + + false + + + + + + diff --git a/kafka-binder-native-app/src/main/java/sample/aot/nativeApp/KafkaBinderNativeApp.java b/kafka-binder-native-app/src/main/java/sample/aot/nativeApp/KafkaBinderNativeApp.java new file mode 100644 index 0000000..750bfc9 --- /dev/null +++ b/kafka-binder-native-app/src/main/java/sample/aot/nativeApp/KafkaBinderNativeApp.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.aot.nativeApp; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +@SpringBootApplication +public class KafkaBinderNativeApp { + + public static void main(String[] args) { + SpringApplication.run(KafkaBinderNativeApp.class, args); + } + + @Bean + public Function graalUppercaseFunction() { + return String::toUpperCase; + } + + @Bean + public Consumer graalLoggingConsumer() { + return s -> { + System.out.println("++++++Received:" + s); + }; + } + + @Bean + public Supplier graalSupplier() { + final String[] splitWoodchuck = "How much wood could a woodchuck chuck if a woodchuck could chuck wood?" + .split(" "); + final AtomicInteger wordsIndex = new AtomicInteger(0); + return () -> { + int wordIndex = wordsIndex.getAndAccumulate(splitWoodchuck.length, + (curIndex, numWords) -> curIndex < numWords - 1 ? curIndex + 1 : 0); + return splitWoodchuck[wordIndex]; + }; + } + +} diff --git a/kafka-binder-native-app/src/main/resources/application.yml b/kafka-binder-native-app/src/main/resources/application.yml new file mode 100644 index 0000000..217c55d --- /dev/null +++ b/kafka-binder-native-app/src/main/resources/application.yml @@ -0,0 +1,12 @@ +spring.cloud: + function: + definition: graalSupplier;graalUppercaseFunction;graalLoggingConsumer + stream: + bindings: + graalSupplier-out-0: + destination: graalUppercaseFunction-in-0 + graalLoggingConsumer-in-0: + destination: graalUppercaseFunction-out-0 + kafka: + binder: + brokers: localhost:9092 diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.gitignore b/kafka-streams-samples/kafka-streams-interactive-query-advanced/.gitignore deleted file mode 100644 index 2af7cef..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans - -### IntelliJ IDEA ### -.idea -*.iws -*.iml -*.ipr - -### NetBeans ### -nbproject/private/ -build/ -nbbuild/ -dist/ -nbdist/ -.nb-gradle/ \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.jar b/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.jar deleted file mode 100644 index 9cc84ea..0000000 Binary files a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.properties b/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.properties deleted file mode 100644 index c315043..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/README.adoc b/kafka-streams-samples/kafka-streams-interactive-query-advanced/README.adoc deleted file mode 100644 index e44e1b6..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/README.adoc +++ /dev/null @@ -1,41 +0,0 @@ -== What is this app? - -This is an example of a Spring Cloud Stream processor using Kafka Streams support. - -This example is a Spring Cloud Stream adaptation of this Kafka Streams sample: https://github.com/confluentinc/kafka-streams-examples/tree/4.0.0-post/src/main/java/io/confluent/examples/streams/interactivequeries/kafkamusic - -This sample demonstrates the concept of interactive queries in kafka streams. -There is a REST service provided as part of the application that can be used to query the store interactively. - -=== Running the app: - -We will run 2 instances of the processor application to demonstrate that regardless of which instance hosts the keys, the REST endpoint will serve the requests. -For more information on how this is done, please take a look at the application code. - -1. `docker-compose up -d` - -2. Start the confluent schema registry: The following command is based on the confluent platform and assume that you are at the root of the confluent platform's root directory. - -`./bin/schema-registry-start ./etc/schema-registry/schema-registry.properties` - -3. Go to the root of the repository and do: `./mvnw clean package` - -4. `java -jar target/kafka-streams-interactive-query-advanced-0.0.1-SNAPSHOT.jar` - -5. On another terminal session: - -`java -jar target/kafka-streams-interactive-query-advanced-0.0.1-SNAPSHOT.jar --server.port=8082 --spring.cloud.stream.kafka.streams.binder.configuration.application.server=localhost:8082' - -5. Run the stand-alone `Producers` application to generate data and start the processing. -Keep it running for a while. - -6. Go to the URL: https://localhost:8080/charts/top-five?genre=Punk -keep refreshing the URL and you will see the song play count information changes. - -Take a look at the console sessions for the applications and you will see that it may not be the processor started on 8080 that serves this request. - -7. Go to the URL: https://localhost:8082/charts/top-five?genre=Punk - -Take a look at the console sessions for the applications and you will see that it may not be the processor started on 8082 that serves this request. - -8. Once you are done with running the sample, stop the docker containers and the schema registry. \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/docker-compose.yml b/kafka-streams-samples/kafka-streams-interactive-query-advanced/docker-compose.yml deleted file mode 100644 index f5aaab4..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' -services: - kafka: - image: wurstmeister/kafka - container_name: kafka-iq-advanced - ports: - - "9092:9092" - environment: - - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 - - KAFKA_ADVERTISED_PORT=9092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - depends_on: - - zookeeper - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - environment: - - KAFKA_ADVERTISED_HOST_NAME=zookeeper diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/pom.xml b/kafka-streams-samples/kafka-streams-interactive-query-advanced/pom.xml deleted file mode 100644 index 66bd4ae..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - 4.0.0 - - kafka-streams-interactive-query-advanced - 0.0.1-SNAPSHOT - jar - kafka-streams-interactive-query-advanced - Spring Cloud Stream sample for KStream interactive queries - - - org.springframework.boot - spring-boot-starter-parent - 2.4.4 - - - - - 1.8.2 - 2020.0.2 - 4.0.0 - - - - - - org.springframework.cloud - spring-cloud-dependencies - ${spring-cloud.version} - pom - import - - - - - - - io.confluent - kafka-streams-avro-serde - ${confluent.version} - - - org.slf4j - slf4j-log4j12 - - - - - io.confluent - kafka-avro-serializer - ${confluent.version} - - - io.confluent - kafka-schema-registry-client - ${confluent.version} - - - org.springframework.cloud - spring-cloud-stream-binder-kafka-streams - 3.1.3-SNAPSHOT - - - org.springframework.boot - spring-boot-starter-test - test - - - - org.springframework.boot - spring-boot-starter-actuator - - - org.springframework.boot - spring-boot-starter - - - org.springframework.boot - spring-boot-starter-web - - - - - - - org.apache.avro - avro-maven-plugin - ${avro.version} - - - generate-sources - - schema - - - src/main/resources/avro/kafka/streams/interactive/query - ${project.build.directory}/generated-sources - String - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - - - - - spring-snapshots - Spring Snapshots - https://repo.spring.io/libs-snapshot-local - - true - - - false - - - - spring-milestones - Spring Milestones - https://repo.spring.io/libs-milestone-local - - false - - - - confluent - https://packages.confluent.io/maven/ - - - - - - diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java deleted file mode 100644 index 2f29acf..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/KafkaStreamsInteractiveQuerySample.java +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.streams.interactive.query; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; -import java.util.function.BiConsumer; - -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; -import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; -import kafka.streams.interactive.query.avro.PlayEvent; -import kafka.streams.interactive.query.avro.Song; -import kafka.streams.interactive.query.avro.SongPlayCount; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.LongSerializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.common.utils.Bytes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Joined; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.Serialized; -import org.apache.kafka.streams.state.HostInfo; -import org.apache.kafka.streams.state.KeyValueStore; -import org.apache.kafka.streams.state.QueryableStoreTypes; -import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cloud.stream.binder.kafka.streams.InteractiveQueryService; -import org.springframework.context.annotation.Bean; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.client.RestTemplate; - -@SpringBootApplication -public class KafkaStreamsInteractiveQuerySample { - static final String PLAY_EVENTS = "play-events"; - static final String SONG_FEED = "song-feed"; - static final String TOP_FIVE_KEY = "all"; - static final String TOP_FIVE_SONGS_STORE = "top-five-songs"; - static final String ALL_SONGS = "all-songs"; - - @Autowired - private InteractiveQueryService interactiveQueryService; - - public static void main(String[] args) { - SpringApplication.run(KafkaStreamsInteractiveQuerySample.class, args); - } - - public static class KStreamMusicSampleApplication { - - private static final Long MIN_CHARTABLE_DURATION = 30 * 1000L; - private static final String SONG_PLAY_COUNT_STORE = "song-play-count"; - - static final String TOP_FIVE_SONGS_BY_GENRE_STORE = "top-five-songs-by-genre"; - static final String TOP_FIVE_KEY = "all"; - - @Bean - public BiConsumer, KTable> process() { - - return (s, t) -> { - // create and configure the SpecificAvroSerdes required in this example - final Map serdeConfig = Collections.singletonMap( - AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); - - final SpecificAvroSerde playEventSerde = new SpecificAvroSerde<>(); - playEventSerde.configure(serdeConfig, false); - - final SpecificAvroSerde keySongSerde = new SpecificAvroSerde<>(); - keySongSerde.configure(serdeConfig, true); - - final SpecificAvroSerde valueSongSerde = new SpecificAvroSerde<>(); - valueSongSerde.configure(serdeConfig, false); - - final SpecificAvroSerde songPlayCountSerde = new SpecificAvroSerde<>(); - songPlayCountSerde.configure(serdeConfig, false); - - // Accept play events that have a duration >= the minimum - final KStream playsBySongId = - s.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION) - // repartition based on song id - .map((key, value) -> KeyValue.pair(value.getSongId(), value)); - - // join the plays with song as we will use it later for charting - final KStream songPlays = playsBySongId.leftJoin(t, - (value1, song) -> song, - Joined.with(Serdes.Long(), playEventSerde, valueSongSerde)); - - // create a state store to track song play counts - final KTable songPlayCounts = songPlays.groupBy((songId, song) -> song, - Serialized.with(keySongSerde, valueSongSerde)) - .count(Materialized.>as(SONG_PLAY_COUNT_STORE) - .withKeySerde(valueSongSerde) - .withValueSerde(Serdes.Long())); - - final TopFiveSerde topFiveSerde = new TopFiveSerde(); - - // Compute the top five charts for each genre. The results of this computation will continuously update the state - // store "top-five-songs-by-genre", and this state store can then be queried interactively via a REST API (cf. - // MusicPlaysRestService) for the latest charts per genre. - songPlayCounts.groupBy((song, plays) -> - KeyValue.pair(song.getGenre().toLowerCase(), - new SongPlayCount(song.getId(), plays)), - Serialized.with(Serdes.String(), songPlayCountSerde)) - // aggregate into a TopFiveSongs instance that will keep track - // of the current top five for each genre. The data will be available in the - // top-five-songs-genre store - .aggregate(TopFiveSongs::new, - (aggKey, value, aggregate) -> { - aggregate.add(value); - return aggregate; - }, - (aggKey, value, aggregate) -> { - aggregate.remove(value); - return aggregate; - }, - Materialized.>as(TOP_FIVE_SONGS_BY_GENRE_STORE) - .withKeySerde(Serdes.String()) - .withValueSerde(topFiveSerde) - ); - - // Compute the top five chart. The results of this computation will continuously update the state - // store "top-five-songs", and this state store can then be queried interactively via a REST API (cf. - // MusicPlaysRestService) for the latest charts per genre. - songPlayCounts.groupBy((song, plays) -> - KeyValue.pair(TOP_FIVE_KEY, - new SongPlayCount(song.getId(), plays)), - Serialized.with(Serdes.String(), songPlayCountSerde)) - .aggregate(TopFiveSongs::new, - (aggKey, value, aggregate) -> { - aggregate.add(value); - return aggregate; - }, - (aggKey, value, aggregate) -> { - aggregate.remove(value); - return aggregate; - }, - Materialized.>as(TOP_FIVE_SONGS_STORE) - .withKeySerde(Serdes.String()) - .withValueSerde(topFiveSerde) - ); - }; - - } - } - - @RestController - public class FooController { - - private final Log logger = LogFactory.getLog(getClass()); - - @RequestMapping("/song/idx") - public SongBean song(@RequestParam(value="id") Long id) { - final ReadOnlyKeyValueStore songStore = - interactiveQueryService.getQueryableStore(KafkaStreamsInteractiveQuerySample.ALL_SONGS, QueryableStoreTypes.keyValueStore()); - - final Song song = songStore.get(id); - if (song == null) { - throw new IllegalArgumentException("hi"); - } - return new SongBean(song.getArtist(), song.getAlbum(), song.getName()); - } - - @RequestMapping("/charts/top-five") - @SuppressWarnings("unchecked") - public List topFive(@RequestParam(value="genre") String genre) { - - HostInfo hostInfo = interactiveQueryService.getHostInfo(KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE, - KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, new StringSerializer()); - - if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) { - logger.info("Top Five songs request served from same host: " + hostInfo); - return topFiveSongs(KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE); - } - else { - //find the store from the proper instance. - logger.info("Top Five songs request served from different host: " + hostInfo); - RestTemplate restTemplate = new RestTemplate(); - return restTemplate.postForObject( - String.format("http://%s:%d/%s", hostInfo.host(), - hostInfo.port(), "charts/top-five?genre=Punk"), "punk", List.class); - } - } - - private List topFiveSongs(final String key, - final String storeName) { - final ReadOnlyKeyValueStore topFiveStore = - interactiveQueryService.getQueryableStore(storeName, QueryableStoreTypes.keyValueStore()); - - // Get the value from the store - final TopFiveSongs value = topFiveStore.get(key); - if (value == null) { - throw new IllegalArgumentException(String.format("Unable to find value in %s for key %s", storeName, key)); - } - final List results = new ArrayList<>(); - value.forEach(songPlayCount -> { - - HostInfo hostInfo = interactiveQueryService.getHostInfo(KafkaStreamsInteractiveQuerySample.ALL_SONGS, - songPlayCount.getSongId(), new LongSerializer()); - - if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) { - logger.info("Song info request served from same host: " + hostInfo); - - final ReadOnlyKeyValueStore songStore = - interactiveQueryService.getQueryableStore(KafkaStreamsInteractiveQuerySample.ALL_SONGS, QueryableStoreTypes.keyValueStore()); - - final Song song = songStore.get(songPlayCount.getSongId()); - results.add(new SongPlayCountBean(song.getArtist(),song.getAlbum(), song.getName(), - songPlayCount.getPlays())); - } - else { - logger.info("Song info request served from different host: " + hostInfo); - RestTemplate restTemplate = new RestTemplate(); - SongBean song = restTemplate.postForObject( - String.format("http://%s:%d/%s", hostInfo.host(), - hostInfo.port(), "song/idx?id=" + songPlayCount.getSongId()), "id", SongBean.class); - results.add(new SongPlayCountBean(song.getArtist(),song.getAlbum(), song.getName(), - songPlayCount.getPlays())); - } - - - }); - return results; - } - } - - /** - * Serde for TopFiveSongs - */ - private static class TopFiveSerde implements Serde { - - @Override - public Serializer serializer() { - - return new Serializer() { - @Override - public void configure(final Map map, final boolean b) { - } - - @Override - public byte[] serialize(final String s, final TopFiveSongs topFiveSongs) { - - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - final DataOutputStream - dataOutputStream = - new DataOutputStream(out); - try { - for (SongPlayCount songPlayCount : topFiveSongs) { - dataOutputStream.writeLong(songPlayCount.getSongId()); - dataOutputStream.writeLong(songPlayCount.getPlays()); - } - dataOutputStream.flush(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return out.toByteArray(); - } - }; - } - - @Override - public Deserializer deserializer() { - - return (s, bytes) -> { - if (bytes == null || bytes.length == 0) { - return null; - } - final TopFiveSongs result = new TopFiveSongs(); - - final DataInputStream - dataInputStream = - new DataInputStream(new ByteArrayInputStream(bytes)); - - try { - while(dataInputStream.available() > 0) { - result.add(new SongPlayCount(dataInputStream.readLong(), - dataInputStream.readLong())); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return result; - }; - } - } - - /** - * Used in aggregations to keep track of the Top five songs - */ - static class TopFiveSongs implements Iterable { - private final Map currentSongs = new HashMap<>(); - private final TreeSet topFive = new TreeSet<>((o1, o2) -> { - final int result = Long.compare(o2.getPlays(), o1.getPlays()); - if (result != 0) { - return result; - } - return Long.compare(o1.getSongId(), o2.getSongId()); - }); - - public void add(final SongPlayCount songPlayCount) { - if(currentSongs.containsKey(songPlayCount.getSongId())) { - topFive.remove(currentSongs.remove(songPlayCount.getSongId())); - } - topFive.add(songPlayCount); - currentSongs.put(songPlayCount.getSongId(), songPlayCount); - if (topFive.size() > 5) { - final SongPlayCount last = topFive.last(); - currentSongs.remove(last.getSongId()); - topFive.remove(last); - } - } - - void remove(final SongPlayCount value) { - topFive.remove(value); - currentSongs.remove(value.getSongId()); - } - - - @Override - public Iterator iterator() { - return topFive.iterator(); - } - } - -} diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/Producers.java b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/Producers.java deleted file mode 100644 index e93bf4c..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/Producers.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.streams.interactive.query; - -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; -import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; -import kafka.streams.interactive.query.avro.PlayEvent; -import kafka.streams.interactive.query.avro.Song; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.LongSerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; - -import java.util.*; - -/** - * @author Soby Chacko - */ -public class Producers { - - public static void main(String... args) throws Exception { - - final Map serdeConfig = Collections.singletonMap( - AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); - final SpecificAvroSerializer playEventSerializer = new SpecificAvroSerializer<>(); - playEventSerializer.configure(serdeConfig, false); - final SpecificAvroSerializer songSerializer = new SpecificAvroSerializer<>(); - songSerializer.configure(serdeConfig, false); - - final List songs = Arrays.asList(new Song(1L, - "Fresh Fruit For Rotting Vegetables", - "Dead Kennedys", - "Chemical Warfare", - "Punk"), - new Song(2L, - "We Are the League", - "Anti-Nowhere League", - "Animal", - "Punk"), - new Song(3L, - "Live In A Dive", - "Subhumans", - "All Gone Dead", - "Punk"), - new Song(4L, - "PSI", - "Wheres The Pope?", - "Fear Of God", - "Punk"), - new Song(5L, - "Totally Exploited", - "The Exploited", - "Punks Not Dead", - "Punk"), - new Song(6L, - "The Audacity Of Hype", - "Jello Biafra And The Guantanamo School Of " - + "Medicine", - "Three Strikes", - "Punk"), - new Song(7L, - "Licensed to Ill", - "The Beastie Boys", - "Fight For Your Right", - "Hip Hop"), - new Song(8L, - "De La Soul Is Dead", - "De La Soul", - "Oodles Of O's", - "Hip Hop"), - new Song(9L, - "Straight Outta Compton", - "N.W.A", - "Gangsta Gangsta", - "Hip Hop"), - new Song(10L, - "Fear Of A Black Planet", - "Public Enemy", - "911 Is A Joke", - "Hip Hop"), - new Song(11L, - "Curtain Call - The Hits", - "Eminem", - "Fack", - "Hip Hop"), - new Song(12L, - "The Calling", - "Hilltop Hoods", - "The Calling", - "Hip Hop") - - ); - - Map props = new HashMap<>(); - props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); - props.put(ProducerConfig.RETRIES_CONFIG, 0); - props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); - props.put(ProducerConfig.LINGER_MS_CONFIG, 1); - props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, playEventSerializer.getClass()); - - - Map props1 = new HashMap<>(props); - props1.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); - props1.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, songSerializer.getClass()); - - DefaultKafkaProducerFactory pf1 = new DefaultKafkaProducerFactory<>(props1); - KafkaTemplate template1 = new KafkaTemplate<>(pf1, true); - template1.setDefaultTopic(KafkaStreamsInteractiveQuerySample.SONG_FEED); - - songs.forEach(song -> { - System.out.println("Writing song information for '" + song.getName() + "' to input topic " + - KafkaStreamsInteractiveQuerySample.SONG_FEED); - template1.sendDefault(song.getId(), song); - }); - - - DefaultKafkaProducerFactory pf = new DefaultKafkaProducerFactory<>(props); - KafkaTemplate template = new KafkaTemplate<>(pf, true); - template.setDefaultTopic(KafkaStreamsInteractiveQuerySample.PLAY_EVENTS); - - - final long duration = 60 * 1000L; - final Random random = new Random(); - - // send a play event every 100 milliseconds - while (true) { - final Song song = songs.get(random.nextInt(songs.size())); - System.out.println("Writing play event for song " + song.getName() + " to input topic " + - KafkaStreamsInteractiveQuerySample.PLAY_EVENTS); - template.sendDefault("uk", new PlayEvent(song.getId(), duration)); - - Thread.sleep(100L); - } - } - -} diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongBean.java b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongBean.java deleted file mode 100644 index 70dec5c..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongBean.java +++ /dev/null @@ -1,75 +0,0 @@ -package kafka.streams.interactive.query; - -import java.util.Objects; - -/** - * @author Soby Chacko - */ -public class SongBean { - - private String artist; - private String album; - private String name; - - public SongBean() {} - - public SongBean(final String artist, final String album, final String name) { - - this.artist = artist; - this.album = album; - this.name = name; - } - - public String getArtist() { - return artist; - } - - public void setArtist(final String artist) { - this.artist = artist; - } - - public String getAlbum() { - return album; - } - - public void setAlbum(final String album) { - this.album = album; - } - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - - @Override - public String toString() { - return "SongBean{" + - "artist='" + artist + '\'' + - ", album='" + album + '\'' + - ", name='" + name + '\'' + - '}'; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final SongBean that = (SongBean) o; - return Objects.equals(artist, that.artist) && - Objects.equals(album, that.album) && - Objects.equals(name, that.name); - } - - @Override - public int hashCode() { - return Objects.hash(artist, album, name); - } -} diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java deleted file mode 100644 index bb9808d..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/java/kafka/streams/interactive/query/SongPlayCountBean.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.streams.interactive.query; - -import java.util.Objects; - -/** - * @author Soby Chacko - */ -public class SongPlayCountBean { - - private String artist; - private String album; - private String name; - private Long plays; - - public SongPlayCountBean() {} - - public SongPlayCountBean(final String artist, final String album, final String name, final Long - plays) { - - this.artist = artist; - this.album = album; - this.name = name; - this.plays = plays; - } - - public String getArtist() { - return artist; - } - - public void setArtist(final String artist) { - this.artist = artist; - } - - public String getAlbum() { - return album; - } - - public void setAlbum(final String album) { - this.album = album; - } - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public Long getPlays() { - return plays; - } - - public void setPlays(final Long plays) { - this.plays = plays; - } - - @Override - public String toString() { - return "SongPlayCountBean{" + - "artist='" + artist + '\'' + - ", album='" + album + '\'' + - ", name='" + name + '\'' + - ", plays=" + plays + - '}'; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final SongPlayCountBean that = (SongPlayCountBean) o; - return Objects.equals(artist, that.artist) && - Objects.equals(album, that.album) && - Objects.equals(name, that.name) && - Objects.equals(plays, that.plays); - } - - @Override - public int hashCode() { - return Objects.hash(artist, album, name, plays); - } -} diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/application.yml b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/application.yml deleted file mode 100644 index 34f5403..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/application.yml +++ /dev/null @@ -1,21 +0,0 @@ -spring.cloud.stream.bindings.process-in-0: - destination: play-events -spring.cloud.stream.bindings.process-in-1: - destination: song-feed -spring.cloud.stream.kafka.streams.bindings.process-in-0: - consumer: - valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde -spring.cloud.stream.kafka.streams.bindings.process-in-1: - consumer: - valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde - materializedAs: all-songs -spring.cloud.stream.kafka.streams.binder: - brokers: localhost - configuration: - schema.registry.url: http://localhost:8081 - commit.interval.ms: 1000 -spring.cloud.stream.kafka.streams.binder.autoAddPartitions: true -spring.cloud.stream.kafka.streams.binder.minPartitionCount: 4 -spring.cloud.stream.kafka.streams.binder.configuration.application.server: localhost:8080 - -spring.application.name: kafka-streams-iq-advanced-sample \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc deleted file mode 100644 index ca904b3..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/playevent.avsc +++ /dev/null @@ -1,8 +0,0 @@ -{"namespace": "kafka.streams.interactive.query.avro", - "type": "record", - "name": "PlayEvent", - "fields": [ - {"name": "song_id", "type": "long"}, - {"name": "duration", "type": "long"} - ] -} \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/song.avsc b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/song.avsc deleted file mode 100644 index 2a05f2f..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/song.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{"namespace": "kafka.streams.interactive.query.avro", - "type": "record", - "name": "Song", - "fields": [ - {"name": "id", "type": "long"}, - {"name": "album", "type": "string"}, - {"name": "artist", "type": "string"}, - {"name": "name", "type": "string"}, - {"name": "genre", "type": "string"} - ] -} \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc deleted file mode 100644 index 6017ce7..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/avro/kafka/streams/interactive/query/songplaycount.avsc +++ /dev/null @@ -1,8 +0,0 @@ -{"namespace": "kafka.streams.interactive.query.avro", - "type": "record", - "name": "SongPlayCount", - "fields": [ - {"name": "song_id", "type": "long"}, - {"name": "plays", "type": "long"} - ] -} \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/logback.xml b/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/logback.xml deleted file mode 100644 index 870ac9e..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-advanced/src/main/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - %d{ISO8601} %5p %t %c{2}:%L - %m%n - - - - - - - \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/.jdk8 b/kafka-streams-samples/kafka-streams-interactive-query-basic/.jdk8 deleted file mode 100644 index e69de29..0000000 diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.jar b/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.jar deleted file mode 100644 index 9cc84ea..0000000 Binary files a/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.properties b/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.properties deleted file mode 100644 index c315043..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/README.adoc b/kafka-streams-samples/kafka-streams-interactive-query-basic/README.adoc deleted file mode 100644 index 59d946d..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/README.adoc +++ /dev/null @@ -1,35 +0,0 @@ -== What is this app? - -This is an example of a Spring Cloud Stream processor using Kafka Streams support. - -The example is based on a contrived use case of tracking products by interactively querying their status. -The program accepts product ID's and track their counts hitherto by interactively querying the underlying store. \ -This sample uses lambda expressions and thus requires Java 8+. - -=== Running the app: - -Go to the root of the repository and do: - -`docker-compose up -d` - -`./mvnw clean package` - -`java -jar target/kafka-streams-interactive-query-basic-0.0.1-SNAPSHOT.jar --app.product.tracker.productIds=123,124,125` - -The above command will track products with ID's 123,124 and 125 and print their counts seen so far every 30 seconds. - -Issue the following commands: - -`docker exec -it kafka-iq-basic /opt/kafka/bin/kafka-console-producer.sh --broker-list 127.0.0.1:9092 --topic products` - -Enter the following in the console producer (one line at a time) and watch the output on the console (or IDE) where the application is running. - -``` -{"id":"123"} -{"id":"124"} -{"id":"125"} -{"id":"123"} -{"id":"123"} -{"id":"123"} -``` - diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/docker-compose.yml b/kafka-streams-samples/kafka-streams-interactive-query-basic/docker-compose.yml deleted file mode 100644 index 9c33031..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' -services: - kafka: - image: wurstmeister/kafka - container_name: kafka-iq-basic - ports: - - "9092:9092" - environment: - - KAFKA_ADVERTISED_HOST_NAME=127.0.0.1 - - KAFKA_ADVERTISED_PORT=9092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - depends_on: - - zookeeper - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - environment: - - KAFKA_ADVERTISED_HOST_NAME=zookeeper diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/java/kafka/streams/product/tracker/KafkaStreamsInteractiveQueryApplication.java b/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/java/kafka/streams/product/tracker/KafkaStreamsInteractiveQueryApplication.java deleted file mode 100644 index 28460a2..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/java/kafka/streams/product/tracker/KafkaStreamsInteractiveQueryApplication.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2017 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.streams.product.tracker; - -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.utils.Bytes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.Grouped; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.state.KeyValueStore; -import org.apache.kafka.streams.state.QueryableStoreTypes; -import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.stream.binder.kafka.streams.InteractiveQueryService; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.support.serializer.JsonSerde; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.util.StringUtils; - -@SpringBootApplication -public class KafkaStreamsInteractiveQueryApplication { - - public static void main(String[] args) { - SpringApplication.run(KafkaStreamsInteractiveQueryApplication.class, args); - } - - @EnableConfigurationProperties(ProductTrackerProperties.class) - @EnableScheduling - public static class InteractiveProductCountApplication { - - private static final String STORE_NAME = "prod-id-count-store"; - - @Autowired - private InteractiveQueryService queryService; - - @Autowired - ProductTrackerProperties productTrackerProperties; - - ReadOnlyKeyValueStore keyValueStore; - - @Bean - public Function, KStream> process() { - - return input -> input - .filter((key, product) -> productIds().contains(product.getId())) - .map((key, value) -> new KeyValue<>(value.id, value)) - .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(Product.class))) - .count(Materialized.>as(STORE_NAME) - .withKeySerde(Serdes.Integer()) - .withValueSerde(Serdes.Long())) - .toStream(); - } - - private Set productIds() { - return StringUtils.commaDelimitedListToSet(productTrackerProperties.getProductIds()) - .stream().map(Integer::parseInt).collect(Collectors.toSet()); - } - - - @Scheduled(fixedRate = 30000, initialDelay = 5000) - public void printProductCounts() { - if (keyValueStore == null) { - keyValueStore = queryService.getQueryableStore(STORE_NAME, QueryableStoreTypes.keyValueStore()); - } - - for (Integer id : productIds()) { - System.out.println("Product ID: " + id + " Count: " + keyValueStore.get(id)); - } - } - - } - - @ConfigurationProperties(prefix = "app.product.tracker") - static class ProductTrackerProperties { - - private String productIds; - - public String getProductIds() { - return productIds; - } - - public void setProductIds(String productIds) { - this.productIds = productIds; - } - - } - - static class Product { - - Integer id; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - } -} diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/application.yml b/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/application.yml deleted file mode 100644 index 2b5561e..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/application.yml +++ /dev/null @@ -1,7 +0,0 @@ -spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms: 1000 -spring.cloud.stream.bindings.process-out-0: - destination: product-counts -spring.cloud.stream.bindings.process-in-0: - destination: products -spring.cloud.stream.kafka.streams.binder: -spring.application.name: kafka-streams-iq-basic-sample diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/logback.xml b/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/logback.xml deleted file mode 100644 index 870ac9e..0000000 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/src/main/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - %d{ISO8601} %5p %t %c{2}:%L - %m%n - - - - - - - \ No newline at end of file diff --git a/kafka-streams-samples/kafka-streams-interactive-query/README.adoc b/kafka-streams-samples/kafka-streams-interactive-query/README.adoc new file mode 100644 index 0000000..86367d0 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/README.adoc @@ -0,0 +1,101 @@ +== Spring Cloud Stream Kafka Streams Interactive Query + +This sample demonstrates a Spring Cloud Stream processor using Kafka Streams state store interactive query support. There is a REST service provided as part of the application that can be used to query the store interactively. + +=== Application + +The app is based on a contrived use case of tracking products by interactively querying their status. The program accepts product ID's and tracks their counts (including the top two) and makes the results available by interactively querying the underlying state stores. + +[[build-app]] +=== Building +To build the app simply execute the following command in the base directory: +[source,bash] +---- +./mvnw clean install +---- + +=== Running + +==== Ensure these pre-requisites +**** +* The app has been built by following the <> steps +* Apache Kafka broker available at `localhost:9092` + +[#kafka_tools] +TIP: The included xref:../../../tools/kafka/docker-compose/README.adoc#run_kafka_cluster[Kafka tools] can be used to easily start a broker at the required coordinates +**** + +We will run 2 instances of the app to demonstrate that regardless of which instance hosts the keys, the REST endpoint will serve the requests. + +NOTE: For more information on how this is done, please take a look at the link:./src/main/java/com/example/kafka/streams/music/ProductQueryController.java[ProductQueryController] + +==== Start the streams app (instance 1) +[source,bash] +---- +java -jar target/kafka-streams-interactive-query-advanced-4.0.0-SNAPSHOT.jar --app.product.tracker.product-ids=123,124,125 +---- + +==== Start the streams app (instance 2) +[source,bash] +---- +java -jar target/kafka-streams-interactive-query-advanced-4.0.0-SNAPSHOT.jar --server.port=8082 --spring.cloud.stream.kafka.streams.binder.configuration.application.server=localhost:8082 --app.product.tracker.product-ids=123,124,125 +---- + +==== Start the data generator +Run the stand-alone link:./src/main/java/com/example/kafka/streams/music/GenerateProducts.java[GenerateProducts] app to create random input data. + +NOTE: It can easily be started from within your IDE or on the command line with the following command: + +[source,bash] +---- +./mvnw exec:java -Dexec.mainClass="com.example.kafka.streams.music.GenerateProducts" +---- + +The output should look something like the following: +[source,bash] +---- +GenerateProducts - Writing product event for productId = 124 +GenerateProducts - Writing product event for productId = 127 +GenerateProducts - Writing product event for productId = 127 +GenerateProducts - Writing product event for productId = 124 +---- + +==== Query the state stores + +===== Product count +Navigate to http://localhost:8080/product/123 to view the count for a particular product id. Keep refreshing the URL and you will see the product count increase as the generator runs. + +Now navigate to http://localhost:8082/product/123 to view the count for the same product but served from the second app instance. Keep refreshing the URL and you will see the product count increase as the generator runs. + +Take a look at the console sessions for the applications and you will notice that one of the instances actually serves the count from its state store and the other instance simply proxies the request to that instance to retrieve the count info. + +For example, the first app instance logs may look like: +[source,bash,options=nowrap,subs=attributes] +---- +Product count for productId: 123 served from different host: HostInfo{host='localhost', port=8082} +20 +---- +and the second app instance logs may look like: +[source,bash,options=nowrap,subs=attributes] +---- +Product count for productId: 123 served from same host: HostInfo{host='localhost', port=8082} +---- + +===== Top-two products + +Navigate to http://localhost:8080/product/top-two to view the two products with the highest count at the time of the query. Keep refreshing the URL and you may see the top-two products change as the generator runs. + +Now navigate to http://localhost:8082/product/top-two to view the top-two products but served from the second app instance. Keep refreshing the URL and you may see the top-two products change as the generator runs. + +Take a look at the console sessions for the applications and you will notice that one of the instances actually serves the results from its state store and the other instance simply proxies the request to that instance to retrieve the top-two info. + +For example, the first app instance logs may look like: +[source,bash,options=nowrap,subs=attributes] +---- +Top two products served from different host: HostInfo{host='localhost', port=8082} +---- +and the second app instance logs may look like: +[source,bash,options=nowrap,subs=attributes] +---- +Top two products served from same host: HostInfo{host='localhost', port=8082} +---- diff --git a/kafka-streams-samples/kafka-streams-interactive-query/mvnw b/kafka-streams-samples/kafka-streams-interactive-query/mvnw new file mode 100755 index 0000000..a16b543 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/kafka-streams-samples/kafka-streams-interactive-query/mvnw.cmd b/kafka-streams-samples/kafka-streams-interactive-query/mvnw.cmd new file mode 100644 index 0000000..c8d4337 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-streams-samples/kafka-streams-interactive-query/pom.xml b/kafka-streams-samples/kafka-streams-interactive-query/pom.xml new file mode 100644 index 0000000..2325bf7 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + com.example + kafka-streams-interactive-query + kafka-streams-interactive-query + + + io.spring.cloud.stream.sample + spring-cloud-stream-samples-parent + 0.0.1-SNAPSHOT + ../.. + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-stream-binder-kafka-streams + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + + diff --git a/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/GenerateProducts.java b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/GenerateProducts.java new file mode 100644 index 0000000..fdc73af --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/GenerateProducts.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kafka.streams.music; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.InteractiveProductCountApplication; +import com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.Product; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serde; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.serializer.JsonSerde; + +public class GenerateProducts { + + private static final Logger LOG = LoggerFactory.getLogger(GenerateProducts.class); + + public static void main(String... args) throws Exception { + + Serde productSerde = new JsonSerde<>(Product.class); + + Map props = new HashMap<>(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ProducerConfig.RETRIES_CONFIG, 0); + props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); + props.put(ProducerConfig.LINGER_MS_CONFIG, 1); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, productSerde.serializer().getClass()); + + DefaultKafkaProducerFactory producerFactory = new DefaultKafkaProducerFactory<>(props); + KafkaTemplate template = new KafkaTemplate<>(producerFactory, true); + template.setDefaultTopic(InteractiveProductCountApplication.PRODUCT_TOPIC); + + final List products = Arrays.asList( + new Product(123), + new Product(124), + new Product(125), + new Product(126), + new Product(127)); + + final Random random = new Random(); + + // send a product event every 100 milliseconds + while (true) { + final Product product = products.get(random.nextInt(products.size())); + LOG.debug("Writing product event for productId = {}", product.id()); + template.sendDefault(product.id(), product); + Thread.sleep(1000L); + } + + } +} diff --git a/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/KafkaStreamsInteractiveQueryApplication.java b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/KafkaStreamsInteractiveQueryApplication.java new file mode 100644 index 0000000..85b4adf --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/KafkaStreamsInteractiveQueryApplication.java @@ -0,0 +1,182 @@ +/* + * Copyright 2017-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kafka.streams.music; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; +import jakarta.annotation.PostConstruct; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.KeyValueStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.support.serializer.JsonSerde; + +@SpringBootApplication(proxyBeanMethods = false) +class KafkaStreamsInteractiveQueryApplication { + + private static final Logger LOG = LoggerFactory.getLogger(KafkaStreamsInteractiveQueryApplication.class); + + public static void main(String[] args) { + SpringApplication.run(KafkaStreamsInteractiveQueryApplication.class, args); + } + + @EnableConfigurationProperties(ProductTrackerProperties.class) + static class InteractiveProductCountApplication { + + static final String PRODUCT_TOPIC = "product-events"; + static final String PRODUCT_COUNT_STORE_NAME = "product-count-store"; + static final String TOP_TWO_PRODUCTS_STORE_NAME = "top-two-products-store"; + static final String TOP_TWO_PRODUCTS_KEY = "top-two"; + + @Value("${spring.cloud.stream.kafka.streams.binder.configuration.state.dir:UNKNOWN}") + private String stateStoreDir; + + private final ProductTrackerProperties productTrackerProperties; + + InteractiveProductCountApplication(ProductTrackerProperties productTrackerProperties) { + this.productTrackerProperties = productTrackerProperties; + } + + @PostConstruct + void showStateStoreInfo() { + LOG.info("Using KafkaStreams state.dir = {}", this.stateStoreDir); + } + + @Bean + public Consumer> process() { + return (input) -> { + // Accept product events that are configured to be tracked + final KStream productsByProductId = input + .filter((key, product) -> productTrackerProperties.getProductIds().contains(product.id())) + .map((key, value) -> new KeyValue<>(value.id, value)); + + Serde productSerde = new JsonSerde<>(Product.class); + Serde productCountSerde = new JsonSerde<>(ProductCount.class); + Serde topTwoSerde = new JsonSerde<>(TopTwoProducts.class).noTypeInfo(); + + // Create a state store to track product counts + final KTable productCounts = productsByProductId + .groupBy((productId, product) -> product, Grouped.with(productSerde, productSerde)) + .count(Materialized.>as(PRODUCT_COUNT_STORE_NAME) + .withKeySerde(productSerde) + .withValueSerde(Serdes.Long())); + + // Compute the top two products and store updated result in 'top-two-products-store' + productCounts.groupBy((product, count) -> + KeyValue.pair(TOP_TWO_PRODUCTS_KEY, new ProductCount(product.id(), count)), + Grouped.with(Serdes.String(), productCountSerde)) + .aggregate( + TopTwoProducts::new, + (aggKey, value, aggregate) -> aggregate.add(value), + (aggKey, value, aggregate) -> aggregate.remove(value), + Materialized.>as(TOP_TWO_PRODUCTS_STORE_NAME) + .withKeySerde(Serdes.String()) + .withValueSerde(topTwoSerde)); //new TopTwoProductsSerde())); + }; + } + } + + record Product(Integer id) { } + + record ProductCount(Integer productId, Long count) { } + + @ConfigurationProperties(prefix = "app.product.tracker") + static class ProductTrackerProperties { + + private Set productIds = new HashSet<>(); + + public Set getProductIds() { + return productIds; + } + + public void setProductIds(Set productIds) { + this.productIds = productIds; + } + } + + /** + * Used in aggregations to keep track of the Top two products + */ + static class TopTwoProducts { + + @JsonIgnore + private final Map currentProducts = new HashMap<>(); + + @JsonIgnore + private final TreeSet topTwo = new TreeSet<>((o1, o2) -> { + final int result = Long.compare(o2.count(), o1.count()); + if (result != 0) { + return result; + } + return Integer.compare(o1.productId(), o2.productId()); + }); + + public TopTwoProducts add(final ProductCount productCount) { + if (currentProducts.containsKey(productCount.productId())) { + topTwo.remove(currentProducts.remove(productCount.productId())); + } + topTwo.add(productCount); + currentProducts.put(productCount.productId(), productCount); + if (topTwo.size() > 2) { + final ProductCount last = topTwo.last(); + currentProducts.remove(last.productId()); + topTwo.remove(last); + } + return this; + } + + public TopTwoProducts remove(final ProductCount value) { + topTwo.remove(value); + currentProducts.remove(value.productId()); + return this; + } + + @JsonGetter + public List products() { + return topTwo.stream().toList(); + } + + @JsonSetter + @SuppressWarnings("unused") + public void setProducts(List products) { + products.forEach(this::add); + } + } +} diff --git a/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/ProductQueryController.java b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/ProductQueryController.java new file mode 100644 index 0000000..0ace02a --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/src/main/java/com/example/kafka/streams/music/ProductQueryController.java @@ -0,0 +1,83 @@ +package com.example.kafka.streams.music; + +import java.util.List; +import java.util.Optional; + +import com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.Product; +import com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.ProductCount; +import com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.TopTwoProducts; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.state.HostInfo; +import org.apache.kafka.streams.state.QueryableStoreTypes; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.cloud.stream.binder.kafka.streams.InteractiveQueryService; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.support.serializer.JsonSerde; +import org.springframework.kafka.support.serializer.JsonSerializer; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; + +import static com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.InteractiveProductCountApplication.PRODUCT_COUNT_STORE_NAME; +import static com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.InteractiveProductCountApplication.TOP_TWO_PRODUCTS_KEY; +import static com.example.kafka.streams.music.KafkaStreamsInteractiveQueryApplication.InteractiveProductCountApplication.TOP_TWO_PRODUCTS_STORE_NAME; + +@RestController +class ProductQueryController { + + private static final Logger LOG = LoggerFactory.getLogger(ProductQueryController.class); + + private final InteractiveQueryService queryService; + + ProductQueryController(InteractiveQueryService queryService) { + this.queryService = queryService; + } + + @GetMapping("/product/{id}") + public ResponseEntity productCount(@PathVariable("id") Integer productId) { + JsonSerializer productSerializer = new JsonSerde().serializer(); + HostInfo hostInfo = queryService.getHostInfo(PRODUCT_COUNT_STORE_NAME, new Product(productId), productSerializer); + if (queryService.getCurrentHostInfo().equals(hostInfo)) { + LOG.info("Product count for productId: {} served from same host: {}", productId, hostInfo); + ReadOnlyKeyValueStore productCountStore = + queryService.getQueryableStore(PRODUCT_COUNT_STORE_NAME, QueryableStoreTypes.keyValueStore()); + Long count = productCountStore.get(new Product(productId)); + if (count == null) { + return ResponseEntity.notFound().build(); + } + return ResponseEntity.of(Optional.of(new ProductCount(productId, count))); + } + LOG.info("Product count for productId: {} served from different host: {}", productId, hostInfo); + RestTemplate restTemplate = new RestTemplate(); + return restTemplate.getForEntity( + String.format("http://%s:%d/product/%d", hostInfo.host(), hostInfo.port(), productId), ProductCount.class); + } + + @GetMapping("/product/top-two") + public ResponseEntity> topTwo() { + HostInfo hostInfo = queryService.getHostInfo(TOP_TWO_PRODUCTS_STORE_NAME, TOP_TWO_PRODUCTS_KEY, new StringSerializer()); + if (queryService.getCurrentHostInfo().equals(hostInfo)) { + LOG.info("Top two products served from same host: {}", hostInfo); + ReadOnlyKeyValueStore topTwoProductsStore = + queryService.getQueryableStore(TOP_TWO_PRODUCTS_STORE_NAME, QueryableStoreTypes.keyValueStore()); + TopTwoProducts topTwo = topTwoProductsStore.get(TOP_TWO_PRODUCTS_KEY); + if (topTwo == null) { + return ResponseEntity.notFound().build(); + } + return ResponseEntity.of(Optional.of(topTwo.products())); + } + LOG.info("Top two products served from different host: {}", hostInfo); + RestTemplate restTemplate = new RestTemplate(); + return restTemplate.exchange(String.format("http://%s:%d/product/top-two", hostInfo.host(), hostInfo.port()), + HttpMethod.GET, + null, + new ParameterizedTypeReference<>() {}); + } + +} diff --git a/kafka-streams-samples/kafka-streams-interactive-query/src/main/resources/application.yml b/kafka-streams-samples/kafka-streams-interactive-query/src/main/resources/application.yml new file mode 100644 index 0000000..1384204 --- /dev/null +++ b/kafka-streams-samples/kafka-streams-interactive-query/src/main/resources/application.yml @@ -0,0 +1,15 @@ +spring: + application.name: kafka-streams-iq-adv-sample + cloud: + stream: + bindings: + process-in-0: + destination: product-events + kafka.streams: + binder: + auto-add-partitions: true + min-partition-count: 4 + configuration: + application.server: localhost:8080 + commit.interval.ms: 1000 + state.dir: "${java.io.tmpdir}/kafka-streams_${server.port:0}" diff --git a/kafka-streams-samples/pom.xml b/kafka-streams-samples/pom.xml index d43f857..69a4fed 100644 --- a/kafka-streams-samples/pom.xml +++ b/kafka-streams-samples/pom.xml @@ -15,8 +15,6 @@ kafka-streams-dlq-sample kafka-streams-table-join kafka-streams-global-table-join - kafka-streams-interactive-query-basic - kafka-streams-interactive-query-advanced kafka-streams-message-channel kafka-streams-product-tracker kafka-streams-aggregate @@ -26,6 +24,7 @@ kafka-streams-destination-pattern kafka-streams-jaas-security kafka-streams-multiple-input-topics + kafka-streams-interactive-query diff --git a/pom.xml b/pom.xml index 1c1f194..b80407d 100644 --- a/pom.xml +++ b/pom.xml @@ -19,8 +19,10 @@ + confluent-schema-registry-integration function-based-stream-app-samples kafka-batch-sample + kafka-binder-native-app kafka-e2e-kotlin-sample kafka-native-serialization kafka-security-samples @@ -31,6 +33,8 @@ multi-binder-samples partitioning-samples routing-samples + spring-cloud-stream-schema-registry-integration + stream-function-batch-producer-consumer testing-samples transaction-kafka-samples diff --git a/spring-cloud-stream-schema-registry-integration/README.adoc b/spring-cloud-stream-schema-registry-integration/README.adoc new file mode 100644 index 0000000..d410a6e --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/README.adoc @@ -0,0 +1,205 @@ +== Spring Cloud Stream and Schema Evolution in Action with Kafka Binder. + +:project-version: 4.0.0-SNAPSHOT + +These are a set of Spring Boot applications to demonstrate Schema Evolution using Spring Cloud Stream with Kafka binder. +Producer V1 (`producer1`), Producer V2 (`producer2`), and Consumer (`consumer`) are included in this project. + +=== Requirement +As a developer, I'd like to design my consumer to be resilient to differing payload schemas. + +=== Assumptions +For this demonstration, we will simply assume there are two producers producing events with different payload schemas. +A consumer that consumes both the payload versions will be designed to adapt to evolving schemas. + +Both producers and consumers interact with the SCSt schema registry to register and evolve the schema. + +[[build-apps]] +=== Building + +==== +NOTE: It is expected that this repo has been checked out locally and All commands are executed from this sample's directory `spring-cloud-stream-schema-registry-integration` unless otherwise noted. +==== + +==== Build the schema registry +The schema registry must be built by executing the following command: +[source,bash] +---- +pushd ../../schema-registry +../mvnw clean install +popd +---- + +==== Build the apps +To build the applications simply execute the following command: +[source,bash] +---- +./mvnw clean install +---- + +NOTE: The apps can be built and run from w/in an IDE (such as IntelliJ) but you will need to invoke the Maven `package` goal and then `refresh` the project as the Avro Maven plugin needs to execute so that it generates the required model classes - otherwise you will see compile failures around missing `Sensor` class. + +[[run-apps]] +=== Running + +==== +NOTE: It is expected that this repo has been checked out locally and All commands are executed from this sample's directory `spring-cloud-stream-schema-registry-integration` unless otherwise noted. +==== + +==== Pre-requisites +**** +* The components have all been built by following the <> steps. +* Apache Kafka broker available at `localhost:9092` + +TIP: The included link:../../../tools/kafka/docker-compose/README.adoc#_all_the_things[Kafka tools] can be used to easily start a broker locally on the required coordinates + +* By default, the schema registry is backed by an `H2` database. +** To instead use `Postgres` it must be available at `localhost:5432` +** To instead use `MySQL` it must be available at `localhost:3306`. + +TIP: Docker compose files are provided for both link:./postgres.yml[Postgres] and link:./mysql.yml[MySQL]. You can simply run `docker-compose -f ` to start/stop the database server +**** + +==== Steps +Make sure the above pre-requisites are satisfied and follow the steps below. + +===== Start Schema Registry +Start the Schema Registry server (_adjust commands accordingly if you are not on a Unix like platform_) +[source,bash,subs="attributes"] +---- +java -jar ../../schema-registry/spring-cloud-stream-schema-registry-server/target/spring-cloud-stream-schema-registry-server-4.0.0-SNAPSHOT.jar +---- + +By default the schema registry starts with a local `H2` database. + +.To use `Postgres` database instead of `H2`... +[%collapsible] +==== +additional properties must be specified when starting the server: +[source,bash,subs="attributes"] +---- +java -jar ../../schema-registry/spring-cloud-stream-schema-registry-server/target/spring-cloud-stream-schema-registry-server-{project-version}.jar \ + --spring.datasource.url=jdbc:postgresql://localhost:5432/registry \ + --spring.datasource.username=root \ + --spring.datasource.password=rootpw \ + --spring.datasource.driver-class-name=org.postgresql.Driver \ + --spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect \ + --spring.jpa.hibernate.ddl-auto-create=true \ + --spring.jpa.hibernate.ddl-auto=update \ + --spring.jpa.generate-ddl=true +---- +==== + + +.To use `MySQL` database instead of `H2`... +[%collapsible] +==== +additional properties must be specified when starting the server: +[source,bash,subs="attributes"] +---- +java -jar ../../schema-registry/spring-cloud-stream-schema-registry-server/target/spring-cloud-stream-schema-registry-server-{project-version}.jar \ + --spring.datasource.url=jdbc:mariadb://localhost:3306/registry \ + --spring.datasource.username=root \ + --spring.datasource.password=rootpw \ + --spring.datasource.driver-class-name=org.mariadb.jdbc.Driver \ + --spring.jpa.database-platform=org.hibernate.dialect.MariaDB53Dialect \ + --spring.jpa.hibernate.ddl-auto-create=true \ + --spring.jpa.hibernate.ddl-auto=update \ + --spring.jpa.generate-ddl=true +---- +==== + +===== Start consumer +Start `consumer` on another terminal session (or run it from an IDE) +[source,bash,subs="attributes"] +---- +java -jar schema-registry-consumer-kafka/target/schema-registry-consumer-kafka-{project-version}.jar +---- + +===== Start V1 producer +Start `producer1` on another terminal session (or run it from an IDE) +[source,bash,subs="attributes"] +---- +java -jar schema-registry-producer1-kafka/target/schema-registry-producer1-kafka-{project-version}.jar +---- + +===== Start V2 producer +Start `producer2` on another terminal session (or run it from an IDE) +[source,bash,subs="attributes"] +---- +java -jar schema-registry-producer2-kafka/target/schema-registry-producer2-kafka-{project-version}.jar +---- + +=== Sample Data +Both the producers in the demonstration are _also_ REST controllers. We will hit the `/randomMessage` endpoint on each producer +to POST sample data. + +_Example:_ +[source,bash] +---- +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9010/randomMessage +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9009/randomMessage +curl -X POST http://localhost:9010/randomMessage +---- + +=== Output +The consumer should log the results. + +[source,bash,options=nowrap,subs=attributes] +---- +{"id": "d5657e55-c2cd-48f0-a22e-d28d1ef10873-v1", "internalTemperature": 19.534815, "externalTemperature": 0.0, "acceleration": 5.286502, "velocity": 25.349945} +{"id": "6a6de265-997c-4bf9-8eae-97accccb78e9-v2", "internalTemperature": 39.443855, "externalTemperature": 40.365253, "acceleration": 1.8879288, "velocity": 2.5296867} +{"id": "f09defad-828f-43ae-93a4-e777754cf57a-v1", "internalTemperature": 15.895501, "externalTemperature": 0.0, "acceleration": 1.9341749, "velocity": 52.868507} +{"id": "b39b8c73-eec3-4abd-b8d2-cc405eb39bd7-v1", "internalTemperature": 44.90698, "externalTemperature": 0.0, "acceleration": 1.5393275, "velocity": 87.0358} +{"id": "19d5c20e-ec18-4b35-a82a-c8322d7fea27-v2", "internalTemperature": 19.203693, "externalTemperature": 47.290142, "acceleration": 1.125809, "velocity": 11.153614} +---- + +NOTE: Refer to the payload suffix in the `id` field. Each of them are appended with `-v1` or `-v2` indicating they are from +`producer1` and `producer2` respectively. + +=== What just happened? +The schema evolved on the `temperature` field. That field is now split into `internalTemperature` and `externalTemperature`, +as two separate fields. The `producer1` produces payload only with `temperature` and on the other hand, `producer2` produces +payload with `internalTemperature` and `externalTemperature` fields in it. + +The `consumer` is coded against a base schema that include the split fields. + +The `consumer` app can happily deserialize the payload with `internalTemperature` and `externalTemperature` fields. However, when +a `producer1` payload arrives (which includes `temperature` field), the schema evolution and compatibility check are automatically +applied. + +Because each payload also includes the payload version in the header, Spring Cloud Stream with the help of Schema Registry server and Avro, the schema evolution occurs behind the scenes. +The automatic mapping of `temperature` to `internalTemperature` field is applied, since that's the field where the `aliases` is defined. + +=== Using Confluent Schema Registry with Spring Cloud Stream Schema Registry AVRO Converter Clients + +In the examples above, we used Spring Cloud Stream Schema Registry Server with AVRO converter clients in Spring Cloud Stream. +What if you want to use these converters, but against Confluent Schema Registry Server? +In order to make it work, you need to provide a custom `SchemaRegistryClient` bean in your applications that knows how to interact with Confluent Schema Registry. + +``` +@Configuration +static class ConfluentSchemaRegistryConfiguration { + @Bean + public SchemaRegistryClient schemaRegistryClient(@Value("${spring.cloud.stream.schema-registry-client.endpoint:http://localhost:8081}") String endpoint){ + ConfluentSchemaRegistryClient client = new ConfluentSchemaRegistryClient(); + client.setEndpoint(endpoint); + return client; + } +} +``` + +As you can see, we are using a specific implementation of `SchemaRegistryClient` - `ConfluentSchemaRegistryClient`. + +You need to add this to both the producers and consumer applications. + +If you started Confluent Schema Registry server at a non-default server/port (`localhost:8081`), then you need to provide that using the following property. + +``` +spring.cloud.stream.schema-registry-client.endpoint +``` + +That's all there to it. +The same applications that previously interacted with Spring Cloud Stream Schema Registry server now interacts with the Confluent Schema Registry Server using the same set of AVRO message converters provided by Spring Cloud Stream Schema Registry. diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/mvnw b/spring-cloud-stream-schema-registry-integration/mvnw similarity index 99% rename from kafka-streams-samples/kafka-streams-interactive-query-basic/mvnw rename to spring-cloud-stream-schema-registry-integration/mvnw index 2d1e3cf..0ce08e9 100755 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/mvnw +++ b/spring-cloud-stream-schema-registry-integration/mvnw @@ -218,8 +218,9 @@ fi WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain -exec "$JAVACMD" \ +"$JAVACMD" \ $MAVEN_OPTS \ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" + diff --git a/spring-cloud-stream-schema-registry-integration/mvnw.cmd b/spring-cloud-stream-schema-registry-integration/mvnw.cmd new file mode 100644 index 0000000..7ecd01d --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/mvnw.cmd @@ -0,0 +1,145 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +set MAVEN_CMD_LINE_ARGS=%* + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/spring-cloud-stream-schema-registry-integration/mysql.yml b/spring-cloud-stream-schema-registry-integration/mysql.yml new file mode 100644 index 0000000..b3371c6 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/mysql.yml @@ -0,0 +1,13 @@ +version: '3' +services: + mysql: + image: mysql:5.7.25 + container_name: schema-registry-mysql + environment: + MYSQL_DATABASE: registry + MYSQL_USER: root + MYSQL_ROOT_PASSWORD: rootpw + expose: + - 3306 + ports: + - 3306:3306 diff --git a/spring-cloud-stream-schema-registry-integration/pom.xml b/spring-cloud-stream-schema-registry-integration/pom.xml new file mode 100644 index 0000000..57ac4af --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/pom.xml @@ -0,0 +1,165 @@ + + + 4.0.0 + spring-cloud-stream-schema-registry-integration + 4.1.0-SNAPSHOT + pom + spring-cloud-stream-schema-registry-integration + SCSt Schema Registry Integration Sample App + + + io.spring.cloud.stream.sample + spring-cloud-stream-samples-parent + 0.0.1-SNAPSHOT + .. + + + + schema-registry-producer1-kafka + schema-registry-producer2-kafka + schema-registry-consumer-kafka + + + + 1.11.0 + 7.0.1 + 2023.0.0-SNAPSHOT + + + + + + org.springframework.cloud + spring-cloud-stream-dependencies + ${project.version} + pom + import + + + + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-sources + + schema + protocol + idl-protocol + + + src/main/resources/avro + + + + + + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + org.springframework.cloud + spring-cloud-stream-schema-registry-client + + + org.apache.avro + avro + ${avro.version} + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream-test-binder + test + + + org.awaitility + awaitility + test + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + confluent + https://packages.confluent.io/maven/ + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + true + + + false + + + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + false + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release-local + + false + + + + + diff --git a/spring-cloud-stream-schema-registry-integration/postgres.yml b/spring-cloud-stream-schema-registry-integration/postgres.yml new file mode 100644 index 0000000..3662b43 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/postgres.yml @@ -0,0 +1,14 @@ +version: '3' +services: + postgres: + image: postgres:10 + container_name: schema-registry-postgres + restart: always + environment: + POSTGRES_DB: registry + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpw + expose: + - 5432 + ports: + - 5432:5432 diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/pom.xml b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/pom.xml new file mode 100644 index 0000000..033d6bd --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + schema-registry-consumer-kafka + 4.1.0-SNAPSHOT + jar + schema-registry-consumer-kafka + SCSt Schema Registry Consumer + + + org.springframework.cloud + spring-cloud-stream-schema-registry-integration + 4.1.0-SNAPSHOT + + + + + + org.apache.avro + avro-maven-plugin + + + + diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/java/sample/consumer/ConsumerApplication.java b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/java/sample/consumer/ConsumerApplication.java new file mode 100644 index 0000000..5e97d59 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/java/sample/consumer/ConsumerApplication.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.consumer; + +import java.util.function.Consumer; + +import com.example.Sensor; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.schema.registry.client.EnableSchemaRegistryClient; +import org.springframework.context.annotation.Bean; + +@SpringBootApplication(proxyBeanMethods = false) +@EnableSchemaRegistryClient +public class ConsumerApplication { + + private final Log logger = LogFactory.getLog(getClass()); + + public static void main(String[] args) { + SpringApplication.run(ConsumerApplication.class, args); + } + + @Bean + public Consumer process() { + return input -> logger.info("[INPUT-RECEIVED]: " + input); + } + +} diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/application.yml b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/application.yml new file mode 100644 index 0000000..0154e79 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/application.yml @@ -0,0 +1,12 @@ +spring: + cloud: + stream: + bindings: + process-in-0: + destination: sensor-topic + schema-registry-client: + endpoint: http://localhost:8990 + schema: + avro: + schema-locations: classpath:avro/sensor.avsc +server.port: 9999 diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/avro/sensor.avsc b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..95afdfc --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/main/resources/avro/sensor.avsc @@ -0,0 +1,12 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"internalTemperature", "type":"float", "default":0.0, "aliases":["temperature"]}, + {"name":"externalTemperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} \ No newline at end of file diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/test/java/sample/consumer/ConsumerApplicationTests.java b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/test/java/sample/consumer/ConsumerApplicationTests.java new file mode 100644 index 0000000..c181396 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-consumer-kafka/src/test/java/sample/consumer/ConsumerApplicationTests.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.consumer; + +import com.example.Sensor; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.util.MimeType; + +import java.time.Duration; +import java.util.Random; +import java.util.UUID; + +@ExtendWith(OutputCaptureExtension.class) +public class ConsumerApplicationTests { + + private static final int AWAIT_DURATION = 10; + + @Test + void test(CapturedOutput output) { + try (ConfigurableApplicationContext context = + new SpringApplicationBuilder( + TestChannelBinderConfiguration + .getCompleteConfiguration(ConsumerApplication.class)) + .web(WebApplicationType.NONE) + .run( + "--spring.cloud.schema.avro.ignore-schema-registry-server=true" + ) + ) { + + StreamBridge streamBridge = context.getBean("streamBridgeUtils", StreamBridge.class); + Random random = new Random(); + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID() + "-v1"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + MimeType mimeType = new MimeType("application", "*+avro"); + streamBridge.send("process-in-0", sensor, mimeType); + + Awaitility.await().atMost(Duration.ofSeconds(AWAIT_DURATION)) + .until(() -> output.toString().contains("[INPUT-RECEIVED]: {\"id\":")); + } + } +} diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw new file mode 100755 index 0000000..0ce08e9 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw @@ -0,0 +1,226 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +echo $MAVEN_PROJECTBASEDIR +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +"$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" + diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw.cmd b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw.cmd new file mode 100644 index 0000000..7ecd01d --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/mvnw.cmd @@ -0,0 +1,145 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +set MAVEN_CMD_LINE_ARGS=%* + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/pom.xml b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/pom.xml new file mode 100644 index 0000000..9959d61 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + schema-registry-producer1-kafka + 4.1.0-SNAPSHOT + jar + schema-registry-producer1-kafka + SCSt Schema Registry Producer 1 + + + org.springframework.cloud + spring-cloud-stream-schema-registry-integration + 4.1.0-SNAPSHOT + + + + + + org.apache.avro + avro-maven-plugin + + + + diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/java/sample/producer1/Producer1ApplicationKafka.java b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/java/sample/producer1/Producer1ApplicationKafka.java new file mode 100644 index 0000000..8e40bd7 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/java/sample/producer1/Producer1ApplicationKafka.java @@ -0,0 +1,61 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.producer1; + +import java.util.Random; +import java.util.UUID; + +import com.example.Sensor; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.registry.client.EnableSchemaRegistryClient; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@SpringBootApplication(proxyBeanMethods = false) +@EnableSchemaRegistryClient +@RestController +public class Producer1ApplicationKafka { + + private final Random random = new Random(); + + @Autowired + private StreamBridge streamBridge; + + public static void main(String[] args) { + SpringApplication.run(Producer1ApplicationKafka.class, args); + } + + @RequestMapping(value = "/randomMessage", method = RequestMethod.POST) + public String sendRandomMessage() { + streamBridge.send("supplier-out-0", randomSensor()); + return "ok, have fun with v1 payload!"; + } + + private Sensor randomSensor() { + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID() + "-v1"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + sensor.setTemperature(random.nextFloat() * 50); + return sensor; + } +} diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/application.yml b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/application.yml new file mode 100644 index 0000000..99e0275 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/application.yml @@ -0,0 +1,13 @@ +spring: + cloud: + stream: + bindings: + supplier-out-0: + contentType: application/*+avro + destination: sensor-topic + schema-registry-client: + endpoint: http://localhost:8990 + schema: + avro: + schema-locations: classpath:avro/sensor.avsc +server.port: 9009 \ No newline at end of file diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/avro/sensor.avsc b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..c0e060d --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer1-kafka/src/main/resources/avro/sensor.avsc @@ -0,0 +1,11 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"temperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/pom.xml b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/pom.xml new file mode 100644 index 0000000..08d9372 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + schema-registry-producer2-kafka + 4.1.0-SNAPSHOT + jar + schema-registry-producer2-kafka + SCSt Schema Registry Producer 2 + + + org.springframework.cloud + spring-cloud-stream-schema-registry-integration + 4.1.0-SNAPSHOT + + + + + + org.apache.avro + avro-maven-plugin + + + + diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/java/sample/producer2/Producer2ApplicationKafka.java b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/java/sample/producer2/Producer2ApplicationKafka.java new file mode 100644 index 0000000..6c3225c --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/java/sample/producer2/Producer2ApplicationKafka.java @@ -0,0 +1,68 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.producer2; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.cloud.stream.schema.registry.client.EnableSchemaRegistryClient; +import org.springframework.context.annotation.Bean; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collections; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Supplier; + +import com.example.Sensor; + +@SpringBootApplication(proxyBeanMethods = false) +@EnableSchemaRegistryClient +@RestController +public class Producer2ApplicationKafka { + + private final Random random = new Random(); + + @Autowired + private StreamBridge streamBridge; + + public static void main(String[] args) { + SpringApplication.run(Producer2ApplicationKafka.class, args); + } + + @PostMapping("/randomMessage") + public String sendRandomMessage() { + streamBridge.send("supplier-out-0", randomSensor()); + return "ok, have fun with v2 payload!"; + } + + private Sensor randomSensor() { + Sensor sensor = new Sensor(); + sensor.setId(UUID.randomUUID().toString() + "-v2"); + sensor.setAcceleration(random.nextFloat() * 10); + sensor.setVelocity(random.nextFloat() * 100); + sensor.setInternalTemperature(random.nextFloat() * 50); + sensor.setExternalTemperature(random.nextFloat() * 50); + return sensor; + } +} diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/application.yml b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/application.yml new file mode 100644 index 0000000..2519a80 --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/application.yml @@ -0,0 +1,13 @@ +spring: + cloud: + stream: + bindings: + supplier-out-0: + contentType: application/*+avro + destination: sensor-topic + schema-registry-client: + endpoint: http://localhost:8990 + schema: + avro: + schema-locations: classpath:avro/sensor.avsc +server.port: 9010 \ No newline at end of file diff --git a/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/avro/sensor.avsc b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/avro/sensor.avsc new file mode 100644 index 0000000..5da668b --- /dev/null +++ b/spring-cloud-stream-schema-registry-integration/schema-registry-producer2-kafka/src/main/resources/avro/sensor.avsc @@ -0,0 +1,12 @@ +{ + "namespace" : "com.example", + "type" : "record", + "name" : "Sensor", + "fields" : [ + {"name":"id","type":"string"}, + {"name":"internalTemperature", "type":"float", "default":0.0, "aliases":["temperature"]}, + {"name":"externalTemperature", "type":"float", "default":0.0}, + {"name":"acceleration", "type":"float","default":0.0}, + {"name":"velocity","type":"float","default":0.0} + ] +} diff --git a/stream-function-batch-producer-consumer/README.adoc b/stream-function-batch-producer-consumer/README.adoc new file mode 100644 index 0000000..9f05b50 --- /dev/null +++ b/stream-function-batch-producer-consumer/README.adoc @@ -0,0 +1,56 @@ +== Spring Function Batch Producer Consumer + +This sample application demonstrates a Spring Cloud Stream functional interface implementation of batch consuming and batch producing messages. + +=== Application + +The application consists of a supplier that is generating a random UUID every second and a function that is configured through the kafka binder to wait 5 seconds until polling which then removes all integers from the UUID. +The UUID is then produced as part of a batch, finally reaching a consumer that simply logs the digit-less UUID. + +The application comes with a standard configuration properties yaml file that activates all the functions in the application. +It then specifies the necessary destinations for all the bindings. +Some bindings rely on the default binding destinations (see the `application.yml` for details). +In addition, the application expects Kafka to be available on `localhost:9092`. +If that is not the case, please ensure to update that in `application.yml`. + +[[build-app]] +=== Building +To build the app simply execute the following command in the base directory: +[source,bash] +---- +./mvnw clean install +---- + +=== Running + +==== Ensure these pre-requisites +**** +* The app has been built by following the <> steps +* Apache Kafka broker available at `localhost:9092` + +**** + +==== Start the streams app +[source,bash] +---- +java -jar target/stream-function-batch-producer-consumer-0.0.1-SNAPSHOT.jar +---- + +The application should automatically begin executing. All messages are printed in real-time as received by different parts of the application. + +The output should look like the following: +[source,bash] +---- +5cce1056-34fc-4b7e-aadf-88c37eaa8a82 -> batch-in +3dc2e76d-d483-4cf8-98b8-44320124920a -> batch-in +6c00d120-0aa7-4373-bf3d-3bdbacb11adb -> batch-in +b4e8a7ff-19a7-44aa-8900-619caa78e885 -> batch-in +1626e7e6-87ba-4faa-97bf-a229e538bf7a -> batch-in +Removed digits from batch of 5 +batch-out -> cce-fc-be-aadf-ceaaa +batch-out -> dced-d-cf-b-a +batch-out -> cd-aa--bfd-bdbacbadb +batch-out -> beaff-a-aa--caae +batch-out -> ee-ba-faa-bf-aebfa +---- + diff --git a/stream-function-batch-producer-consumer/mvnw b/stream-function-batch-producer-consumer/mvnw new file mode 100755 index 0000000..a16b543 --- /dev/null +++ b/stream-function-batch-producer-consumer/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/stream-function-batch-producer-consumer/mvnw.cmd b/stream-function-batch-producer-consumer/mvnw.cmd new file mode 100644 index 0000000..c8d4337 --- /dev/null +++ b/stream-function-batch-producer-consumer/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/kafka-streams-samples/kafka-streams-interactive-query-basic/pom.xml b/stream-function-batch-producer-consumer/pom.xml similarity index 64% rename from kafka-streams-samples/kafka-streams-interactive-query-basic/pom.xml rename to stream-function-batch-producer-consumer/pom.xml index 4d55725..8b69c8d 100644 --- a/kafka-streams-samples/kafka-streams-interactive-query-basic/pom.xml +++ b/stream-function-batch-producer-consumer/pom.xml @@ -1,26 +1,52 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - kafka-streams-interactive-query-basic - 0.0.1-SNAPSHOT - jar - - kafka-streams-interactive-query-basic - Spring Cloud Stream sample for KStream interactive queries - org.springframework.boot spring-boot-starter-parent - 2.4.4 + 3.2.0-SNAPSHOT - + com.example + stream-function-batch-producer-consumer + 0.0.1 + stream-function-batch-producer-consumer + Sample app using stream functions with batch producer and consumer using kafka - 2020.0.2 + 17 + 2023.0.0-SNAPSHOT + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + org.springframework.kafka + spring-kafka + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream-test-binder + test + + + org.springframework.kafka + spring-kafka-test + test + + @@ -33,45 +59,18 @@ - - - org.springframework.cloud - spring-cloud-stream-binder-kafka-streams - 3.1.3-SNAPSHOT - - - org.springframework.boot - spring-boot-starter-test - test - - - org.springframework.kafka - spring-kafka-test - test - - - org.apache.kafka - kafka-streams-test-utils - ${kafka.version} - test - - - - org.springframework.boot - spring-boot-starter-actuator - - - org.springframework.boot - spring-boot-starter - - - org.springframework.boot - spring-boot-starter-web - - - + + maven-deploy-plugin + + true + + + + org.graalvm.buildtools + native-maven-plugin + org.springframework.boot spring-boot-maven-plugin @@ -99,7 +98,20 @@ false + + spring-releases + Spring Releases + https://repo.spring.io/release + + false + + + + confluent + https://packages.confluent.io/maven/ + + spring-snapshots @@ -130,4 +142,5 @@ + diff --git a/stream-function-batch-producer-consumer/src/main/java/sample/stream/batch/CloudStreamsFunctionBatch.java b/stream-function-batch-producer-consumer/src/main/java/sample/stream/batch/CloudStreamsFunctionBatch.java new file mode 100644 index 0000000..d9b4918 --- /dev/null +++ b/stream-function-batch-producer-consumer/src/main/java/sample/stream/batch/CloudStreamsFunctionBatch.java @@ -0,0 +1,68 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sample.stream.batch; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; + +import java.util.List; +import java.util.UUID; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * @author Steven Gantz + */ +@SpringBootApplication +public class CloudStreamsFunctionBatch { + + public static void main(String[] args) { + SpringApplication.run(CloudStreamsFunctionBatch.class, args); + } + + @Bean + public Supplier stringSupplier() { + return () -> { + var uuid = UUID.randomUUID(); + System.out.println(uuid + " -> batch-in"); + return uuid; + }; + } + + @Bean + public Function, List>> digitRemovingConsumer() { + return idBatch -> { + System.out.println("Removed digits from batch of " + idBatch.size()); + return idBatch.stream() + .map(UUID::toString) + // Remove all digits from the UUID + .map(uuid -> uuid.replaceAll("\\d","")) + .map(noDigitString -> MessageBuilder.withPayload(noDigitString).build()) + .toList(); + }; + } + + @KafkaListener(id = "batch-out", topics = "batch-out") + public void listen(String in) { + System.out.println("batch-out -> " + in); + } + +} diff --git a/stream-function-batch-producer-consumer/src/main/resources/application.yml b/stream-function-batch-producer-consumer/src/main/resources/application.yml new file mode 100644 index 0000000..21c6169 --- /dev/null +++ b/stream-function-batch-producer-consumer/src/main/resources/application.yml @@ -0,0 +1,26 @@ +spring: + cloud: + function: + definition: stringSupplier;digitRemovingConsumer + stream: + bindings: + stringSupplier-out-0: + destination: batch-in + digitRemovingConsumer-in-0: + destination: batch-in + group: batch-in + consumer: + batch-mode: true + digitRemovingConsumer-out-0: + destination: batch-out + kafka: + binder: + brokers: localhost:9092 + bindings: + digitRemovingConsumer-in-0: + consumer: + configuration: + # Forces consumer to wait 5 seconds before polling for messages + fetch.max.wait.ms: 5000 + fetch.min.bytes: 1000000000 + max.poll.records: 10000000